diff --git a/.gitignore b/.gitignore index 99c30f52..e6555d1a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # Python-generated files __pycache__/ *.py[oc] +.pytest_cache/ build/ dist/ wheels/ @@ -16,8 +17,18 @@ queue/ CLAUDE.md AGENTS.md +# Prompt audit output (generated by tests) +component_system/prompt_audit/ + # Experimental code/artifacts dev/ # Results file results.tsv + +# Component-system runtime artifacts (logs, queue, state, worktrees under history/) +component_system/history/ +component_system/baseline_branches.json +component_system/baseline_metrics.json +*.log +.ipynb_checkpoints/ \ No newline at end of file diff --git a/README.md b/README.md index 2bc30516..15ee32f5 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,8 @@ Hi have a look at program.md and let's kick off a new experiment! let's do the s The `program.md` file is essentially a super lightweight "skill". +For the component-system workflow, see `component_system/README.md`. + ## Project structure ``` diff --git a/component_system/PDCA-DO-CHECK-ACTION.md b/component_system/PDCA-DO-CHECK-ACTION.md new file mode 100644 index 00000000..d90418c9 --- /dev/null +++ b/component_system/PDCA-DO-CHECK-ACTION.md @@ -0,0 +1,75 @@ +# DCA — Do, Check, Action + +## Responsibility +Take the generated plan from P, adapt/fix it in the seed worktree, +run the canonical training entrypoint, evaluate results against baseline, and +promote only when the signal is positive. Do not propose new ideas or optimize for better metrics; only adapt/fix so the plan runs and report outcomes. + +## Workspace and paths +**CWD = seed worktree.** Read and edit only inside it; use relative paths only. Treat `component_system/` in the worktree as canonical context. + +## Input +- Runner prompt (task content). +- Baseline: `component_system/baseline_branches.json`, `component_system/baseline_metrics.json`. +- Worktree-local files only. + +## Baseline measurement (seed_id __baseline__) +Retry until the run succeeds and you report real metrics. No empty metrics. + +- **OOM:** Reduce `device_batch_size` in `component_system/components/trainer.py` (default 128); keep `total_batch_size % (device_batch_size * sequence_length) == 0`. Rerun until training completes. +- Only trivial fixes (e.g. batch size); no model/training logic changes. +- **Commit before reporting.** Uncommitted changes break the follow-up merge. + +## Workflow +1. Work in the seed worktree (one branch per seed). +2. Adapt/fix until it runs (runtime only: bugs, OOM, imports, config; no model/hyperparameter/training-logic changes for better metrics). +3. Run canonical command (**≥900s**): `timeout 900 uv run --active component_system/entrypoint.py > training.log 2>&1` (or `... 2>&1 | tee training.log` to also see output). **Must set command/tool timeout ≥900s**. After the run, inspect `training.log` to confirm completion and recover or verify metrics. +4. On bug/OOM: fix and rerun; for baseline, retry until success. +5. Commit on seed branch before reporting. +6. Print DCA summary block with `commit_sha` in JSON. +7. Runner evaluates signal and handles promotion. + +## Output Format +Print the summary block. Put metrics in JSON; runner falls back to stdout/stderr parsing if missing. + +```text +AUTORESEARCH_DCA_SUMMARY_BEGIN +{"checks":["entrypoint"],"notes":"...","completed_at":"YYYY-MM-DD HH:MM:SS","commit_sha":"git sha","metrics":{"val_bpb":1.24,...}} +AUTORESEARCH_DCA_SUMMARY_END +``` + +If no final metrics, use `"metrics": {}`. Runner extracts from stdout/stderr: `val_bpb`, `training_seconds`, `total_seconds`, `peak_vram_mb`, `mfu_percent`, `total_tokens_M`, `num_steps`, `num_params_M`, `depth`. No metrics → recovery DCA inspects logs; only then treat as failed. + +## Check: Signal Rules + +| Condition | Signal | +|-----------|--------| +| `val_bpb` drops >= 0.001 vs baseline | `positive_signal` | +| `val_bpb` rises >= 0.001 vs baseline | `negative_signal` | +| difference < 0.001 | `neutral` | +| no historical baseline (best_val_bpb) | `positive_signal` (first recording) | +| metrics missing or training error | `error` | + +The threshold is defined in `component_system/config.py` (`PROMOTION_THRESHOLD`). + +## Action: Promotion Rules + +Only DCA may trigger a merge into baseline; P must not. Runner records `commit_sha`; on positive signal the workflow merges seed into baseline first, then updates metrics/state. Merge conflict → system queues merge-resolution DCA. + +### Promotion (`positive_signal`) +1. System merges seed into baseline first (you do not run merge). +2. Workflow updates `baseline_metrics.json` / `baseline_branches.json`. +3. Metadata in seed/run state. + +### Merge failure +- **Normal seed:** In seed worktree: `git merge __baseline__`, resolve conflicts, commit, print DCA summary for retry. +- **Baseline seed (__baseline__):** Merge __baseline__ into target (e.g. master). Run from worktree that has target checked out (`git worktree list`); do not run from __baseline__ worktree or `git merge master` there. + +### Non-promotion +`neutral` / `negative_signal` / `error`: log only. Failure info in queue/state logs. + +## Constraints +- No model/optimizer/training-logic changes for better metrics; only make the plan run (bugs, OOM, etc.). +- Use `run_mainline_training` (or equivalent); do not skip `val_bpb` evaluation. +- Do not edit baseline JSON files; only DCA promotion updates them. +- Canonical runner: `component_system/entrypoint.py`. Traceability: git + state files. diff --git a/component_system/PDCA-PLAN.md b/component_system/PDCA-PLAN.md new file mode 100644 index 00000000..7564ffeb --- /dev/null +++ b/component_system/PDCA-PLAN.md @@ -0,0 +1,61 @@ +# P - Seed Planning And Generation + +## Responsibility +Extract exactly one testable improvement hypothesis from the seed prompt, +generate the first implementation in a candidate worktree, and hand the result +to DCA through the runner. + +## Workspace and paths +**CWD = seed worktree.** Read and edit only inside it; use relative paths only. + +## arXiv search (CLI) + +Run from repo root with uv (e.g. `uv run python component_system/run_arxiv.py ...`); arxiv is already a project dependency. + +### Search (CLI script) + +From repo root, use the script in this component: + +```bash +uv run python component_system/run_arxiv.py --query "machine learning" --max-results 5 +uv run python component_system/run_arxiv.py --id 1605.08386v1 --output json +``` + +**CLI arguments:** `--query` / `-q`, `--id` (one or more arXiv IDs; overrides query), `--max-results` / `-n`, `--sort-by` (relevance | submittedDate | lastUpdatedDate), `--sort-order` (ascending | descending), `--output` / `-o` (text | json), `--download-dir`, `--verbose` / `-v`. + +### Hypothesis from results +1. Read abstracts; pick one concrete change (not just a concept). +2. Map to component: `model`, `optimizer`, or `trainer`. +3. State expected benefit; reduce to one isolated, evaluable improvement. + +## Input +- **results.tsv** in cwd (if present) ? read first to avoid duplicating tried/discarded ideas. +- arXiv via arxiv-search; past failures in `queue/done/`; manual seed files. + +## One-Improvement Rule + +One seed = one hypothesis = one causal change. Do not bundle ideas. If the prompt has several options, pick the single best for this run. Prefer the smallest coherent change that tests the hypothesis. + +**Good:** one optimizer schedule change; one architectural block; one training heuristic. **Bad:** model + optimizer + batch together; multiple paper ideas in one seed; "cleanup + new feature" in one candidate. + +## Output Format +Print a summary block for the runner: +```text +AUTORESEARCH_P_SUMMARY_BEGIN +{"idea":"short title","target_component":"model | optimizer | trainer","description":"change details, hypothesis, expected benefit","source_refs":["arXiv:"],"commit_sha":"git sha","completed_at":"YYYY-MM-DD HH:MM:SS"} +AUTORESEARCH_P_SUMMARY_END +``` + +## Runner / worktree +Before each P run, the runner syncs the seed worktree with its baseline branch (merge baseline into seed) so P always starts from the latest baseline. + +## Steps +1. Read `results.tsv` if present. +2. Refine prompt ? one concrete idea ? one isolated improvement; name target component. +3. Implement in worktree (from baseline); commit on seed branch. +4. Print summary block (runner records commit). Description must be enough for DCA. + +## Constraints +- One component, one improvement per seed. Smallest viable implementation. +- No exploratory cleanup or opportunistic refactors unless required for the one change. +- Commit on seed branch; runner does not merge. **P must never merge;** only DCA triggers merge into baseline. diff --git a/component_system/README.md b/component_system/README.md new file mode 100644 index 00000000..baa4f3a9 --- /dev/null +++ b/component_system/README.md @@ -0,0 +1,99 @@ +# autoresearch + +![teaser](progress.png) + +*One day, frontier AI research used to be done by meat computers in between eating, sleeping, having other fun, and synchronizing once in a while using sound wave interconnect in the ritual of "group meeting". That era is long gone. Research is now entirely the domain of autonomous swarms of AI agents running across compute cluster megastructures in the skies. The agents claim that we are now in the 10,205th generation of the code base, in any case no one could tell if that's right or wrong as the "code" is now a self-modifying binary that has grown beyond human comprehension. This repo is the story of how it all began. -@karpathy, March 2026*. + +The idea: give an AI agent a small but real LLM training setup and let it experiment autonomously overnight. It modifies the code, trains for 5 minutes, checks if the result improved, keeps or discards, and repeats. You wake up in the morning to a log of experiments and (hopefully) a better model. The training code here is a simplified single-GPU implementation of [nanochat](https://github.com/karpathy/nanochat). The core idea is that you're not touching any of the Python files like you normally would as a researcher. Instead, you are programming the `program.md` Markdown files that provide context to the AI agents and set up your autonomous research org. The default `program.md` in this repo is intentionally kept as a bare bones baseline, though it's obvious how one would iterate on it over time to find the "research org code" that achieves the fastest research progress, how you'd add more agents to the mix, etc. A bit more context on this project is here in this [tweet](https://x.com/karpathy/status/2029701092347630069). + +## How it works + +The repo is deliberately kept small and only really has a three files that matter: + +- **`prepare.py`** — fixed constants, one-time data prep (downloads training data, trains a BPE tokenizer), and runtime utilities (dataloader, evaluation). Not modified. +- **`train.py`** — the single file the agent edits. Contains the full GPT model, optimizer (Muon + AdamW), and training loop. Everything is fair game: architecture, hyperparameters, optimizer, batch size, etc. **This file is edited and iterated on by the agent**. +- **`program.md`** — baseline instructions for one agent. Point your agent here and let it go. **This file is edited and iterated on by the human**. + +By design, training runs for a **fixed 5-minute time budget** (wall clock, excluding startup/compilation), regardless of the details of your compute. The metric is **val_bpb** (validation bits per byte) — lower is better, and vocab-size-independent so architectural changes are fairly compared. + +## Quick start + +**Requirements:** A single NVIDIA GPU (tested on H100), Python 3.10+, [uv](https://docs.astral.sh/uv/). + +```bash + +# 1. Install uv project manager (if you don't already have it) +curl -LsSf https://astral.sh/uv/install.sh | sh + +# 2. Install dependencies +uv sync + +# 3. Download data and train tokenizer (one-time, ~2 min) +uv run prepare.py + +# 4. Manually run a single training experiment (~5 min) +uv run train.py +``` + +If the above commands all work ok, your setup is working and you can go into autonomous research mode. + +## Running the agent + +Simply spin up your Claude/Codex or whatever you want in this repo (and disable all permissions), then you can prompt something like: + +``` +Hi have a look at program.md and let's kick off a new experiment! let's do the setup first. +``` + +The `program.md` file is essentially a super lightweight "skill". + +### Component-system workflow + +**Seed → P → DCA** loop: daemon runs two workers that poll a file queue and dispatch to an external agent (Claude, Codex, or OpenCode). + +1. **Dashboard** (optional): `uv run uvicorn component_system.web.app:app --reload` → http://127.0.0.1:8000/component-system +2. **Daemon:** `uv run component_system/run.py` (or `PDCA_AGENT=codex|opencode` for other backends) +3. **Bootstrap:** Have the agent follow `component_system/protocol.md`, create a seed and queue it for P, then start the daemon. Do not run P/DCA stages manually in-session. + +Seeds flow: `queue/p/` → P → `queue/dca/` → DCA → `state/`. Results in dashboard. + +## Project structure + +``` +prepare.py — constants, data prep + runtime utilities (do not modify) +train.py — model, optimizer, training loop (agent modifies this) +program.md — agent instructions +pyproject.toml — dependencies +``` + +## Design choices + +- **Single file to modify.** The agent only touches `train.py`. This keeps the scope manageable and diffs reviewable. +- **Fixed time budget.** Training always runs for exactly 5 minutes, regardless of your specific platform. This means you can expect approx 12 experiments/hour and approx 100 experiments while you sleep. There are two upsides of this design decision. First, this makes experiments directly comparable regardless of what the agent changes (model size, batch size, architecture, etc). Second, this means that autoresearch will find the most optimal model for your platform in that time budget. The downside is that your runs (and results) become not comparable to other people running on other compute platforms. +- **Self-contained.** No external dependencies beyond PyTorch and a few small packages. No distributed training, no complex configs. One GPU, one file, one metric. + +## Platform support + +This code currently requires that you have a single NVIDIA GPU. In principle it is quite possible to support CPU, MPS and other platforms but this would also bloat the code. I'm not 100% sure that I want to take this on personally right now. People can reference (or have their agents reference) the full/parent nanochat repository that has wider platform support and shows the various solutions (e.g. a Flash Attention 3 kernels fallback implementation, generic device support, autodetection, etc.), feel free to create forks or discussions for other platforms and I'm happy to link to them here in the README in some new notable forks section or etc. + +Seeing as there seems to be a lot of interest in tinkering with autoresearch on much smaller compute platforms than an H100, a few extra words. If you're going to try running autoresearch on smaller computers (Macbooks etc.), I'd recommend one of the forks below. On top of this, here are some recommendations for how to tune the defaults for much smaller models for aspiring forks: + +1. To get half-decent results I'd use a dataset with a lot less entropy, e.g. this [TinyStories dataset](https://huggingface.co/datasets/karpathy/tinystories-gpt4-clean). These are GPT-4 generated short stories. Because the data is a lot narrower in scope, you will see reasonable results with a lot smaller models (if you try to sample from them after training). +2. You might experiment with decreasing `vocab_size`, e.g. from 8192 down to 4096, 2048, 1024, or even - simply byte-level tokenizer with 256 possibly bytes after utf-8 encoding. +3. In `prepare.py`, you'll want to lower `MAX_SEQ_LEN` a lot, depending on the computer even down to 256 etc. As you lower `MAX_SEQ_LEN`, you may want to experiment with increasing `DEVICE_BATCH_SIZE` in `train.py` slightly to compensate. The number of tokens per fwd/bwd pass is the product of these two. +4. Also in `prepare.py`, you'll want to decrease `EVAL_TOKENS` so that your validation loss is evaluated on a lot less data. +5. In `train.py`, the primary single knob that controls model complexity is the `DEPTH` (default 8, here). A lot of variables are just functions of this, so e.g. lower it down to e.g. 4. +6. You'll want to most likely use `WINDOW_PATTERN` of just "L", because "SSSL" uses alternating banded attention pattern that may be very inefficient for you. Try it. +7. You'll want to lower `TOTAL_BATCH_SIZE` a lot, but keep it powers of 2, e.g. down to `2**14` (~16K) or so even, hard to tell. + +I think these would be the reasonable hyperparameters to play with. Ask your favorite coding agent for help and copy paste them this guide, as well as the full source code. + +## Notable forks + +- [miolini/autoresearch-macos](https://github.com/miolini/autoresearch-macos) (MacOS) +- [trevin-creator/autoresearch-mlx](https://github.com/trevin-creator/autoresearch-mlx) (MacOS) +- [jsegov/autoresearch-win-rtx](https://github.com/jsegov/autoresearch-win-rtx) (Windows) + +## License + +MIT diff --git a/component_system/components/model.py b/component_system/components/model.py new file mode 100644 index 00000000..f74d8938 --- /dev/null +++ b/component_system/components/model.py @@ -0,0 +1,380 @@ +from __future__ import annotations + +from dataclasses import dataclass + +import torch +import torch.nn as nn +import torch.nn.functional as F +from kernels import get_kernel + +from prepare import MAX_SEQ_LEN + + +def _get_fa3(): + if torch.cuda.is_available(): + cap = torch.cuda.get_device_capability() + repo = "varunneal/flash-attention-3" if cap == (9, 0) else "kernels-community/flash-attn3" + return get_kernel(repo).flash_attn_interface + return None + +_fa3 = None + +def get_fa3(): + global _fa3 + if _fa3 is None: + _fa3 = _get_fa3() + return _fa3 + + +@dataclass +class GPTConfig: + sequence_len: int = 2048 + vocab_size: int = 32768 + n_layer: int = 12 + n_head: int = 6 + n_kv_head: int = 6 + n_embd: int = 768 + window_pattern: str = "SSSL" + + +def norm(x: torch.Tensor) -> torch.Tensor: + return F.rms_norm(x, (x.size(-1),)) + + +def has_ve(layer_idx: int, n_layer: int) -> bool: + return layer_idx % 2 == (n_layer - 1) % 2 + + +def apply_rotary_emb(x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor: + assert x.ndim == 4 + d = x.shape[3] // 2 + x1, x2 = x[..., :d], x[..., d:] + y1 = x1 * cos + x2 * sin + y2 = x1 * (-sin) + x2 * cos + return torch.cat([y1, y2], 3) + + +class CausalSelfAttention(nn.Module): + def __init__(self, config: GPTConfig, layer_idx: int) -> None: + super().__init__() + self.n_head = config.n_head + self.n_kv_head = config.n_kv_head + self.n_embd = config.n_embd + self.head_dim = self.n_embd // self.n_head + assert self.n_embd % self.n_head == 0 + assert self.n_kv_head <= self.n_head and self.n_head % self.n_kv_head == 0 + self.c_q = nn.Linear(self.n_embd, self.n_head * self.head_dim, bias=False) + self.c_k = nn.Linear(self.n_embd, self.n_kv_head * self.head_dim, bias=False) + self.c_v = nn.Linear(self.n_embd, self.n_kv_head * self.head_dim, bias=False) + self.c_proj = nn.Linear(self.n_embd, self.n_embd, bias=False) + self.ve_gate_channels = 32 + self.ve_gate = ( + nn.Linear(self.ve_gate_channels, self.n_kv_head, bias=False) + if has_ve(layer_idx, config.n_layer) + else None + ) + + def forward( + self, + x: torch.Tensor, + ve: torch.Tensor | None, + cos_sin: tuple[torch.Tensor, torch.Tensor], + window_size: tuple[int, int], + ) -> torch.Tensor: + batch_size, seq_len, _ = x.size() + q = self.c_q(x).view(batch_size, seq_len, self.n_head, self.head_dim) + k = self.c_k(x).view(batch_size, seq_len, self.n_kv_head, self.head_dim) + v = self.c_v(x).view(batch_size, seq_len, self.n_kv_head, self.head_dim) + + # Value residual (ResFormer): mix in value embedding with input-dependent gate per head + if ve is not None: + ve = ve.view(batch_size, seq_len, self.n_kv_head, self.head_dim) + gate = 2 * torch.sigmoid(self.ve_gate(x[..., : self.ve_gate_channels])) + v = v + gate.unsqueeze(-1) * ve + + cos, sin = cos_sin + q, k = apply_rotary_emb(q, cos, sin), apply_rotary_emb(k, cos, sin) + q, k = norm(q), norm(k) + + fa3 = get_fa3() + if fa3 is None: + raise RuntimeError("Flash Attention 3 is unavailable; component_system model should match train.py and requires the same kernel path.") + y = fa3.flash_attn_func(q, k, v, causal=True, window_size=window_size) + y = y.contiguous().view(batch_size, seq_len, -1) + return self.c_proj(y) + + +class MLP(nn.Module): + def __init__(self, config: GPTConfig) -> None: + super().__init__() + self.c_fc = nn.Linear(config.n_embd, 4 * config.n_embd, bias=False) + self.c_proj = nn.Linear(4 * config.n_embd, config.n_embd, bias=False) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = self.c_fc(x) + x = F.relu(x).square() + x = self.c_proj(x) + return x + + +class Block(nn.Module): + def __init__(self, config: GPTConfig, layer_idx: int) -> None: + super().__init__() + self.attn = CausalSelfAttention(config, layer_idx) + self.mlp = MLP(config) + + def forward( + self, + x: torch.Tensor, + ve: torch.Tensor | None, + cos_sin: tuple[torch.Tensor, torch.Tensor], + window_size: tuple[int, int], + ) -> torch.Tensor: + x = x + self.attn(norm(x), ve, cos_sin, window_size) + x = x + self.mlp(norm(x)) + return x + + +class GPT(nn.Module): + def __init__(self, config: GPTConfig) -> None: + super().__init__() + self.config = config + self.window_sizes = self._compute_window_sizes(config) + self.transformer = nn.ModuleDict( + { + "wte": nn.Embedding(config.vocab_size, config.n_embd), + "h": nn.ModuleList([Block(config, i) for i in range(config.n_layer)]), + } + ) + self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False) + self.resid_lambdas = nn.Parameter(torch.ones(config.n_layer)) + self.x0_lambdas = nn.Parameter(torch.zeros(config.n_layer)) + head_dim = config.n_embd // config.n_head + kv_dim = config.n_kv_head * head_dim + self.value_embeds = nn.ModuleDict( + { + str(i): nn.Embedding(config.vocab_size, kv_dim) + for i in range(config.n_layer) + if has_ve(i, config.n_layer) + } + ) + self.rotary_seq_len = config.sequence_len * 10 + cos, sin = self._precompute_rotary_embeddings(self.rotary_seq_len, head_dim) + self.register_buffer("cos", cos, persistent=False) + self.register_buffer("sin", sin, persistent=False) + + @torch.no_grad() + def init_weights(self) -> None: + torch.nn.init.normal_(self.transformer.wte.weight, mean=0.0, std=1.0) + torch.nn.init.normal_(self.lm_head.weight, mean=0.0, std=0.001) + n_embd = self.config.n_embd + scale = 3**0.5 * n_embd**-0.5 + for block in self.transformer.h: + torch.nn.init.uniform_(block.attn.c_q.weight, -scale, scale) + torch.nn.init.uniform_(block.attn.c_k.weight, -scale, scale) + torch.nn.init.uniform_(block.attn.c_v.weight, -scale, scale) + torch.nn.init.zeros_(block.attn.c_proj.weight) + torch.nn.init.uniform_(block.mlp.c_fc.weight, -scale, scale) + torch.nn.init.zeros_(block.mlp.c_proj.weight) + self.resid_lambdas.fill_(1.0) + self.x0_lambdas.fill_(0.1) + for ve in self.value_embeds.values(): + torch.nn.init.uniform_(ve.weight, -scale, scale) + for block in self.transformer.h: + if block.attn.ve_gate is not None: + torch.nn.init.zeros_(block.attn.ve_gate.weight) + head_dim = self.config.n_embd // self.config.n_head + cos, sin = self._precompute_rotary_embeddings(self.rotary_seq_len, head_dim) + self.cos, self.sin = cos, sin + self.transformer.wte.to(dtype=torch.bfloat16) + for ve in self.value_embeds.values(): + ve.to(dtype=torch.bfloat16) + + def _precompute_rotary_embeddings( + self, + seq_len: int, + head_dim: int, + base: int = 10000, + device: torch.device | None = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + if device is None: + device = self.transformer.wte.weight.device + channel_range = torch.arange(0, head_dim, 2, dtype=torch.float32, device=device) + inv_freq = 1.0 / (base ** (channel_range / head_dim)) + t = torch.arange(seq_len, dtype=torch.float32, device=device) + freqs = torch.outer(t, inv_freq) + cos, sin = freqs.cos(), freqs.sin() + cos, sin = cos.bfloat16(), sin.bfloat16() + return cos[None, :, None, :], sin[None, :, None, :] + + def _compute_window_sizes(self, config: GPTConfig) -> list[tuple[int, int]]: + pattern = config.window_pattern.upper() + assert all(c in "SL" for c in pattern) + long_window = config.sequence_len + short_window = long_window // 2 + char_to_window = {"L": (long_window, 0), "S": (short_window, 0)} + window_sizes = [] + for layer_idx in range(config.n_layer): + char = pattern[layer_idx % len(pattern)] + window_sizes.append(char_to_window[char]) + window_sizes[-1] = (long_window, 0) + return window_sizes + + def estimate_flops(self) -> float: + nparams = sum(p.numel() for p in self.parameters()) + value_embeds_numel = sum(ve.weight.numel() for ve in self.value_embeds.values()) + nparams_exclude = ( + self.transformer.wte.weight.numel() + + value_embeds_numel + + self.resid_lambdas.numel() + + self.x0_lambdas.numel() + ) + n_head = self.config.n_head + head_dim = self.config.n_embd // self.config.n_head + seq_len = self.config.sequence_len + attn_flops = 0 + for window_size in self.window_sizes: + window = window_size[0] + effective_seq = seq_len if window < 0 else min(window, seq_len) + attn_flops += 12 * n_head * head_dim * effective_seq + return 6 * (nparams - nparams_exclude) + attn_flops + + def num_scaling_params(self) -> dict[str, int]: + wte = sum(p.numel() for p in self.transformer.wte.parameters()) + value_embeds = sum(p.numel() for p in self.value_embeds.parameters()) + lm_head = sum(p.numel() for p in self.lm_head.parameters()) + transformer_matrices = sum(p.numel() for p in self.transformer.h.parameters()) + scalars = self.resid_lambdas.numel() + self.x0_lambdas.numel() + total = wte + value_embeds + lm_head + transformer_matrices + scalars + return { + "wte": wte, + "value_embeds": value_embeds, + "lm_head": lm_head, + "transformer_matrices": transformer_matrices, + "scalars": scalars, + "total": total, + } + + def setup_optimizer( + self, + unembedding_lr: float = 0.004, + embedding_lr: float = 0.2, + matrix_lr: float = 0.02, + weight_decay: float = 0.0, + adam_betas: tuple[float, float] = (0.8, 0.95), + scalar_lr: float = 0.5, + ): + from component_system.components.optimizer import MuonAdamW + + model_dim = self.config.n_embd + matrix_params = list(self.transformer.h.parameters()) + value_embeds_params = list(self.value_embeds.parameters()) + embedding_params = list(self.transformer.wte.parameters()) + lm_head_params = list(self.lm_head.parameters()) + resid_params = [self.resid_lambdas] + x0_params = [self.x0_lambdas] + assert len(list(self.parameters())) == ( + len(matrix_params) + + len(embedding_params) + + len(lm_head_params) + + len(value_embeds_params) + + len(resid_params) + + len(x0_params) + ) + # Scale LR ∝ 1/√dmodel (tuned at 768 dim) + dmodel_lr_scale = (model_dim / 768) ** -0.5 + print(f"Scaling AdamW LRs by 1/sqrt({model_dim}/768) = {dmodel_lr_scale:.6f}") + param_groups = [ + dict(kind="adamw", params=lm_head_params, lr=unembedding_lr * dmodel_lr_scale, betas=adam_betas, eps=1e-10, weight_decay=0.0), + dict(kind="adamw", params=embedding_params, lr=embedding_lr * dmodel_lr_scale, betas=adam_betas, eps=1e-10, weight_decay=0.0), + dict(kind="adamw", params=value_embeds_params, lr=embedding_lr * dmodel_lr_scale, betas=adam_betas, eps=1e-10, weight_decay=0.0), + dict(kind="adamw", params=resid_params, lr=scalar_lr * 0.01, betas=adam_betas, eps=1e-10, weight_decay=0.0), + dict(kind="adamw", params=x0_params, lr=scalar_lr, betas=(0.96, 0.95), eps=1e-10, weight_decay=0.0), + ] + for shape in sorted({p.shape for p in matrix_params}): + group_params = [p for p in matrix_params if p.shape == shape] + param_groups.append( + dict( + kind="muon", + params=group_params, + lr=matrix_lr, + momentum=0.95, + ns_steps=5, + beta2=0.95, + weight_decay=weight_decay, + ) + ) + optimizer = MuonAdamW(param_groups) + for group in optimizer.param_groups: + group["initial_lr"] = group["lr"] + return optimizer + + def forward( + self, + idx: torch.Tensor, + targets: torch.Tensor | None = None, + reduction: str = "mean", + ) -> torch.Tensor: + _, seq_len = idx.size() + assert seq_len <= self.cos.size(1) + cos_sin = self.cos[:, :seq_len], self.sin[:, :seq_len] + x = self.transformer.wte(idx) + x = norm(x) + x0 = x + for layer_idx, block in enumerate(self.transformer.h): + x = self.resid_lambdas[layer_idx] * x + self.x0_lambdas[layer_idx] * x0 + ve = self.value_embeds[str(layer_idx)](idx) if str(layer_idx) in self.value_embeds else None + x = block(x, ve, cos_sin, self.window_sizes[layer_idx]) + x = norm(x) + logits = self.lm_head(x).float() + softcap = 15 + logits = softcap * torch.tanh(logits / softcap) + if targets is None: + return logits + return F.cross_entropy( + logits.view(-1, logits.size(-1)), + targets.view(-1), + ignore_index=-1, + reduction=reduction, + ) + + +def build_model_config( + depth: int, + *, + vocab_size: int, + aspect_ratio: int = 64, + head_dim: int = 128, + window_pattern: str = "SSSL", +) -> GPTConfig: + base_dim = depth * aspect_ratio + model_dim = ((base_dim + head_dim - 1) // head_dim) * head_dim + num_heads = model_dim // head_dim + return GPTConfig( + sequence_len=MAX_SEQ_LEN, + vocab_size=vocab_size, + n_layer=depth, + n_head=num_heads, + n_kv_head=num_heads, + n_embd=model_dim, + window_pattern=window_pattern, + ) + + +def create_model( + config: GPTConfig, + *, + device: torch.device | None = None, + compile_model: bool = True, +) -> tuple[GPT, dict[str, int], float]: + if device is None: + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + with torch.device("meta"): + model = GPT(config) + model.to_empty(device=device) + model.init_weights() + param_counts = model.num_scaling_params() + num_flops_per_token = model.estimate_flops() + if compile_model: + model = torch.compile(model, dynamic=False) + return model, param_counts, num_flops_per_token diff --git a/component_system/components/optimizer.py b/component_system/components/optimizer.py new file mode 100644 index 00000000..227caaea --- /dev/null +++ b/component_system/components/optimizer.py @@ -0,0 +1,179 @@ +from __future__ import annotations + +import torch + + +polar_express_coeffs = [ + (8.156554524902461, -22.48329292557795, 15.878769915207462), + (4.042929935166739, -2.808917465908714, 0.5000178451051316), + (3.8916678022926607, -2.772484153217685, 0.5060648178503393), + (3.285753657755655, -2.3681294933425376, 0.46449024233003106), + (2.3465413258596377, -1.7097828382687081, 0.42323551169305323), +] + + +@torch.compile(dynamic=False, fullgraph=True) +def adamw_step_fused( + p: torch.Tensor, + grad: torch.Tensor, + exp_avg: torch.Tensor, + exp_avg_sq: torch.Tensor, + step_t: torch.Tensor, + lr_t: torch.Tensor, + beta1_t: torch.Tensor, + beta2_t: torch.Tensor, + eps_t: torch.Tensor, + wd_t: torch.Tensor, +) -> None: + p.mul_(1 - lr_t * wd_t) + exp_avg.lerp_(grad, 1 - beta1_t) + exp_avg_sq.lerp_(grad.square(), 1 - beta2_t) + bias1 = 1 - beta1_t**step_t + bias2 = 1 - beta2_t**step_t + denom = (exp_avg_sq / bias2).sqrt() + eps_t + step_size = lr_t / bias1 + p.add_(exp_avg / denom, alpha=-step_size) + + +@torch.compile(dynamic=False, fullgraph=True) +def muon_step_fused( + stacked_grads: torch.Tensor, + stacked_params: torch.Tensor, + momentum_buffer: torch.Tensor, + second_momentum_buffer: torch.Tensor, + momentum_t: torch.Tensor, + lr_t: torch.Tensor, + wd_t: torch.Tensor, + beta2_t: torch.Tensor, + ns_steps: int, + red_dim: int, +) -> None: + momentum = momentum_t.to(stacked_grads.dtype) + momentum_buffer.lerp_(stacked_grads, 1 - momentum) + g = stacked_grads.lerp_(momentum_buffer, momentum) + x = g.bfloat16() + x = x / (x.norm(dim=(-2, -1), keepdim=True) * 1.02 + 1e-6) + if g.size(-2) > g.size(-1): + for a, b, c in polar_express_coeffs[:ns_steps]: + a_matrix = x.mT @ x + b_matrix = b * a_matrix + c * (a_matrix @ a_matrix) + x = a * x + x @ b_matrix + else: + for a, b, c in polar_express_coeffs[:ns_steps]: + a_matrix = x @ x.mT + b_matrix = b * a_matrix + c * (a_matrix @ a_matrix) + x = a * x + b_matrix @ x + g = x + beta2 = beta2_t.to(g.dtype) + v_mean = g.float().square().mean(dim=red_dim, keepdim=True) + red_dim_size = g.size(red_dim) + v_norm_sq = v_mean.sum(dim=(-2, -1), keepdim=True) * red_dim_size + v_norm = v_norm_sq.sqrt() + second_momentum_buffer.lerp_(v_mean.to(dtype=second_momentum_buffer.dtype), 1 - beta2) + step_size = second_momentum_buffer.clamp_min(1e-10).rsqrt() + scaled_sq_sum = (v_mean * red_dim_size) * step_size.float().square() + v_norm_new = scaled_sq_sum.sum(dim=(-2, -1), keepdim=True).sqrt() + final_scale = step_size * (v_norm / v_norm_new.clamp_min(1e-10)) + g = g * final_scale.to(g.dtype) + lr = lr_t.to(g.dtype) + wd = wd_t.to(g.dtype) + mask = (g * stacked_params) >= 0 + stacked_params.sub_(lr * g + lr * wd * stacked_params * mask) + + +class MuonAdamW(torch.optim.Optimizer): + def __init__(self, param_groups: list[dict]) -> None: + super().__init__(param_groups, defaults={}) + self._adamw_step_t = torch.tensor(0.0, dtype=torch.float32, device="cpu") + self._adamw_lr_t = torch.tensor(0.0, dtype=torch.float32, device="cpu") + self._adamw_beta1_t = torch.tensor(0.0, dtype=torch.float32, device="cpu") + self._adamw_beta2_t = torch.tensor(0.0, dtype=torch.float32, device="cpu") + self._adamw_eps_t = torch.tensor(0.0, dtype=torch.float32, device="cpu") + self._adamw_wd_t = torch.tensor(0.0, dtype=torch.float32, device="cpu") + self._muon_momentum_t = torch.tensor(0.0, dtype=torch.float32, device="cpu") + self._muon_lr_t = torch.tensor(0.0, dtype=torch.float32, device="cpu") + self._muon_wd_t = torch.tensor(0.0, dtype=torch.float32, device="cpu") + self._muon_beta2_t = torch.tensor(0.0, dtype=torch.float32, device="cpu") + + def _step_adamw(self, group: dict) -> None: + for p in group["params"]: + if p.grad is None: + continue + grad = p.grad + state = self.state[p] + if not state: + state["step"] = 0 + state["exp_avg"] = torch.zeros_like(p) + state["exp_avg_sq"] = torch.zeros_like(p) + state["step"] += 1 + self._adamw_step_t.fill_(state["step"]) + self._adamw_lr_t.fill_(group["lr"]) + self._adamw_beta1_t.fill_(group["betas"][0]) + self._adamw_beta2_t.fill_(group["betas"][1]) + self._adamw_eps_t.fill_(group["eps"]) + self._adamw_wd_t.fill_(group["weight_decay"]) + adamw_step_fused( + p, + grad, + state["exp_avg"], + state["exp_avg_sq"], + self._adamw_step_t, + self._adamw_lr_t, + self._adamw_beta1_t, + self._adamw_beta2_t, + self._adamw_eps_t, + self._adamw_wd_t, + ) + + def _step_muon(self, group: dict) -> None: + params = group["params"] + if not params: + return + first_param = params[0] + state = self.state[first_param] + num_params = len(params) + shape, device, dtype = first_param.shape, first_param.device, first_param.dtype + if "momentum_buffer" not in state: + state["momentum_buffer"] = torch.zeros(num_params, *shape, dtype=dtype, device=device) + if "second_momentum_buffer" not in state: + state_shape = (num_params, shape[-2], 1) if shape[-2] >= shape[-1] else (num_params, 1, shape[-1]) + state["second_momentum_buffer"] = torch.zeros(state_shape, dtype=dtype, device=device) + red_dim = -1 if shape[-2] >= shape[-1] else -2 + stacked_grads = torch.stack([p.grad for p in params]) + stacked_params = torch.stack(params) + self._muon_momentum_t.fill_(group["momentum"]) + self._muon_beta2_t.fill_(group["beta2"] if group["beta2"] is not None else 0.0) + self._muon_lr_t.fill_(group["lr"] * max(1.0, shape[-2] / shape[-1]) ** 0.5) + self._muon_wd_t.fill_(group["weight_decay"]) + muon_step_fused( + stacked_grads, + stacked_params, + state["momentum_buffer"], + state["second_momentum_buffer"], + self._muon_momentum_t, + self._muon_lr_t, + self._muon_wd_t, + self._muon_beta2_t, + group["ns_steps"], + red_dim, + ) + torch._foreach_copy_(params, list(stacked_params.unbind(0))) + + @torch.no_grad() + def step(self) -> None: + for group in self.param_groups: + if group["kind"] == "adamw": + self._step_adamw(group) + elif group["kind"] == "muon": + self._step_muon(group) + + +def create_optimizer(model: torch.nn.Module, settings: object) -> MuonAdamW: + return model.setup_optimizer( + unembedding_lr=settings.unembedding_lr, + embedding_lr=settings.embedding_lr, + matrix_lr=settings.matrix_lr, + weight_decay=settings.weight_decay, + adam_betas=settings.adam_betas, + scalar_lr=settings.scalar_lr, + ) diff --git a/component_system/components/trainer.py b/component_system/components/trainer.py new file mode 100644 index 00000000..fd300348 --- /dev/null +++ b/component_system/components/trainer.py @@ -0,0 +1,191 @@ +from __future__ import annotations + +import gc +import time +from dataclasses import dataclass +from typing import Any + +import torch + +from prepare import MAX_SEQ_LEN, TIME_BUDGET, evaluate_bpb, make_dataloader + + +H100_BF16_PEAK_FLOPS = 989.5e12 + + +@dataclass +class TrainingSettings: + aspect_ratio: int = 64 + head_dim: int = 128 + window_pattern: str = "SSSL" + total_batch_size: int = 2**19 + embedding_lr: float = 0.6 + unembedding_lr: float = 0.004 + matrix_lr: float = 0.04 + scalar_lr: float = 0.5 + weight_decay: float = 0.2 + adam_betas: tuple[float, float] = (0.8, 0.95) + warmup_ratio: float = 0.0 + warmdown_ratio: float = 0.5 + final_lr_frac: float = 0.0 + depth: int = 8 + device_batch_size: int = 32 # 24GB vram + seed: int = 42 + compile_model: bool = True + + +def default_training_settings() -> TrainingSettings: + return TrainingSettings() + + +def get_lr_multiplier(progress: float, settings: TrainingSettings) -> float: + if progress < settings.warmup_ratio: + return progress / settings.warmup_ratio if settings.warmup_ratio > 0 else 1.0 + if progress < 1.0 - settings.warmdown_ratio: + return 1.0 + cooldown = (1.0 - progress) / settings.warmdown_ratio + return cooldown + (1 - cooldown) * settings.final_lr_frac + + +def get_muon_momentum(step: int) -> float: + frac = min(step / 300, 1) + return (1 - frac) * 0.85 + frac * 0.95 + + +def get_weight_decay(progress: float, settings: TrainingSettings) -> float: + return settings.weight_decay * (1 - progress) + + +def run_training_session( + *, + model: torch.nn.Module, + optimizer: torch.optim.Optimizer, + tokenizer: Any, + settings: TrainingSettings, + param_counts: dict[str, int], + num_flops_per_token: float, + baseline_binding: dict[str, Any], +) -> dict[str, Any]: + t_start = time.time() + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + autocast_device = "cuda" if device.type == "cuda" else "cpu" + autocast_ctx = torch.amp.autocast(device_type=autocast_device, dtype=torch.bfloat16) + + tokens_per_fwdbwd = settings.device_batch_size * MAX_SEQ_LEN + assert settings.total_batch_size % tokens_per_fwdbwd == 0 + grad_accum_steps = settings.total_batch_size // tokens_per_fwdbwd + train_loader = make_dataloader(tokenizer, settings.device_batch_size, MAX_SEQ_LEN, "train") + x, y, epoch = next(train_loader) + + print(f"Vocab size: {tokenizer.get_vocab_size():,}") + print(f"Time budget: {TIME_BUDGET}s") + print(f"Gradient accumulation steps: {grad_accum_steps}") + print("Training session started") + + t_start_training = time.time() + smooth_train_loss = 0.0 + total_training_time = 0.0 + step = 0 + + while True: + if device.type == "cuda": + torch.cuda.synchronize(device=device) + t0 = time.time() + for _ in range(grad_accum_steps): + with autocast_ctx: + loss = model(x, y) + train_loss = loss.detach() + loss = loss / grad_accum_steps + loss.backward() + x, y, epoch = next(train_loader) + + progress = min(total_training_time / TIME_BUDGET, 1.0) + lrm = get_lr_multiplier(progress, settings) + muon_momentum = get_muon_momentum(step) + muon_weight_decay = get_weight_decay(progress, settings) + for group in optimizer.param_groups: + group["lr"] = group["initial_lr"] * lrm + if group["kind"] == "muon": + group["momentum"] = muon_momentum + group["weight_decay"] = muon_weight_decay + + optimizer.step() + model.zero_grad(set_to_none=True) + train_loss_f = train_loss.item() + if train_loss_f > 100: + raise RuntimeError("Training aborted because loss exceeded the fast-fail threshold.") + + torch.cuda.synchronize(device=device) + dt = time.time() - t0 + if step > 10: + total_training_time += dt + + ema_beta = 0.9 + smooth_train_loss = ema_beta * smooth_train_loss + (1 - ema_beta) * train_loss_f + debiased_smooth_loss = smooth_train_loss / (1 - ema_beta ** (step + 1)) + pct_done = 100 * progress + tok_per_sec = int(settings.total_batch_size / dt) + mfu = 100 * num_flops_per_token * settings.total_batch_size / dt / H100_BF16_PEAK_FLOPS + remaining = max(0.0, TIME_BUDGET - total_training_time) + print( + f"\rstep {step:05d} ({pct_done:.1f}%) | loss: {debiased_smooth_loss:.6f} | " + f"lrm: {lrm:.2f} | dt: {dt*1000:.0f}ms | tok/sec: {tok_per_sec:,} | " + f"mfu: {mfu:.1f}% | epoch: {epoch} | remaining: {remaining:.0f}s ", + end="", + flush=True, + ) + + if step == 0: + gc.collect() + gc.freeze() + gc.disable() + elif (step + 1) % 5000 == 0: + gc.collect() + + step += 1 + if step > 10 and total_training_time >= TIME_BUDGET: + break + + print() + total_tokens = step * settings.total_batch_size + model.eval() + with autocast_ctx: + val_bpb = evaluate_bpb(model, tokenizer, settings.device_batch_size) + + t_end = time.time() + peak_vram_mb = torch.cuda.max_memory_allocated() / 1024 / 1024 + steady_state_mfu = ( + 100 + * num_flops_per_token + * settings.total_batch_size + * (step - 10) + / total_training_time + / H100_BF16_PEAK_FLOPS + if total_training_time > 0 + else 0.0 + ) + num_params = param_counts["total"] + metrics = { + "val_bpb": float(val_bpb), + "training_seconds": float(total_training_time), + "total_seconds": float(t_end - t_start), + "peak_vram_mb": float(peak_vram_mb), + "mfu_percent": float(steady_state_mfu), + "total_tokens_M": float(total_tokens / 1e6), + "num_steps": int(step), + "num_params_M": float(num_params / 1e6), + "depth": int(settings.depth), + "startup_seconds": float(t_start_training - t_start), + } + + print("---") + print(f"val_bpb: {metrics['val_bpb']:.6f}") + print(f"training_seconds: {metrics['training_seconds']:.1f}") + print(f"total_seconds: {metrics['total_seconds']:.1f}") + print(f"peak_vram_mb: {metrics['peak_vram_mb']:.1f}") + print(f"mfu_percent: {metrics['mfu_percent']:.2f}") + print(f"total_tokens_M: {metrics['total_tokens_M']:.1f}") + print(f"num_steps: {metrics['num_steps']}") + print(f"num_params_M: {metrics['num_params_M']:.1f}") + print(f"depth: {metrics['depth']}") + return metrics diff --git a/component_system/config.py b/component_system/config.py new file mode 100644 index 00000000..ffe3ef7c --- /dev/null +++ b/component_system/config.py @@ -0,0 +1,31 @@ +"""Static configuration for the component system. No dynamic or per-run values.""" +from __future__ import annotations + +from pathlib import Path + +COMPONENT_SYSTEM_ROOT = Path(__file__).resolve().parent + +# Module import paths for training (used by mainline assembler) +MODEL_MODULE = "component_system.components.model" +OPTIMIZER_MODULE = "component_system.components.optimizer" +TRAINING_STEP_MODULE = "component_system.components.trainer" + +# Promotion threshold: improve val_bpb by at least this much to promote +PROMOTION_THRESHOLD = 0.001 + +# Worktree root relative to project +WORKTREE_ROOT = "component_system/history/worktrees" + +# Default branch name suggested in UI when no branches exist (not a global baseline) +DEFAULT_BASELINE_BRANCH = "master" + + +def get_training_binding() -> dict[str, str | float]: + """Return a static dict used by training mainline/trainer (no baseline_version).""" + return { + "model_module": MODEL_MODULE, + "optimizer_module": OPTIMIZER_MODULE, + "training_step_module": TRAINING_STEP_MODULE, + "promotion_threshold": PROMOTION_THRESHOLD, + "worktree_root": WORKTREE_ROOT, + } diff --git a/component_system/docs/SEED_LIFECYCLE_AND_CONCURRENCY_REVIEW.md b/component_system/docs/SEED_LIFECYCLE_AND_CONCURRENCY_REVIEW.md new file mode 100644 index 00000000..854cbf0e --- /dev/null +++ b/component_system/docs/SEED_LIFECYCLE_AND_CONCURRENCY_REVIEW.md @@ -0,0 +1,177 @@ +# Seed Lifecycle, State Transitions, and Concurrency Review + +## 1. Seed lifecycle and state transitions + +### 1.1 Seed status enum (`SeedStatus`) + +| Status | Meaning | +|-------------|---------| +| `draft` | Newly created, not yet queued for Plan | +| `queued` | Plan run created and task in queue (or waiting for baseline) | +| `planning` | P run in progress | +| `generated` | P completed; code generated, DCA not yet queued | +| `dca_queued`| DCA run created and task in queue (includes sync/merge resolution) | +| `adapting` | DCA run in progress | +| `running` | **Never set in code** — see gap below | +| `passed` | DCA completed, no promotion | +| `failed` | Terminal failure (P failed, DCA failed, or reconciled from passed+error) | +| `promoted` | DCA completed with positive signal; seed merged into baseline | + +### 1.2 Documented transitions (from code) + +``` +draft → queued queue_p / _enqueue_plan_run +queued → queued queue_p (waiting for baseline; latest_run_id cleared) +queued → planning mark_run_started (stage P) +planning → generated finish_p_run +planning → failed mark_run_failed (P failed) +generated → dca_queued queue_dca +generated → queued finish_sync_resolution (then _enqueue_plan_run) +dca_queued → adapting mark_run_started (stage DCA) +adapting → passed finish_dca_run (neutral/negative signal) +adapting → failed finish_dca_run (error) or mark_run_failed +adapting → promoted finish_dca_run (positive_signal) +adapting → dca_queued finish_dca_run (merge/sync failed → queue_dca merge_resolution=True) +adapting → generated finish_dca_run (ralph neutral_signal) +passed → failed _reconcile_seed_status_signal (passed but latest_signal=="error") +``` + +Baseline seed: `draft` → `generated` (ensure_baseline_result) → `dca_queued` → … → `passed` / `failed`. + +--- + +## 2. Gaps and issues in state transitions + +### 2.1 `SeedStatus.running` is never set + +- **Code:** `SeedStatus.running` appears only in: + - `is_seed_eligible_for_stage` (P not eligible if `adapting`, `running`, or `dca_queued`) + - `ensure_baseline_result` (early return if `dca_queued`, `adapting`, `running`) + - Dashboard `status_column_map` → `activeDca` +- **Issue:** No assignment `seed.status = SeedStatus.running` anywhere. DCA-in-progress uses `adapting`. +- **Recommendation:** Either remove `running` from the enum and all checks, or document it as reserved and start setting it (e.g. for a future “running but not adapting” phase). Otherwise it’s dead code and the enum is misleading. + +### 2.2 Sync failure in `mark_run_started` (P): run/seed consistency + +- **Flow:** When a P run is started, `mark_run_started`: + 1. Sets `run.status = RunStatus.running` (in memory). + 2. Calls `ensure_seed_worktree_ready`, then `sync_seed_worktree_with_baseline`. + 3. On `GitCommandError`, calls `queue_sync_resolution(seed_id)` and raises `SyncResolutionQueued`. + 4. Only at the end (after sync and other logic) does it `run_repo.save(run)` and `seed_repo.save(seed)`. +- **Effect:** When sync fails: + - The **run** is never saved as `running`; it remains `queued` in the run repo. + - The **seed** is updated by `queue_sync_resolution`: `seed.status = dca_queued`, new DCA run and task written. + - The **P task** is moved to error in `run.py` (`move_to_error(task_path)`). +- **Result:** The original P **run** is orphaned: it stays `queued` forever and is never completed or failed. `seed.latest_run_id` points to the new sync-resolution DCA run. A later Plan enqueue creates a new P run and task. +- **Recommendation:** When raising `SyncResolutionQueued`, either: + - Mark the current P run as failed (e.g. “sync_failed”) and save it, or + - Explicitly not create a run for that P task until after sync succeeds (e.g. move run creation to after sync). That would require a larger refactor. + +### 2.3 Other transitions + +- All other transitions are consistent with the intended flow: P → generated/failed, DCA → passed/failed/promoted/dca_queued/generated, and reconciliation of `passed` + `error` → `failed`. + +--- + +## 3. Multiple seeds running at the same time — race conditions and conflicts + +### 3.1 Task claiming is atomic per task + +- **Mechanism:** `claim_pending` uses `path.rename(path, IN_PROGRESS_DIR / path.name)`. Only one process can rename a given file; others get `FileNotFoundError` or `OSError` and skip that task. +- **Effect:** Each task file is claimed by at most one worker; no double execution of the same task. + +### 3.2 Per-seed eligibility prevents P vs DCA overlap + +- **Mechanism:** Before running a task, the worker calls `claim_pending(..., eligible_fn=eligible)`. `eligible` uses `WORKFLOW.is_seed_eligible_for_stage(seed_id, stage)`: + - **P:** eligible only if `seed.status not in (adapting, running, dca_queued)`. + - **DCA:** eligible only if `seed.status is not SeedStatus.planning`. +- **Effect:** For a given seed, P and DCA are never both considered eligible. So the same seed cannot have a P task and a DCA task running at the same time, and a seed in `planning` or `adapting` will not get another stage started until the run finishes. + +### 3.3 Read–modify–write on seed/run state + +- **Risk:** Multiple workers can run concurrently (e.g. 2 P workers, 1 DCA-GPU, 1 DCA-AUX). Each worker loads seed/run, modifies, and saves. There is no locking or optimistic concurrency (e.g. version field). +- **Mitigation in practice:** + - Each **task** is for a specific (seed_id, run_id). Different tasks imply different runs (and usually different seeds for P/DCA). + - Eligibility ensures that for a given seed, only one “kind” of work (P or DCA) is allowed at a time. +- **Remaining risk:** If two tasks for the same seed could ever be in flight (e.g. due to a bug or a restored task), two workers could both read the same seed, update it, and save; the last write would win and one update could be lost. With the current design (one active run per seed per stage), this should not happen for normal execution. + +### 3.4 Git worktrees + +- **Design:** Each seed has its own worktree (path `worktrees/`). Different seeds use different directories. +- **Effect:** No filesystem conflict between seeds; multiple seeds can run P or DCA in parallel in separate worktrees. Baseline seed uses `worktrees/__baseline__`. + +### 3.5 Shared JSON state (repos) + +- **State:** Seeds, runs, metrics, branch map, and queue dirs are file-based (JSON under `history/state/`, `history/queue/`). +- **Risk:** Two workers writing different seeds at the same time can overwrite each other only if they wrote the same file (same seed or same run). Since each task is bound to one run and one seed, and eligibility prevents overlapping stages for the same seed, concurrent updates to the same seed/run are not expected for correct flows. +- **Recommendation:** For extra safety, consider short-lived file locking or atomic write (write to temp + rename) for seed/run saves if the daemon scales to many workers. + +--- + +## 4. Edge case: automatic merge fails — can dependent tasks start prematurely? + +### 4.1 Sync failure (merge baseline into seed) before P + +- **When:** In `mark_run_started` (stage P), `sync_seed_worktree_with_baseline(seed)` raises `GitCommandError`. +- **What happens:** + 1. `queue_sync_resolution(seed_id)` runs: seed set to `dca_queued`, new DCA task with `sync_resolution: True` is written. + 2. `SyncResolutionQueued` is raised; in `run.py` the P task is moved to error (not re-queued). + 3. Seed remains `dca_queued`; only the sync-resolution DCA task is for that seed. +- **Eligibility:** For P, a seed in `dca_queued` is **not** eligible. So no other P task for this seed can start. No dependent “normal” P runs until the sync-resolution DCA completes and Plan is re-queued in `finish_sync_resolution`. So **dependent tasks do not start prematurely**. + +### 4.2 DCA merge into baseline fails (normal or baseline seed) + +- **When:** In `finish_dca_run`, `promote_seed_branch` raises `GitCommandError`. +- **What happens:** + 1. A new DCA run is queued with `merge_resolution=True` (and seed stays `dca_queued`). + 2. No new P run or normal DCA run is enqueued for that seed until the merge-resolution DCA finishes. +- **Eligibility:** While seed is `dca_queued` or `adapting`, P is not eligible. So **dependent tasks do not start prematurely**. + +### 4.3 Baseline merge fails + +- Same pattern: baseline seed gets a merge-resolution DCA task, stays `dca_queued`. `_release_seeds_waiting_for_baseline` is only called after a successful merge (or after the “loop avoided” path). Waiting seeds are not released until baseline is merged. So **dependent tasks do not start prematurely**. + +**Conclusion:** When the workflow’s automatic merge (sync or promote) fails, the seed is put in `dca_queued` with a resolution DCA task. Eligibility and the fact that no new P/normal DCA is enqueued until resolution completes ensure that dependent tasks do **not** start before merge resolution. + +--- + +## 5. Other edge cases + +### 5.1 Restored in-progress tasks + +- On daemon start, `restore_in_progress_tasks()` moves all tasks from `in_progress/` back to the stage queue. Those tasks are then eligible to be claimed again. +- **Risk:** If a task was in progress (worker had already called `mark_run_started` and set seed to `planning`/`adapting`) and the daemon died before the worker finished, the run and seed are already updated. After restore, the task is back in the queue; a worker can claim it and call `mark_run_started` again. That would re-use the same run_id and could lead to duplicate “started” events or inconsistent state (e.g. two workers both thinking they own the run). The code does not detect “this run was already started.” +- **Recommendation:** Before updating run/seed in `mark_run_started`, check that `run.status` is still `queued`; if it is already `running`, treat the task as a duplicate (e.g. move to error or skip and don’t run again). + +### 5.2 Ralph loop and merge_resolution / metrics_recovery + +- After a failed DCA, `mark_run_failed` can call `queue_p(seed_id)` for Ralph seeds, but only when the task is not `merge_resolution` and not `metrics_recovery`. So Ralph does not re-queue P on merge-resolution or metrics-recovery DCA failure, which is correct. + +### 5.3 Baseline seed and sync + +- Baseline seed does not call `sync_seed_worktree_with_baseline` (early return in that function). So sync failure path does not apply to __baseline__. `queue_sync_resolution` explicitly raises if seed is baseline. No issue. + +--- + +## 6. Summary table + +| Area | Status | Notes | +|-----------------------------|--------|--------| +| Seed status enum | Gap | `SeedStatus.running` never set; remove or use. | +| P/DCA transition consistency| OK | Transitions match design. | +| Sync fail (before P) | Bug | P run left `queued`; orphaned run. | +| Task claiming | OK | Atomic rename prevents double run of same task. | +| P vs DCA same seed | OK | Eligibility prevents concurrent P and DCA for one seed. | +| Multiple seeds concurrent | OK | Different worktrees; eligibility per seed. | +| Merge/sync fail → dependents| OK | Seed stays `dca_queued`; no premature P/DCA. | +| Restored in-progress tasks | Risk | Re-claiming can lead to duplicate start for same run. | + +--- + +## 7. Implemented fixes + +1. **Sync failure in `mark_run_started`:** Before raising `SyncResolutionQueued`, mark the current P run as failed (e.g. error “sync with baseline failed”) and save it, so the run is not orphaned. +2. **`SeedStatus.running`:** Either remove it from the enum and from all checks, or introduce a clear rule (e.g. “DCA in progress” = `adapting` only) and document that `running` is unused. +3. **Restored tasks:** In `mark_run_started`, if `run.status != RunStatus.queued`, do not update run/seed and do not run the agent; move the task to error or a “duplicate” bucket and return. + +**Not changed:** `SeedStatus.running` is still never set; it could be removed from the enum in a follow-up or left as reserved. diff --git a/component_system/domain/models.py b/component_system/domain/models.py new file mode 100644 index 00000000..d105569e --- /dev/null +++ b/component_system/domain/models.py @@ -0,0 +1,93 @@ +from __future__ import annotations + +from enum import Enum +from typing import Any + +from pydantic import BaseModel, Field + + +class SeedStatus(str, Enum): + draft = "draft" + queued = "queued" + planning = "planning" + generated = "generated" + dca_queued = "dca_queued" + adapting = "adapting" + running = "running" + failed = "failed" + passed = "passed" + promoted = "promoted" + + +class StageName(str, Enum): + p = "p" + dca = "dca" + direct = "direct" + + +class RunStatus(str, Enum): + queued = "queued" + running = "running" + succeeded = "succeeded" + failed = "failed" + + +class PlanIdea(BaseModel): + title: str = "" + target_component: str = "model" + description: str = "" + source_refs: list[str] = Field(default_factory=list) + commit_sha: str | None = None + + +class StageRun(BaseModel): + run_id: str + seed_id: str + stage: StageName + status: RunStatus + task_id: str + created_at: float + updated_at: float + log_path: str | None = None + stderr_log_path: str | None = None + prompt_path: str | None = None + summary: dict[str, Any] = Field(default_factory=dict) + metrics: dict[str, Any] = Field(default_factory=dict) + signal: str | None = None + error: str | None = None + + +class SeedRecord(BaseModel): + seed_id: str + prompt: str + status: SeedStatus = SeedStatus.draft + created_at: float + updated_at: float + baseline_branch: str = "baseline" + worktree_path: str | None = None + latest_run_id: str | None = None + ralph_loop_enabled: bool = False + latest_signal: str | None = None + latest_metrics: dict[str, Any] = Field(default_factory=dict) + plan: PlanIdea | None = None + last_error: str | None = None + """Baseline val_bpb at sync-before-P time; used for positive/negative/neutral judgement in DCA.""" + former_val_bpb: float | None = None + + +class DashboardColumn(BaseModel): + id: str + title: str + description: str + seeds: list[SeedRecord] + + +class DashboardViewModel(BaseModel): + setup_error: str | None = None + baseline_metrics_by_branch: dict[str, dict[str, object]] = Field(default_factory=dict) + default_baseline_branch: str = "master" + available_branches: list[str] = Field(default_factory=list) + seed_count: int + columns: list[DashboardColumn] + selected_seed: SeedRecord | None = None + daemon_status: str = "stopped" # "running" | "stopped" diff --git a/component_system/entrypoint.py b/component_system/entrypoint.py new file mode 100644 index 00000000..33fc2d42 --- /dev/null +++ b/component_system/entrypoint.py @@ -0,0 +1,18 @@ +"""Standalone entrypoint for the component_system baseline.""" +from __future__ import annotations + +import sys +from pathlib import Path + +if __package__ in {None, ""}: + sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from component_system.training.mainline import run_mainline_training + + +def main() -> None: + run_mainline_training() + + +if __name__ == "__main__": + main() diff --git a/component_system/package.json b/component_system/package.json new file mode 100644 index 00000000..5ae45136 --- /dev/null +++ b/component_system/package.json @@ -0,0 +1,13 @@ +{ + "name": "autoresearch-component-system-ui", + "private": true, + "scripts": { + "build:css": "tailwindcss -i ./web/static/tailwind.input.css -o ./web/static/app.css --minify", + "watch:css": "tailwindcss -i ./web/static/tailwind.input.css -o ./web/static/app.css --watch" + }, + "devDependencies": { + "autoprefixer": "^10.4.20", + "postcss": "^8.4.49", + "tailwindcss": "^3.4.17" + } +} diff --git a/component_system/postcss.config.js b/component_system/postcss.config.js new file mode 100644 index 00000000..5cbc2c7d --- /dev/null +++ b/component_system/postcss.config.js @@ -0,0 +1,6 @@ +module.exports = { + plugins: { + tailwindcss: {}, + autoprefixer: {} + } +}; diff --git a/component_system/protocol.md b/component_system/protocol.md new file mode 100644 index 00000000..56d43941 --- /dev/null +++ b/component_system/protocol.md @@ -0,0 +1,290 @@ +# autoresearch — Component-System Protocol + +This document is the operating manual for the component-system workflow. +The system runs a continuous Seed -> P -> DCA loop to discover, generate, +adapt, evaluate, and promote improvements to the training stack. + +The main objective is simple: improve `val_bpb` against the current baseline +without breaking the canonical component-system entrypoint or introducing +unreasonable complexity. + +VRAM is a first-class constraint. Higher memory use is acceptable only when the +quality gain is meaningful; avoid candidates that produce small or ambiguous +`val_bpb` gains while causing large memory growth. + +## Top-Level Bootstrap Rule + +If you are an interactive code agent that was merely told to "follow this +protocol", do not manually simulate the entire workflow inside one foreground +session. + +The intended control flow is: +1. Read this file and the required context files. +2. Ensure the queue and state layout exist. +3. Create or refine a seed from a human prompt. +4. Queue the seed for P. +5. Start the daemon: `uv run component_system/run.py`. +6. Let daemon workers execute P and DCA via file-based handoff. +7. Monitor the daemon, queue, and logs; do not simulate stages in-session. + +Manual execution of an individual stage is only for the agent process that was +invoked by the daemon for that specific task. + +## Architecture + +```text +component_system/ + protocol.md <- overall workflow protocol + entrypoint.py <- canonical training entrypoint + PDCA-PLAN.md <- P stage rules + PDCA-DO-CHECK-ACTION.md <- DCA stage rules + run.py <- resident daemon and worker dispatch + task.py <- queue and JSON state helpers + baseline_branches.json <- per-branch baseline mapping (workflow-managed; read-only) + baseline_metrics.json <- baseline run metrics (workflow-managed; read-only) + config.py <- promotion threshold and static binding + history/ <- runtime dir (auto-created) + logs/ <- agent stdout/stderr logs + queue/{p,dca,done,error}/ <- stage handoff and archival + state/{seeds,runs,events}/<- durable workflow state + worktrees/ <- per-seed git worktrees + components/ + model.py + optimizer.py + trainer.py + training/ + mainline.py +``` + +## Core Goal and Decision Rule + +Optimize for lower `val_bpb`. A candidate is worth promoting only when the gain +is real, the implementation is understandable, and the cost in memory or +complexity is justified. + +Apply this bias consistently: +- Lower `val_bpb` is the primary success metric. +- VRAM is a soft but important constraint: some increase is acceptable, but + dramatic growth needs correspondingly strong quality gains. +- Simpler changes are preferred when results are similar. +- A tiny gain that adds brittle complexity is usually not worth promotion. +- A tiny gain that materially increases VRAM is usually not worth promotion. +- A simplification that preserves or slightly improves quality is a strong outcome. +- If the signal is ambiguous, treat it as `neutral` and do not promote. + +## Required Reading Before Any Work + +Read in this order: +1. `component_system/protocol.md` +2. The stage-specific document (right after protocol): `component_system/PDCA-DO-CHECK-ACTION.md` for DCA, `component_system/PDCA-PLAN.md` for P +3. `prepare.py` for fixed data and evaluation behavior; never modify it +4. `component_system/entrypoint.py` for the canonical execution path +5. `component_system/config.py` for promotion threshold and static binding + +Baseline files (workflow-managed; read-only): `baseline_branches.json`, `baseline_metrics.json`. For interactive bootstrap, inspect recent `queue/done/` and baseline state. + +## Workspace and Path Rules + +When the daemon invokes you for a P or DCA task, your current working directory +is the seed worktree. In that mode: + +- Read and edit only within the seed worktree. +- Use only relative paths from the current working directory. +- Do not request or depend on absolute paths or files outside the worktree. + +## Hard Constraints + +1. Never modify `prepare.py`. +2. `uv run component_system/entrypoint.py` must remain the canonical, + working component-system training command. +3. The root repo must stay compatible with the upstream implementation; + do not require changes to root `train.py`. +4. Stage-to-stage handoff must happen through files under `queue/`, not + merely in memory or only in agent conversation state. +5. Only the DCA promotion flow may update `baseline_metrics.json` and `baseline_branches.json`. +6. Do not bypass the baseline mechanism by manually merging branches or + force-advancing the baseline outside workflow control. + +## Baseline-First Rule + +Establish baseline before evaluating seeds: if `baseline_metrics.json` has no baseline result for the branch (no records), run the baseline (no-changes) measurement first. Use that result as the reference for promotion. + +```mermaid +flowchart TD + A[Create seed] --> B{Baseline result exists?} + B -- No --> C[Create or reuse __baseline__ seed] + C --> D[Queue baseline DCA] + D --> E[Run baseline measurement from project root] + E --> F[Save baseline metrics in baseline_metrics.json] + F --> G[Release waiting seeds] + B -- Yes --> G + G --> H[Seed stays in draft or queued with no worktree] + H --> I[Queue P run] + I --> J[Create seed worktree at P start] + J --> K[P agent plans and commits on seed branch] + K --> L[Queue DCA run] + L --> M[DCA agent adapts, runs training, and reports metrics] + M --> N{Promotion signal?} + N -- Positive --> O[Merge seed branch into baseline] + O --> P{Merge conflict?} + P -- No --> Q[Update baseline metadata and finish seed] + P -- Yes --> R[Queue conflict-resolution DCA] + R --> M + N -- Neutral or Negative --> S[Keep result in state only] +``` + +## Workflow Stages + +The sections below describe what each daemon-dispatched stage worker does. +They are not instructions for a top-level interactive agent to perform the +entire lifecycle manually. + +### P — Discovery / Plan / Initial Generation + +Read `component_system/PDCA-PLAN.md`. + +Responsibilities: +- Refine the seed prompt into a concrete plan. +- Create or refresh the seed worktree from the active baseline. +- Generate the first candidate implementation in the worktree. +- Keep the change focused enough that DCA can evaluate it cleanly. +- Commit the generated candidate on the seed branch so DCA receives a stable snapshot. + +P is about producing a plausible, testable first version, not claiming success. + +### DCA — Delivery / Check / Action + +Read `component_system/PDCA-DO-CHECK-ACTION.md`. + +Responsibilities: +- Adapt and fix the generated candidate inside the seed worktree. +- Run the canonical training/evaluation entrypoint. +- Read the structured metrics from the run output. +- Decide whether the result is positive, neutral, or negative relative to baseline. +- Promote the seed branch into baseline only when the signal is strong enough. + +DCA is the stage that turns a raw idea into a measured outcome. + +## Canonical Run and Output + +The canonical component-system execution path is: + +```bash +uv run component_system/entrypoint.py +``` + +Allow **at least 900 seconds** when DCA runs this (e.g. `timeout 900 uv run ...`). + +DCA must report a structured JSON summary (including `metrics`). Runner uses it first; falls back to stdout/stderr parsing if missing. No metrics → recovery DCA inspects logs. Canonical metrics: + +```text +--- +val_bpb: 0.997900 +training_seconds: 300.1 +total_seconds: 325.9 +peak_vram_mb: 45060.2 +mfu_percent: 39.80 +total_tokens_M: 499.6 +num_steps: 953 +num_params_M: 50.3 +depth: 8 +startup_seconds: 25.8 +``` + +Treat `val_bpb` as the primary metric. `peak_vram_mb`, total runtime, and code +complexity are secondary constraints that influence promotion decisions. + +## VRAM Rule + +Track `peak_vram_mb` on every serious evaluation run and treat it as required +decision input, not a cosmetic metric. + +- Some VRAM growth is acceptable when it buys a clear `val_bpb` improvement. +- Large VRAM increases require a correspondingly strong quality gain. +- If two candidates are similar on `val_bpb`, prefer the lower-VRAM one. +- If a candidate regresses or barely improves `val_bpb` while increasing VRAM + substantially, treat it as a bad trade and do not promote it. +- Avoid changes that risk blowing up memory usage unless the expected upside is + compelling enough to justify the experiment. + +## Promotion Rule + +A run is promotable only if all of the following hold: +- The run completed successfully. +- `val_bpb` improved enough over the active baseline to count as a real win. +- VRAM growth is not unreasonable for the magnitude of the gain. +- The change is understandable, maintainable, and reversible. + +If the candidate is equal, worse, noisy, or hard to justify, do not promote it. +Record the outcome and move on. + +## Failure Handling + +Use the same judgment standard as the original autoresearch loop: + +- If a run crashes because of a simple bug, fix it, rerun, and update the same + run record. +- If the idea is fundamentally flawed, archive it without promotion. +- If the task cannot be recovered quickly, move it into the error flow and + persist the failure details. +- Crashes are negative evidence; they should not silently disappear. + +## Bootstrap Procedure for Interactive Sessions + +1. Read baseline files and recent queue/state. +2. Ensure queue/state/worktree layout exists. +3. Create a seed from the human prompt and queue it for P. +4. Start `uv run component_system/run.py` and monitor; do not run P/DCA manually. + +## Operating Loop + +1. Seed persisted in `state/seeds/`, queued to `queue/p/`. +2. P refreshes worktree, generates code, commits on seed branch. +3. Daemon queues DCA. +4. DCA adapts, runs, evaluates; promotes or archives. +5. State persisted under `state/`; daemon continues with next work. + +## State and Logging + +- `component_system/history/state/seeds/`, `component_system/history/state/runs/`, `component_system/history/state/events/` — seed and run state. +- `component_system/history/queue/done/`, `component_system/history/queue/error/` — completed and failed tasks. +- `component_system/history/logs/` — agent stdout/stderr. + +Use filesystem state as source of truth, not chat context. + +## Daemon + +`run.py` runs two single-threaded workers polling `component_system/history/queue/p/` and `component_system/history/queue/dca/`. Workers dispatch to an external code agent; the agent reads files, edits the worktree, runs the entrypoint, and prints structured summaries. + +Start: + +```bash +# Default backend +uv run component_system/run.py + +# Alternate backends +PDCA_AGENT=codex uv run component_system/run.py +PDCA_AGENT=opencode uv run component_system/run.py +``` + +### Agent Backends + +| `PDCA_AGENT` | CLI invoked | Prompt delivery | +|--------------|-------------|-----------------| +| `claude` (default) | `claude -p --verbose` | stdin | +| `codex` | `codex exec -a never --sandbox workspace-write` | positional arg | +| `opencode` | `opencode run` | positional arg | + +### Timeouts + +Each stage has a default timeout in seconds and can be overridden through the +environment: + +| Variable | Default | Purpose | +|----------|---------|---------| +| `PDCA_TIMEOUT_P` | 900 | Planning and initial code generation | +| `PDCA_TIMEOUT_DCA` | 3600 | Adaptation, training, evaluation, and promotion | + +### Logs + +Agent stdout/stderr → `component_system/history/logs/`. diff --git a/component_system/repositories/state.py b/component_system/repositories/state.py new file mode 100644 index 00000000..ef6beec5 --- /dev/null +++ b/component_system/repositories/state.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +from typing import Any + +from component_system.domain.models import SeedRecord, StageRun +from component_system.task import ( + append_event, + list_runs, + list_seeds, + load_baseline_branch_map, + load_baseline_metrics, + load_events, + load_run, + load_seed, + save_baseline_branch_map, + save_baseline_metrics, + save_run, + save_seed, +) + + +class BaselineBranchMapRepository: + """Per-seed baseline branch mapping (seed_id -> baseline_branch).""" + + def set_branch_for_seed(self, seed_id: str, branch: str) -> None: + m = load_baseline_branch_map() + m[seed_id] = branch + save_baseline_branch_map(m) + + +def _branch_metrics_view(history: list[dict[str, Any]]) -> dict[str, Any]: + """Build view with best_val_bpb (min over history) and promoted_* from the record that achieved it.""" + if not history: + return {"best_val_bpb": None, "history": []} + vals = [r["val_bpb"] for r in history if r.get("val_bpb") is not None] + best_val_bpb = min(vals) if vals else None + best_record = next((r for r in history if r.get("val_bpb") == best_val_bpb), history[-1]) + view: dict[str, Any] = { + "best_val_bpb": best_val_bpb, + "history": history, + } + if best_record.get("promoted_branch") is not None: + view["promoted_branch"] = best_record["promoted_branch"] + if best_record.get("promoted_idea") is not None: + view["promoted_idea"] = best_record["promoted_idea"] + if best_record.get("promoted_at") is not None: + view["promoted_at"] = best_record["promoted_at"] + if best_record.get("commit_sha") is not None: + view["commit_sha"] = best_record["commit_sha"] + return view + + +class BaselineMetricsRepository: + """Per-baseline-branch metrics: list of records per branch (val_bpb, promoted_*, etc.).""" + + def get_all(self) -> dict[str, dict[str, Any]]: + """Return branch -> view (best_val_bpb, promoted_branch, commit_sha, history) for dashboard.""" + data = load_baseline_metrics() + return {branch: _branch_metrics_view(hist) for branch, hist in data.items()} + + def get_for_branch(self, branch: str) -> dict[str, Any] | None: + """Return view for one branch (best_val_bpb, history, promoted_branch?, commit_sha?).""" + data = load_baseline_metrics() + hist = data.get(branch) + if hist is None: + return None + return _branch_metrics_view(hist) + + def append_promotion_for_branch(self, branch: str, record: dict[str, Any]) -> None: + """Append a promotion record: val_bpb, promoted_branch, promoted_idea, promoted_at, commit_sha.""" + data = load_baseline_metrics() + data.setdefault(branch, []).append(dict(record)) + save_baseline_metrics(data) + + def append_baseline_run(self, branch: str, val_bpb: float) -> None: + """Append a baseline measurement (no promotion).""" + data = load_baseline_metrics() + data.setdefault(branch, []).append({"val_bpb": val_bpb}) + save_baseline_metrics(data) + + +class SeedRepository: + def list(self) -> list[SeedRecord]: + return [SeedRecord.model_validate(seed) for seed in list_seeds()] + + def get(self, seed_id: str) -> SeedRecord | None: + data = load_seed(seed_id) + return SeedRecord.model_validate(data) if data else None + + def save(self, seed: SeedRecord) -> SeedRecord: + save_seed(seed.model_dump(mode="json")) + return seed + + def append_event(self, seed_id: str, kind: str, message: str, **payload: Any) -> list[dict[str, Any]]: + return append_event(seed_id, {"kind": kind, "message": message, **payload}) + + def events(self, seed_id: str) -> list[dict[str, Any]]: + return load_events(seed_id) + + +class RunRepository: + def list(self, seed_id: str | None = None) -> list[StageRun]: + return [StageRun.model_validate(run) for run in list_runs(seed_id)] + + def get(self, run_id: str) -> StageRun | None: + data = load_run(run_id) + return StageRun.model_validate(data) if data else None + + def save(self, run: StageRun) -> StageRun: + save_run(run.model_dump(mode="json")) + return run diff --git a/component_system/run.py b/component_system/run.py new file mode 100644 index 00000000..b7f98974 --- /dev/null +++ b/component_system/run.py @@ -0,0 +1,889 @@ +"""Seed -> P -> DCA daemon for the component-system web app.""" +from __future__ import annotations + +if __package__ in {None, ""}: + import sys + from pathlib import Path + + sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +import json +import os +import shutil +import signal +import subprocess +import sys +import threading +import time +import traceback +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import Any + +from component_system.domain.models import StageName +from component_system.services.workflow import ( + BASELINE_SEED_ID, + DuplicateRunStartError, + SyncResolutionQueued, + WorkflowService, +) +from component_system.task import ( + BASELINE_BRANCHES_PATH, + BASELINE_METRICS_PATH, + COMPONENT_SYSTEM_ROOT, + claim_pending, + DAEMON_HEARTBEAT_PATH, + daemon_heartbeat, + ensure_queue_layout, + LOG_ROOT, + move_to_done, + move_to_error, + read_task, + restore_in_progress_tasks, +) + +PROJECT_ROOT = COMPONENT_SYSTEM_ROOT.parent +LOG_DIR = LOG_ROOT +RESULTS_TSV = PROJECT_ROOT / "results.tsv" +PROGRESS_PNG = PROJECT_ROOT / "progress.png" + +POLL_INTERVAL = 10.0 +_shutdown = False +WORKFLOW = WorkflowService() + +DEFAULT_TIMEOUTS = {"p": 900, "dca": 3600, "direct": 3600} + +# Canonical DCA entrypoint run: require ≥900s so training can complete. Agent must set command/tool timeout ≥ this. +DCA_CANONICAL_RUN_TIMEOUT_SECONDS = 900 + +STAGE_DOCS = { + "p": ["PDCA-PLAN.md"], + "dca": ["PDCA-DO-CHECK-ACTION.md"], +} + +AGENT_CONFIGS: dict[str, dict[str, Any]] = { + "claude": {"cmd": ["claude", "-p", "--verbose"], "via": "stdin"}, + "codex": {"cmd": ["codex", "exec", "-a", "never", "--sandbox", "workspace-write"], "via": "arg"}, + "opencode": {"cmd": ["opencode", "run"], "via": "arg"}, +} + + +def _signal_handler(_sig: int, _frame: Any) -> None: + global _shutdown + _shutdown = True + print("\n[daemon] shutdown requested") + + +def _get_timeout(stage: str) -> int: + return int(os.environ.get(f"PDCA_TIMEOUT_{stage.upper()}", DEFAULT_TIMEOUTS.get(stage, 900))) + + +def _build_log_paths(run_id: str) -> tuple[Path, Path]: + LOG_DIR.mkdir(parents=True, exist_ok=True) + stdout_path = LOG_DIR / f"{run_id}.stdout.log" + stderr_path = LOG_DIR / f"{run_id}.stderr.log" + return stdout_path, stderr_path + + +def _write_prompt_file(run_id: str, prompt: str) -> Path: + """Save the agent prompt to a file for debugging. Returns the path.""" + LOG_DIR.mkdir(parents=True, exist_ok=True) + prompt_path = LOG_DIR / f"{run_id}.prompt.txt" + prompt_path.write_text(prompt, encoding="utf-8") + return prompt_path + + +def _is_root_venv_active() -> bool: + expected = (PROJECT_ROOT / ".venv").resolve() + active = os.environ.get("VIRTUAL_ENV") + if not active: + return False + try: + return Path(active).resolve() == expected + except OSError: + return False + + +def _dca_command_guidance() -> tuple[str, str]: + timeout_prefix = f"timeout {DCA_CANONICAL_RUN_TIMEOUT_SECONDS}" + if _is_root_venv_active(): + return ( + f"{timeout_prefix} uv run --active component_system/entrypoint.py", + "Root .venv is active; use --active to reuse it from the worktree.", + ) + return ( + f"{timeout_prefix} uv run component_system/entrypoint.py", + "No active root .venv; uv run without --active.", + ) + + +def _build_direct_code_prompt(prompt: str) -> str: + return ( + "You are running as a direct code agent from the project root of this repository.\n" + "Execute the user's request directly in the current working tree.\n" + "Do not switch into seed worktrees for this task.\n\n" + "User request:\n" + f"{prompt.strip()}\n" + ) + + +def _stream_pipe_to_file(pipe: Any, handle: Any, chunks: list[str]) -> None: + try: + while True: + piece = pipe.readline() + if not piece: + break + chunks.append(piece) + handle.write(piece) + handle.flush() + finally: + try: + pipe.close() + except Exception: + pass + + +def _combined_output(stdout: str, stderr: str) -> str: + if stdout and stderr: + return f"{stdout}\n{stderr}" + return stdout or stderr + + +def _agent_failure_reason(exit_code: int, stdout: str, stderr: str) -> str: + combined = _combined_output(stdout, stderr) + if "timeout after " in combined: + return combined.strip().splitlines()[-1] + if exit_code == -1: + if combined.strip(): + return combined.strip().splitlines()[-1] + return "Agent execution failed before completion. See stdout/stderr logs for details." + return f"Agent exited with code {exit_code}. See stdout/stderr logs for details." + + +def _should_salvage_completed_dca(stage: str, exit_code: int, output_text: str) -> bool: + """Accept a DCA run when canonical metrics were printed despite agent exit issues.""" + if stage != "dca" or exit_code == 0: + return False + summary = WORKFLOW.extract_summary(output_text, StageName.dca) or {} + metrics = WORKFLOW.extract_dca_metrics(output_text, summary) + return metrics.get("val_bpb") is not None + + +def _agent_cwd(worktree_path: str | None) -> str: + """Resolve cwd for the agent: seed worktree when provided and present, else project root.""" + if not worktree_path: + return str(PROJECT_ROOT) + path = Path(worktree_path) + if not path.is_absolute(): + path = PROJECT_ROOT / path + resolved = path.resolve() + return str(resolved) if resolved.is_dir() else str(PROJECT_ROOT) + + +def _resolve_worktree_path(worktree_path: str | None) -> Path | None: + """Resolve worktree path to absolute Path, or None if invalid/missing.""" + if not worktree_path: + return None + path = Path(worktree_path) + if not path.is_absolute(): + path = PROJECT_ROOT / path + resolved = path.resolve() + return resolved if resolved.is_dir() else None + + +def _sync_results_tsv_into_worktree(worktree_path: str | None) -> None: + """Copy the latest root results.tsv into the seed worktree if it exists. Non-fatal on failure.""" + resolved = _resolve_worktree_path(worktree_path) + if resolved is None or not RESULTS_TSV.exists(): + return + dest = resolved / "results.tsv" + try: + shutil.copy2(RESULTS_TSV, dest) + except OSError as err: + print(f"[P] could not copy results.tsv into worktree: {err}", file=sys.stderr) + + +def _sync_baseline_json_into_worktree(worktree_path: str | None) -> None: + """Copy baseline_metrics.json and baseline_branches.json from project component_system into the worktree. + Worktrees check out from baseline-branch; without this sync the agent would see stale or missing baseline data.""" + resolved = _resolve_worktree_path(worktree_path) + if resolved is None: + return + dest_dir = resolved / "component_system" + dest_dir.mkdir(parents=True, exist_ok=True) + for src_path, name in [ + (BASELINE_METRICS_PATH, "baseline_metrics.json"), + (BASELINE_BRANCHES_PATH, "baseline_branches.json"), + ]: + if not src_path.exists(): + continue + dest = dest_dir / name + try: + shutil.copy2(src_path, dest) + except OSError as err: + print(f"[P] could not copy {name} into worktree: {err}", file=sys.stderr) + + +def _sync_worktree_context(worktree_path: str | None) -> None: + """Sync all workflow-managed live data into the worktree so the agent sees current state. + Call before invoking the agent when cwd is a worktree (P or DCA).""" + _sync_results_tsv_into_worktree(worktree_path) + _sync_baseline_json_into_worktree(worktree_path) + + +def _invoke_agent( + prompt: str, stage: str, run_id: str, worktree_path: str | None = None +) -> tuple[int, str, str, Path | None, Path | None]: + agent_name = os.environ.get("PDCA_AGENT", "claude") + config = AGENT_CONFIGS.get(agent_name) + if config is None: + raise ValueError(f"Unknown PDCA_AGENT={agent_name!r}. Supported: {', '.join(AGENT_CONFIGS)}") + + cmd = list(config["cmd"]) + timeout = _get_timeout(stage) + cwd = _agent_cwd(worktree_path) + # PYTHONUNBUFFERED=1 so child Python (e.g. uv run entrypoint.py) flushes stdout + # immediately instead of block-buffering when stdout is a pipe; otherwise + # stdout log only appears in one shot after the task finishes. + env = {**os.environ, "PYTHONUNBUFFERED": "1"} + if agent_name == "opencode": + project_root_glob = str(PROJECT_ROOT.resolve().as_posix()) + "/**" + existing = {} + try: + if os.environ.get("OPENCODE_PERMISSION"): + existing = json.loads(os.environ["OPENCODE_PERMISSION"]) + except (json.JSONDecodeError, KeyError): + pass + ext_dir = dict(existing.get("external_directory", {})) + ext_dir[project_root_glob] = "allow" + env["OPENCODE_PERMISSION"] = json.dumps({"external_directory": ext_dir}) + popen_kwargs: dict[str, Any] = { + "cwd": cwd, + "env": env, + "stdout": subprocess.PIPE, + "stderr": subprocess.PIPE, + "text": True, + "encoding": "utf-8", + "errors": "replace", + "bufsize": 1, + } + if config["via"] == "stdin": + popen_kwargs["stdin"] = subprocess.PIPE + else: + # Use DEVNULL so the agent never reads from parent's stdin (avoids EBADF under nohup/redirects). + popen_kwargs["stdin"] = subprocess.DEVNULL + cmd.append(prompt) + + print(f"[{stage.upper()}] invoking {agent_name} (timeout={timeout}s)") + stdout_path, stderr_path = _build_log_paths(run_id) + try: + process = subprocess.Popen(cmd, **popen_kwargs) + except FileNotFoundError: + msg = f"{agent_name!r} binary not found. Install it or set PDCA_AGENT to a different backend." + return -1, "", msg, None, None + + if config["via"] == "stdin" and process.stdin is not None: + process.stdin.write(prompt) + process.stdin.close() + + stdout_chunks: list[str] = [] + stderr_chunks: list[str] = [] + with open(stdout_path, "w", encoding="utf-8") as stdout_handle, open( + stderr_path, "w", encoding="utf-8" + ) as stderr_handle: + stdout_handle.write(f"stage: {stage.upper()}\nagent: {agent_name}\n") + stdout_handle.write(f"timestamp: {time.strftime('%Y%m%d-%H%M%S')}\n\n") + stdout_handle.flush() + stderr_handle.write(f"stage: {stage.upper()}\nagent: {agent_name}\n") + stderr_handle.write(f"timestamp: {time.strftime('%Y%m%d-%H%M%S')}\n\n") + stderr_handle.flush() + + stdout_thread = threading.Thread( + target=_stream_pipe_to_file, + args=(process.stdout, stdout_handle, stdout_chunks), + daemon=True, + ) + stderr_thread = threading.Thread( + target=_stream_pipe_to_file, + args=(process.stderr, stderr_handle, stderr_chunks), + daemon=True, + ) + stdout_thread.start() + stderr_thread.start() + + timed_out = False + try: + process.wait(timeout=timeout) + except subprocess.TimeoutExpired: + timed_out = True + process.kill() + + stdout_thread.join() + stderr_thread.join() + + stdout = "".join(stdout_chunks) + stderr = "".join(stderr_chunks) + if timed_out: + timeout_message = f"timeout after {timeout}s" + if stderr: + stderr = f"{stderr}\n{timeout_message}" + else: + stderr = timeout_message + return -1, stdout, stderr, stdout_path, stderr_path + + return process.returncode, stdout, stderr, stdout_path, stderr_path + + +def _build_metrics_recovery_prompt(task: dict[str, Any]) -> str: + """Lightweight prompt for metrics-recovery DCA: no protocol/docs, just task, log paths, report shape.""" + task_json = json.dumps(task, indent=2) + source_run_id = task.get("source_run_id", "unknown") + stdout_log = task.get("source_stdout_log_path", "missing") + stderr_log = task.get("source_stderr_log_path", "missing") + report_json = json.dumps({ + "checks": ["log_metrics_recovery"], + "notes": "Recovered metrics from saved logs.", + "completed_at": "YYYY-MM-DD HH:MM:SS", + "commit_sha": "", + "metrics": { + "val_bpb": 1.24, + "training_seconds": 300.1, + "total_seconds": 360.4, + "startup_seconds": 25.8, + "peak_vram_mb": 11967.8, + "mfu_percent": 2.15, + "total_tokens_M": 140.5, + "num_steps": 268, + "num_params_M": 11.5, + "depth": 4, + }, + }, indent=2) + return ( + "METRICS RECOVERY (focused task). Do not read protocol or stage docs.\n\n" + "Task (inline):\n" + f"{task_json}\n\n" + "Do not rerun training. Do not edit code. Do not create a commit.\n\n" + f"Inspect logs for source run {source_run_id!r}:\n" + f" stdout: {stdout_log}\n" + f" stderr: {stderr_log}\n\n" + "Recover canonical metrics from those logs if present, then print the summary block below. " + "If unrecoverable, use empty \"metrics\": {} and explain in notes.\n\n" + "AUTORESEARCH_DCA_SUMMARY_BEGIN\n" + f"{report_json}\n" + "AUTORESEARCH_DCA_SUMMARY_END\n" + ) + + +def _build_sync_resolution_prompt(task: dict[str, Any]) -> str: + """Prompt for sync-resolution: merge baseline into seed in the seed worktree, resolve conflicts, commit.""" + baseline_branch = task.get("baseline_branch", "master") + seed_id = task.get("seed_id", "") + return ( + "SYNC RESOLUTION (merge baseline into seed). You are in the seed worktree; the current branch is the seed branch.\n\n" + "The run could not sync this worktree with the baseline branch because the merge had conflicts.\n\n" + "Steps:\n" + f"1. Merge the baseline branch into the current branch: git merge {baseline_branch!r}\n" + "2. Resolve any conflicts, then commit the merge (e.g. git add . && git commit -m 'Merge baseline into seed').\n" + "3. Do not run the training entrypoint.\n" + "4. Print the following block so the runner can confirm success:\n\n" + "AUTORESEARCH_DCA_SUMMARY_BEGIN\n" + '{"checks":["sync_resolution"],"notes":"Merged baseline into seed; conflicts resolved.","completed_at":"YYYY-MM-DD HH:MM:SS","commit_sha":"","metrics":{}}\n' + "AUTORESEARCH_DCA_SUMMARY_END\n" + ) + + +def _build_merge_resolution_prompt(task: dict[str, Any]) -> str: + """Lightweight prompt for merge-resolution DCA: no protocol/docs, just commit, merge, report.""" + task_json = json.dumps(task, indent=2) + target_branch = task.get("baseline_branch", "master") # branch we want to merge into (e.g. master) + worktree_path = task.get("worktree_path") or "" + seed_id = task.get("seed_id", "") + last_metrics = task.get("last_metrics") or {} + last_summary = task.get("last_summary") or {} + notes = last_summary.get("notes", "Merge resolution: committed and merged into baseline.") + completed_at = last_summary.get("completed_at", "YYYY-MM-DD HH:MM:SS") + report_json = json.dumps({ + "checks": ["merge_resolution"], + "notes": notes, + "completed_at": completed_at, + "commit_sha": "", + "metrics": last_metrics, + }, indent=2) + + if seed_id == BASELINE_SEED_ID: + # We are resolving the merge of __baseline__ INTO target_branch (e.g. master). + # git merge X = merge X into current branch; so we need to be on target_branch, then git merge __baseline__. + cwd_note = ( + "Your working directory is the project root (main repo). " + "Do NOT run the merge from the __baseline__ worktree: that would merge the wrong way.\n\n" + ) + steps = ( + "Steps:\n" + f"1. Find where {target_branch!r} is checked out: run git worktree list and identify the path whose branch is {target_branch!r} (often the main repo).\n" + f"2. cd to that directory, then run: git merge {BASELINE_SEED_ID!r}. Resolve any conflicts, then commit the merge.\n" + f" Correct example (merge __baseline__ into {target_branch}):\n" + f" git worktree list\n" + f" cd # e.g. main repo\n" + f" git merge {BASELINE_SEED_ID!r}\n" + " Wrong (do not do this): cd to the __baseline__ worktree and run git merge master — that merges master into __baseline__.\n" + "3. Do not run the training entrypoint; the experiment already completed and metrics exist.\n" + "4. Print the DCA summary block below (same metrics as the previous run). Include the current commit SHA (after committing the merge) in the DCA summary JSON.\n\n" + ) + else: + # Normal seed: we need to merge the SEED branch INTO the baseline branch (so baseline gets the seed's changes). + # Do NOT merge baseline into seed — that is the wrong direction. Work in the project root on the baseline branch. + if worktree_path: + cwd_note = ( + "Your working directory is the project root. " + f"The seed worktree is at {worktree_path!r} (use it only to commit any pending changes on the seed branch).\n\n" + ) + else: + cwd_note = ( + "Your working directory is the project root. " + f"The seed worktree is at component_system/history/worktrees/{seed_id!r} (use it only to commit any pending changes).\n\n" + ) + steps = ( + "Steps:\n" + "1. Commit any uncommitted changes in the seed worktree so the seed branch is complete.\n" + f"2. In the project root (main repo): checkout the baseline branch, then merge the seed branch into it:\n" + f" git checkout {target_branch!r}\n" + f" git merge {seed_id!r}\n" + " Resolve any conflicts, then commit the merge. The result must be: the baseline branch contains the seed's changes (merge direction: seed → baseline).\n" + "3. Do not run the training entrypoint; the experiment already completed and metrics exist.\n" + "4. Print the DCA summary block below (same metrics as the previous run). Use the merge commit SHA from the baseline branch (after the merge, from project root: git rev-parse HEAD).\n\n" + ) + + return ( + "MERGE RESOLUTION (focused task). Do not read protocol or stage docs.\n\n" + "Task (inline):\n" + f"{task_json}\n\n" + f"{cwd_note}" + f"{steps}" + "AUTORESEARCH_DCA_SUMMARY_BEGIN\n" + f"{report_json}\n" + "AUTORESEARCH_DCA_SUMMARY_END\n" + ) + + +def _build_prompt(stage: str, task: dict[str, Any], task_path: Path) -> str: + """Build the agent prompt for a stage. Prompt types (by weight): + - P: full header (protocol, stage doc, baseline files, task) + P workflow. Heavy. + - DCA metrics_recovery: lightweight; task + log paths, report shape (no protocol/docs). Light. + - DCA merge_resolution: lightweight; task + commit, merge, report (no protocol/docs). Light. + - DCA baseline_measurement: full header + baseline retry/OOM/commit/run. Heavy. + - DCA normal: full header + adapt/run/commit/report. Heavy. + """ + task_json = json.dumps(task, indent=2) + rel_task = task_path.relative_to(PROJECT_ROOT).as_posix() + worktree_path = task.get("worktree_path", "component_system/history/worktrees") + agent_cwd = _agent_cwd(worktree_path) + worktree_dir = Path(agent_cwd) + + # Worktree runs must stay entirely within the copied seed workspace to avoid external_directory requests. + in_worktree = worktree_dir.resolve() != PROJECT_ROOT.resolve() + if in_worktree: + context_protocol = " - component_system/protocol.md" + docs = "\n".join(f" - component_system/{doc}" for doc in STAGE_DOCS[stage]) + task_block = ( + "Task content (provided inline; do not look up any external task file):\n" + f"{task_json}\n\n" + ) + worktree_note = ( + "Your working directory is the assigned workflow worktree (your current directory).\n" + "All required file context is already copied into this worktree under component_system/.\n" + "Use only paths relative to your current working directory. " + "Do not request access to absolute paths, parent-directory paths, or files outside the worktree.\n" + ) + scope_note = "Do not edit files outside the worktree unless the prompt explicitly requires it.\n\n" + else: + context_protocol = " - component_system/protocol.md" + docs = "\n".join(f" - component_system/{doc}" for doc in STAGE_DOCS[stage]) + task_path_rel = f" - {rel_task}" + task_block = f"Task file:\n{task_path_rel}\n\nTask content:\n{task_json}\n\n" + worktree_note = "Your working directory is the project root.\n" + scope_note = "Do not edit files outside your current directory (project root) unless the prompt explicitly requires it.\n\n" + + required_context = ( + "Required context (read first; paths relative to your cwd):\n" + f" - component_system/protocol.md\n" + f"{docs}\n" + ) + baseline_files_note = ( + "Baseline reference files (workflow-managed; read-only):\n" + " - component_system/baseline_branches.json (per-branch baseline mapping)\n" + " - component_system/baseline_metrics.json (baseline run metrics)\n" + "The workflow writes these; only read them for context.\n\n" + ) + header = ( + "You are working on the autoresearch component-system workflow.\n\n" + f"{required_context}\n" + f"{baseline_files_note}" + f"{task_block}" + f"{worktree_note}" + f"{scope_note}" + ) + + if stage == "p": + if in_worktree: + p_workflow = ( + "Workflow:\n" + "1. Refine the seed prompt into a concrete implementation idea.\n" + "2. Implement the first generated version of that idea in the provided worktree.\n" + "3. Create a git commit in the seed branch (current branch in the worktree).\n" + "4. Print a JSON summary between these exact markers:\n" + "AUTORESEARCH_P_SUMMARY_BEGIN\n" + '{"idea":"...","target_component":"model|optimizer|trainer","description":"...","source_refs":["..."],"commit_sha":"...","completed_at":"YYYY-MM-DD HH:MM:SS"}\n' + "AUTORESEARCH_P_SUMMARY_END\n" + "One branch per seed: you are already on the seed branch in the worktree.\n" + "Do not merge branches; only the DCA stage may trigger a merge into baseline.\n" + ) + else: + p_workflow = ( + "Workflow:\n" + "1. Refine the seed prompt into a concrete implementation idea.\n" + "2. Implement the first generated version of that idea in the current directory (project root).\n" + "3. Create a git commit on the current branch.\n" + "4. Print a JSON summary between these exact markers:\n" + "AUTORESEARCH_P_SUMMARY_BEGIN\n" + '{"idea":"...","target_component":"model|optimizer|trainer","description":"...","source_refs":["..."],"commit_sha":"...","completed_at":"YYYY-MM-DD HH:MM:SS"}\n' + "AUTORESEARCH_P_SUMMARY_END\n" + "One branch per seed: you are in the project root; use the current branch for your commit.\n" + "Do not merge branches; only the DCA stage may trigger a merge into baseline.\n" + ) + return header + ( + "You are the P stage.\n\n" + "## Read results.tsv first (avoid idea duplication)\n" + "Before choosing a hypothesis, read `results.tsv` in your cwd if it exists. " + "Use it to avoid proposing ideas already tried or discarded; only repeat an idea if you have a clear new angle (e.g. different implementation or target component). " + "See component_system/PDCA-PLAN.md for full guidance.\n\n" + f"{p_workflow}" + ) + if stage == "dca": + sync_resolution = task.get("sync_resolution") is True + merge_resolution = task.get("merge_resolution") is True + metrics_recovery = task.get("metrics_recovery") is True + if sync_resolution: + return _build_sync_resolution_prompt(task) + if merge_resolution: + return _build_merge_resolution_prompt(task) + if metrics_recovery: + return _build_metrics_recovery_prompt(task) + dca_cmd, dca_note = _dca_command_guidance() + baseline_measurement = task.get("seed_id") == "__baseline__" + conflict_block = "" + if baseline_measurement: + return header + conflict_block + ( + "BASELINE MEASUREMENT: establish the first reference metrics in the dedicated baseline worktree.\n" + "You must retry until the run completes successfully and you can report real metrics. Do not report empty metrics and stop.\n" + "If training fails with CUDA out of memory (OOM): the default batch size is for H100. Reduce device_batch_size (and if needed total_batch_size) in component_system/components/trainer.py (TrainingSettings) so training fits in available VRAM, then rerun until the baseline run completes. Only trivial execution fixes (e.g. batch size) are allowed; do not change model architecture or training logic.\n" + "If you modified any files (e.g. batch size for OOM), you must commit those changes on the baseline branch before reporting. An uncommitted worktree causes the follow-up merge to fail.\n" + f"Run the canonical command (≥{DCA_CANONICAL_RUN_TIMEOUT_SECONDS}s): `{dca_cmd} > training.log 2>&1`. Set your command/tool timeout to at least {DCA_CANONICAL_RUN_TIMEOUT_SECONDS} seconds. After the run, inspect training.log to confirm completion and recover or verify metrics.\n" + f"({dca_note})\n" + "Report the final result in JSON between these exact markers once training has completed successfully. Include the current commit SHA in the summary (commit any changes first).\n" + "AUTORESEARCH_DCA_SUMMARY_BEGIN\n" + '{"checks":["baseline_measurement"],"notes":"Measured the current baseline in the dedicated baseline worktree.","completed_at":"YYYY-MM-DD HH:MM:SS","commit_sha":"...","metrics":{"val_bpb":1.239972,"training_seconds":300.1,"total_seconds":360.4,"startup_seconds":25.8,"peak_vram_mb":11967.8,"mfu_percent":2.15,"total_tokens_M":140.5,"num_steps":268,"num_params_M":11.5,"depth":4}}\n' + "AUTORESEARCH_DCA_SUMMARY_END\n" + "If after all retries (including batch size reduction) metrics are still unavailable, only then print the same object with an empty metrics object and explain in notes.\n" + ) + return header + conflict_block + ( + "You are the DCA stage.\n" + "Do not put forward new ideas or optimize for better metrics. Your only goal is to make the P-stage code run and report the result. " + '"Adapt or fix" means: fix bugs, import/runtime errors, OOM (e.g. reduce batch size), and config/path issues only. ' + "Do not change model architecture, optimizer logic, hyperparameters, or training logic to improve results. " + "The task \"prompt\" is for context only; do not treat it as a goal to achieve in this stage.\n\n" + "Workflow:\n" + "1. Adapt or fix the generated code in the seed worktree until it runs.\n" + f"2. Run the canonical command (≥{DCA_CANONICAL_RUN_TIMEOUT_SECONDS}s): `{dca_cmd} > training.log 2>&1` (or `... 2>&1 | tee training.log` to also see output). Set your command/tool timeout to at least {DCA_CANONICAL_RUN_TIMEOUT_SECONDS} seconds. After the run, inspect training.log to confirm completion and recover or verify metrics.\n" + f" ({dca_note})\n" + "3. If it fails for a simple reason, fix and rerun.\n" + "4. Create a git commit in the seed branch for your changes.\n" + "5. Report the final result in JSON between these exact markers. Include the current commit SHA in the summary.\n" + " Use this exact shape and include numeric metric values when available:\n" + "AUTORESEARCH_DCA_SUMMARY_BEGIN\n" + '{"checks":["entrypoint"],"notes":"what you adapted or fixed","completed_at":"YYYY-MM-DD HH:MM:SS","commit_sha":"...","metrics":{"val_bpb":1.239972,"training_seconds":300.1,"total_seconds":360.4,"startup_seconds":25.8,"peak_vram_mb":11967.8,"mfu_percent":2.15,"total_tokens_M":140.5,"num_steps":268,"num_params_M":11.5,"depth":4}}\n' + "AUTORESEARCH_DCA_SUMMARY_END\n" + " Do not omit the markers. Prefer this exact JSON report over prose. If metrics are unavailable,\n" + " still print the same object with an empty metrics object.\n" + "Do not edit baseline_branches.json or baseline_metrics.json (workflow writes them; read only). Do not merge branches yourself; the system will evaluate and promote if appropriate.\n" + ) + raise ValueError(f"Unknown stage: {stage}") + + +def _append_results_tsv(seed_id: str, run_metrics: dict[str, Any], signal: str, description: str) -> None: + status = "KEEP" if signal == "positive_signal" else "DISCARD" + val_bpb = run_metrics.get("val_bpb", "") + peak_vram_mb = run_metrics.get("peak_vram_mb", 0) + memory_gb = round(float(peak_vram_mb) / 1024, 2) if peak_vram_mb else "" + write_header = not RESULTS_TSV.exists() + with open(RESULTS_TSV, "a", encoding="utf-8") as handle: + if write_header: + handle.write("commit\tval_bpb\tmemory_gb\tstatus\tdescription\n") + handle.write(f"{seed_id}\t{val_bpb}\t{memory_gb}\t{status}\t{description}\n") + + +def _regenerate_progress_png() -> None: + try: + import matplotlib + + matplotlib.use("Agg") + import matplotlib.pyplot as plt + import pandas as pd + except ImportError: + return + + if not RESULTS_TSV.exists(): + return + + try: + df = pd.read_csv(RESULTS_TSV, sep="\t") + df["val_bpb"] = pd.to_numeric(df["val_bpb"], errors="coerce") + df["memory_gb"] = pd.to_numeric(df["memory_gb"], errors="coerce") + df["status"] = df["status"].str.strip().str.upper() + valid = df[df["val_bpb"].notna()].copy().reset_index(drop=True) + if valid.empty: + return + + baseline_bpb = valid.loc[0, "val_bpb"] + kept = valid[valid["status"] == "KEEP"] + best = float(kept["val_bpb"].min()) if not kept.empty else float(baseline_bpb) + + fig, ax = plt.subplots(figsize=(14, 7)) + ax.scatter(valid.index, valid["val_bpb"], c="#94a3b8", s=18, alpha=0.6, label="Runs") + if not kept.empty: + ax.scatter(kept.index, kept["val_bpb"], c="#38bdf8", s=42, label="Promoted") + ax.step(kept.index, kept["val_bpb"].cummin(), where="post", color="#0ea5e9", linewidth=2) + ax.set_xlabel("Experiment #") + ax.set_ylabel("Validation BPB (lower is better)") + ax.set_title("Component System Progress") + margin = (baseline_bpb - best) * 0.15 if baseline_bpb != best else 0.005 + ax.set_ylim(best - margin, float(baseline_bpb) + margin) + ax.grid(True, alpha=0.2) + ax.legend(loc="upper right") + plt.tight_layout() + plt.savefig(PROGRESS_PNG, dpi=150, bbox_inches="tight") + plt.close(fig) + except Exception: + traceback.print_exc() + + +def _worker(stage: str, lane: str = "any") -> None: + worker_name = stage.upper() if lane == "any" else f"{stage.upper()}-{lane.upper()}" + print(f"[daemon] worker-{worker_name} started") + def eligible(payload: dict) -> bool: + return bool(WORKFLOW.is_seed_eligible_for_stage(payload.get("seed_id"), stage)) + + while not _shutdown: + task_path = claim_pending(stage, lane=lane, eligible_fn=eligible) + if task_path is None: + time.sleep(POLL_INTERVAL) + continue + + try: + task = read_task(task_path) + seed_id = task.get("seed_id") + run_id = task.get("run_id") + if not seed_id or not run_id: + move_to_error(task_path) + continue + started_seed = None + if stage == "direct": + started_seed, _ = WORKFLOW.mark_direct_code_run_started(seed_id, run_id) + else: + started_seed, _ = WORKFLOW.mark_run_started(seed_id, run_id) + if ( + stage == "dca" + and task.get("metrics_recovery") is not True + ): + started_seed = WORKFLOW.ensure_seed_worktree_ready(seed_id) + print(f"[{stage.upper()}] picked up {task['task_id']} for {seed_id}") + + worktree_path = task.get("worktree_path") + if started_seed is not None and started_seed.worktree_path is not None: + worktree_path = started_seed.worktree_path + # Merge-resolution and metrics_recovery DCA run from project root; sync_resolution runs in seed worktree + if stage == "dca" and ( + task.get("merge_resolution") is True or task.get("metrics_recovery") is True + ) and task.get("sync_resolution") is not True: + worktree_path = None + + if worktree_path: + _sync_worktree_context(worktree_path) + + if stage == "direct": + prompt = _build_direct_code_prompt(task["prompt"]) + else: + prompt = _build_prompt(stage, task, task_path) + prompt_path = _write_prompt_file(run_id, prompt) + prompt_path_str = str(prompt_path) + exit_code, stdout, stderr, stdout_log_path, stderr_log_path = _invoke_agent( + prompt, stage, run_id, worktree_path=worktree_path + ) + + combined_output = _combined_output(stdout, stderr) + salvaged_dca = _should_salvage_completed_dca(stage, exit_code, combined_output) + if exit_code == 0 or salvaged_dca: + if stage == "p": + WORKFLOW.finish_p_run( + seed_id, + run_id, + stdout, + str(stdout_log_path) if stdout_log_path else None, + str(stderr_log_path) if stderr_log_path else None, + prompt_path_str, + ) + elif stage == "direct": + WORKFLOW.finish_direct_code_run( + seed_id, + run_id, + stdout, + stderr=stderr, + log_path=str(stdout_log_path) if stdout_log_path else None, + stderr_log_path=str(stderr_log_path) if stderr_log_path else None, + prompt_path=prompt_path_str, + ) + else: + if task.get("sync_resolution") is True: + WORKFLOW.finish_sync_resolution(seed_id, run_id) + else: + run = WORKFLOW.finish_dca_run( + seed_id, + run_id, + stdout, + stderr=stderr, + log_path=str(stdout_log_path) if stdout_log_path else None, + stderr_log_path=str(stderr_log_path) if stderr_log_path else None, + prompt_path=prompt_path_str, + metrics_recovery=task.get("metrics_recovery") is True, + merge_resolution=task.get("merge_resolution") is True, + ) + if not run.summary.get("metrics_recovery_queued"): + description = run.summary.get("idea") or run.summary.get("notes") or seed_id + _append_results_tsv(seed_id, run.metrics, run.signal or "error", str(description)) + _regenerate_progress_png() + if salvaged_dca: + WORKFLOW.seed_repo.append_event( + seed_id, + "dca.salvaged", + f"DCA output contained final metrics, so the run was accepted despite agent exit code {exit_code}.", + run_id=run_id, + ) + move_to_done(task_path) + print(f"[{stage.upper()}] task {task['task_id']} done") + else: + if stage == "direct": + WORKFLOW.mark_direct_code_run_failed( + seed_id, + run_id, + _agent_failure_reason(exit_code, stdout, stderr), + task_path=task_path, + prompt_path=prompt_path_str, + log_path=str(stdout_log_path) if stdout_log_path else None, + stderr_log_path=str(stderr_log_path) if stderr_log_path else None, + ) + else: + WORKFLOW.mark_run_failed( + seed_id, + run_id, + _agent_failure_reason(exit_code, stdout, stderr), + task_path=task_path, prompt_path=prompt_path_str, + ) + print(f"[{stage.upper()}] task {task['task_id']} failed") + except SyncResolutionQueued: + # Sync with baseline failed; sync-resolution DCA was queued. Move P task to error so we don't retry it. + if task_path.exists(): + move_to_error(task_path) + continue + except DuplicateRunStartError: + # Run was already started (e.g. restored in-progress task). Move task to error to avoid double run. + if task_path.exists(): + move_to_error(task_path) + continue + except Exception as exc: + traceback.print_exc() + if not task_path.exists(): + continue + try: + task = read_task(task_path) + seed_id = task.get("seed_id") + run_id = task.get("run_id") + if not seed_id or not run_id: + continue + prompt_path_str = None + if run_id: + p_path = LOG_DIR / f"{run_id}.prompt.txt" + if p_path.exists(): + prompt_path_str = str(p_path) + if stage == "direct": + WORKFLOW.mark_direct_code_run_failed( + seed_id, + run_id, + str(exc), + task_path=task_path, + prompt_path=prompt_path_str, + ) + else: + WORKFLOW.mark_run_failed( + seed_id, run_id, str(exc), + task_path=task_path, prompt_path=prompt_path_str, + ) + except Exception: + traceback.print_exc() + + print(f"[daemon] worker-{worker_name} stopped") + + +def main() -> None: + global _shutdown + signal.signal(signal.SIGINT, _signal_handler) + if sys.platform != "win32": + signal.signal(signal.SIGTERM, _signal_handler) + + ensure_queue_layout() + restored = restore_in_progress_tasks() + total_restored = sum(restored.values()) + if total_restored: + print( + "[daemon] restored in_progress tasks " + f"(p={restored['p']}, dca={restored['dca']}, direct={restored['direct']})" + ) + daemon_heartbeat() + agent = os.environ.get("PDCA_AGENT", "claude") + print(f"[daemon] starting component-system daemon — agent={agent}, workers=P/DCA-GPU/DCA-AUX/DIRECT") + + pools: list[ThreadPoolExecutor] = [] + stage_specs = ( + ("p", "any", 2, "pdca-p"), + ("dca", "gpu", 1, "pdca-dca-gpu"), + ("dca", "aux", 1, "pdca-dca-aux"), + ("direct", "any", 1, "pdca-direct"), + ) + for stage, lane, worker_count, prefix in stage_specs: + pool = ThreadPoolExecutor(max_workers=worker_count, thread_name_prefix=prefix) + pools.append(pool) + for _ in range(worker_count): + pool.submit(_worker, stage, lane) + + last_heartbeat = time.monotonic() + try: + while not _shutdown: + time.sleep(1.0) + if not _shutdown and (time.monotonic() - last_heartbeat) >= 5.0: + daemon_heartbeat() + last_heartbeat = time.monotonic() + except KeyboardInterrupt: + pass + finally: + _shutdown = True + if DAEMON_HEARTBEAT_PATH.exists(): + try: + DAEMON_HEARTBEAT_PATH.unlink() + except OSError: + pass + for pool in pools: + pool.shutdown(wait=True) + + print("[daemon] all workers stopped") + + +if __name__ == "__main__": + main() diff --git a/component_system/run_arxiv.py b/component_system/run_arxiv.py new file mode 100644 index 00000000..3595f9f5 --- /dev/null +++ b/component_system/run_arxiv.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +""" +arXiv search/fetch script using the arxiv Python library. +Supports CLI args for query, id_list, max_results, sort, output format, and PDF download. + +In this project the arxiv dependency is provided by uv. Run with: + uv run python component_system/run_arxiv.py --query "machine learning" --max-results 5 + uv run python component_system/run_arxiv.py --id 1605.08386v1 --output json + uv run python component_system/run_arxiv.py --query "transformer" --download-dir ./papers +""" +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +try: + import arxiv +except ImportError: + print("Install the arxiv package: pip install arxiv", file=sys.stderr) + sys.exit(1) + + +def _sort_criterion(s: str) -> arxiv.SortCriterion: + m = { + "relevance": arxiv.SortCriterion.Relevance, + "submitteddate": arxiv.SortCriterion.SubmittedDate, + "lastupdateddate": arxiv.SortCriterion.LastUpdatedDate, + } + key = s.strip().lower().replace(" ", "") + if key not in m: + raise ValueError(f"Invalid sort_by: {s}. Choose: relevance, submittedDate, lastUpdatedDate") + return m[key] + + +def _sort_order(s: str) -> arxiv.SortOrder: + m = { + "ascending": arxiv.SortOrder.Ascending, + "descending": arxiv.SortOrder.Descending, + } + key = s.strip().lower() + if key not in m: + raise ValueError(f"Invalid sort_order: {s}. Choose: ascending, descending") + return m[key] + + +def _result_to_dict(r: arxiv.Result) -> dict: + return { + "entry_id": r.entry_id, + "title": r.title, + "summary": (r.summary or "").strip(), + "authors": [a.name for a in r.authors], + "published": r.published.isoformat() if r.published else None, + "updated": r.updated.isoformat() if r.updated else None, + "primary_category": getattr(r, "primary_category", None) or "", + "categories": getattr(r, "categories", []) or [], + "pdf_url": getattr(r, "pdf_url", None) or "", + "links": [{"href": l.href, "title": getattr(l, "title", None)} for l in (r.links or [])], + } + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Search or fetch arXiv papers via the arxiv Python library.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + "--query", + "-q", + type=str, + default="", + help="Search query (e.g. 'transformer' or 'au:smith AND ti:neural'). Ignored if --id is set.", + ) + parser.add_argument( + "--id", + dest="id_list", + type=str, + nargs="+", + default=None, + metavar="ARXIV_ID", + help="One or more arXiv IDs (e.g. 1605.08386v1). If set, --query is ignored.", + ) + parser.add_argument( + "--max-results", + "-n", + type=int, + default=10, + help="Maximum number of results to return.", + ) + parser.add_argument( + "--sort-by", + type=str, + default="relevance", + choices=["relevance", "submittedDate", "lastUpdatedDate"], + help="Sort criterion for results.", + ) + parser.add_argument( + "--sort-order", + type=str, + default="descending", + choices=["ascending", "descending"], + help="Sort order.", + ) + parser.add_argument( + "--output", + "-o", + type=str, + default="text", + choices=["text", "json"], + help="Output format: text (one line per paper) or json.", + ) + parser.add_argument( + "--download-dir", + type=str, + default=None, + metavar="DIR", + help="If set, download PDF for each result into this directory.", + ) + parser.add_argument( + "--verbose", + "-v", + action="store_true", + help="Print progress (e.g. download paths).", + ) + args = parser.parse_args() + + if args.id_list: + search = arxiv.Search(id_list=args.id_list, max_results=len(args.id_list) or None) + else: + if not args.query.strip(): + parser.error("Either --query or --id must be provided.") + sort_by = _sort_criterion(args.sort_by) + sort_order = _sort_order(args.sort_order) + search = arxiv.Search( + query=args.query, + max_results=args.max_results, + sort_by=sort_by, + sort_order=sort_order, + ) + + client = arxiv.Client() + results = list(client.results(search)) + + if args.download_dir: + d = Path(args.download_dir) + d.mkdir(parents=True, exist_ok=True) + for r in results: + try: + path = r.download_pdf(dirpath=str(d)) + if args.verbose and path: + print(f"Downloaded: {path}", file=sys.stderr) + except Exception as e: + print(f"Download failed for {r.entry_id}: {e}", file=sys.stderr) + + if args.output == "json": + out = [_result_to_dict(r) for r in results] + print(json.dumps(out, indent=2, ensure_ascii=False)) + else: + for r in results: + print(r.title) + print(f" {r.entry_id} {getattr(r, 'pdf_url', '') or ''}") + if r.summary: + summary = (r.summary or "").strip() + if len(summary) > 200: + summary = summary[:200] + "..." + print(f" {summary}") + print() + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/component_system/services/workflow.py b/component_system/services/workflow.py new file mode 100644 index 00000000..3fab2a1e --- /dev/null +++ b/component_system/services/workflow.py @@ -0,0 +1,1796 @@ +from __future__ import annotations + +import json +import re +import shutil +import subprocess +from pathlib import Path +from typing import Any + +from component_system.config import DEFAULT_BASELINE_BRANCH, PROMOTION_THRESHOLD +from component_system.domain.models import ( + DashboardColumn, + DashboardViewModel, + PlanIdea, + RunStatus, + SeedRecord, + SeedStatus, + StageName, + StageRun, +) +from component_system.repositories.state import ( + BaselineBranchMapRepository, + BaselineMetricsRepository, + RunRepository, + SeedRepository, +) +from component_system.task import ( + COMPONENT_SYSTEM_ROOT, + WORKTREE_ROOT, + get_daemon_status, + move_to_error, + now_ts, + new_run_id, + new_seed_id, + read_task, + write_task, +) + +SUMMARY_MARKERS = { + "p": ("AUTORESEARCH_P_SUMMARY_BEGIN", "AUTORESEARCH_P_SUMMARY_END"), + "dca": ("AUTORESEARCH_DCA_SUMMARY_BEGIN", "AUTORESEARCH_DCA_SUMMARY_END"), +} + +BASELINE_SEED_ID = "__baseline__" + +# Short display labels for timeline (kind -> one-line text). Events not in this map use message as-is (truncated if long). +TIMELINE_SHORT_MESSAGES = { + "seed.created": "Seed created", + "seed.updated": "Seed updated", + "seed.worktree_ready": "Worktree ready", + "ralph.enabled": "Ralph loop enabled", + "ralph.disabled": "Ralph loop disabled", + "p.queued": "Plan queued", + "p.started": "Plan started", + "p.completed": "Plan completed", + "p.failed": "Plan failed", + "dca.queued": "DCA queued", + "dca.started": "DCA started", + "dca.completed": "DCA completed", + "dca.merge_failed": "Merge into baseline failed", + "p.sync_resolution_queued": "Sync failed; merge resolution queued", + "p.sync_resolution_done": "Sync resolution done; Plan re-queued", + "dca.failed": "DCA failed", + "direct_code.failed": "Direct code failed", +} + + +def _timeline_display_events(events: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Return events in reverse order (newest first), deduplicated by (kind, message), with concise display text.""" + if not events: + return [] + reversed_list = list(reversed(events)) + seen: set[tuple[str, str]] = set() + out: list[dict[str, Any]] = [] + for e in reversed_list: + kind = e.get("kind", "") + message = e.get("message", "") + key = (kind, message) + if key in seen: + continue + seen.add(key) + display = TIMELINE_SHORT_MESSAGES.get(kind) + if display is not None: + # Keep commit_sha / target_branch in a short suffix when present + parts = [display] + if e.get("commit_sha"): + parts.append(f"commit: {e.get('commit_sha', '')[:7]}") + if e.get("target_branch"): + parts.append(f"→ {e.get('target_branch')}") + display = " · ".join(parts) + else: + display = message if len(message) <= 80 else message[:77] + "..." + out.append({**e, "display_message": display}) + return out + + +class GitCommandError(RuntimeError): + pass + + +class SyncResolutionQueued(RuntimeError): + """Raised when P run cannot start because worktree sync with baseline failed; a sync-resolution DCA task was queued.""" + + +class DuplicateRunStartError(RuntimeError): + """Raised when mark_run_started is called for a run that was already started (e.g. restored in-progress task).""" + + +class GitService: + def __init__(self) -> None: + pass + + def _run_git(self, *args: str, cwd: Path | None = None) -> str: + try: + result = subprocess.run( + ["git", *args], + cwd=str(cwd) if cwd else None, + capture_output=True, + text=True, + check=True, + ) + except FileNotFoundError as exc: + raise GitCommandError("Git is not installed or not available on PATH.") from exc + except subprocess.CalledProcessError as exc: + stderr = (exc.stderr or exc.stdout or "").strip() + raise GitCommandError(stderr or f"git {' '.join(args)} failed") from exc + return result.stdout.strip() + + def repo_root(self) -> Path: + return Path(self._run_git("rev-parse", "--show-toplevel")) + + def current_head(self) -> str: + return self._run_git("rev-parse", "HEAD") + + def branch_exists(self, branch: str) -> bool: + try: + self._run_git("rev-parse", "--verify", branch) + return True + except GitCommandError: + return False + + def ensure_branch(self, branch: str, start_point: str) -> None: + if not self.branch_exists(branch): + self._run_git("branch", branch, start_point) + + def list_branches(self) -> list[str]: + output = self._run_git("branch", "--format=%(refname:short)") + branches = [line.strip() for line in output.splitlines() if line.strip()] + if not branches: + # Unborn repositories can have HEAD pointing to a branch name even before first commit. + try: + head_branch = self._run_git("symbolic-ref", "--short", "HEAD").strip() + if head_branch: + branches.append(head_branch) + except GitCommandError: + pass + return sorted(set(branches)) + + @staticmethod + def is_seed_specific_branch(branch: str) -> bool: + """True if this branch is the single working branch for a seed (seed_id), not a baseline choice.""" + if branch == BASELINE_SEED_ID: + return True + # One branch per seed: seed- + 6 hex chars, e.g. seed-e57b95 + if branch.startswith("seed-") and len(branch) == 11 and all( + c in "abcdef0123456789" for c in branch[5:] + ): + return True + return False + + def setup_error(self) -> str | None: + try: + self.repo_root() + return None + except GitCommandError as exc: + return str(exc) + + def setup_error_for_branches(self, baseline_branch: str) -> str | None: + try: + root = self.repo_root() + if not baseline_branch: + return "Please select a baseline branch." + if not self.branch_exists(baseline_branch): + return ( + f"Git repo found at {root}, but branch {baseline_branch!r} does not exist yet. " + "Select an existing baseline branch." + ) + return None + except GitCommandError as exc: + return str(exc) + + def ensure_seed_worktrees(self, seed: SeedRecord) -> SeedRecord: + """Ensure the seed worktree exists on the single branch for this seed: seed_id (SSOT).""" + repo_head = self.current_head() + self.ensure_branch(seed.baseline_branch, repo_head) + + seed_worktree = WORKTREE_ROOT / seed.seed_id + if seed_worktree.exists(): + seed.worktree_path = str(seed_worktree) + return seed + # One branch per seed: branch name = seed_id, created from baseline. + try: + self._run_git("worktree", "add", "-B", seed.seed_id, str(seed_worktree), seed.baseline_branch) + except GitCommandError as exc: + # Recover from stale git worktree metadata like: + # "__baseline__ is already checked out at /old/path/__baseline__" + if not self._recover_checked_out_worktree_conflict( + seed.seed_id, seed_worktree, seed.baseline_branch, str(exc) + ): + raise + + seed.worktree_path = str(seed_worktree) + return seed + + @staticmethod + def _extract_checked_out_path(error: str) -> Path | None: + # git message example: fatal: '__baseline__' is already checked out at '/path' + match = re.search(r"already checked out at ['\"]([^'\"]+)['\"]", error) + if not match: + return None + return Path(match.group(1)) + + def _recover_checked_out_worktree_conflict( + self, branch: str, target_worktree: Path, start_point: str, error: str + ) -> bool: + if "already checked out at" not in error: + return False + # First, prune stale registrations from missing worktrees. + try: + self._run_git("worktree", "prune") + except GitCommandError: + pass + conflict_path = self._extract_checked_out_path(error) + if conflict_path is not None: + # Force-remove the conflicting worktree from registry (same path after hard reset or different path). + try: + self._run_git("worktree", "remove", "--force", str(conflict_path)) + except GitCommandError: + pass + try: + self._run_git("worktree", "prune") + except GitCommandError: + pass + self._run_git("worktree", "add", "-B", branch, str(target_worktree), start_point) + return True + + def commit_sha(self, ref: str) -> str: + return self._run_git("rev-parse", "--short", ref) + + def _current_branch(self, cwd: Path | None = None) -> str | None: + """Return the current branch name, or None if detached HEAD.""" + try: + branch = self._run_git("branch", "--show-current", cwd=cwd) + return branch.strip() or None + except GitCommandError: + return None + + def head_sha_at(self, cwd: Path) -> str: + """Return the short commit SHA of HEAD in the given worktree directory.""" + return self._run_git("rev-parse", "--short", "HEAD", cwd=cwd) + + def reset_seed_branch_to(self, seed: SeedRecord, ref: str) -> None: + """Reset the seed worktree's branch to the given ref (e.g. commit before P). + No-op for baseline seed or when worktree is missing.""" + if seed.seed_id == BASELINE_SEED_ID: + return + if not seed.worktree_path: + return + worktree_path = Path(seed.worktree_path) + if not worktree_path.is_dir(): + return + self._run_git("reset", "--hard", ref, cwd=worktree_path) + + def sync_seed_worktree_with_baseline(self, seed: SeedRecord) -> None: + """Merge the baseline branch into the seed branch in the seed worktree. + Call before each P run so the worktree has the latest baseline.""" + if seed.seed_id == BASELINE_SEED_ID: + return + if not seed.worktree_path: + return + worktree_path = Path(seed.worktree_path) + if not worktree_path.is_dir(): + return + self._run_git("merge", "--no-edit", seed.baseline_branch, cwd=worktree_path) + + def promote_seed_branch( + self, seed: SeedRecord, target_branch: str | None = None + ) -> str: + """Merge the seed's branch (seed_id) into the target branch. Only DCA Action may call this; Plan must never merge. + If target_branch is None, use seed.baseline_branch (e.g. for normal seed promotion). For __baseline__ completion, + pass the first user seed's selected branch so the merge goes there instead of a fixed config value. + When the target branch is already checked out (e.g. master in the main repo), we merge in place to avoid + 'cannot force update the branch used by worktree' from creating a second worktree on the same branch.""" + merge_into = target_branch if target_branch is not None else seed.baseline_branch + repo_root = self.repo_root() + current = self._current_branch(cwd=repo_root) + + def do_merge(cwd: Path | None) -> None: + self._run_git("merge", "--no-edit", seed.seed_id, cwd=cwd) + + def merge_already_up_to_date(cwd: Path | None) -> bool: + try: + self._run_git( + "merge-base", "--is-ancestor", seed.seed_id, "HEAD", cwd=cwd + ) + return True + except GitCommandError: + return False + + if current == merge_into: + # Target branch is already checked out (e.g. main repo on master). Merge in place. + try: + do_merge(cwd=repo_root) + except GitCommandError as merge_err: + if merge_already_up_to_date(cwd=repo_root): + return self.commit_sha(merge_into) + raise merge_err + return self.commit_sha(merge_into) + + # Target is not current branch: use a temporary worktree with a temp branch so we don't + # try to check out the same branch in two worktrees (Git forbids that). + baseline_worktree = WORKTREE_ROOT / "baseline" + temp_branch = f"__promote_{merge_into}__" + if baseline_worktree.exists(): + try: + self._run_git("worktree", "remove", "--force", str(baseline_worktree)) + except GitCommandError: + pass + if baseline_worktree.exists(): + shutil.rmtree(baseline_worktree, ignore_errors=True) + self._run_git( + "worktree", + "add", + "--force", + "-B", + temp_branch, + str(baseline_worktree), + merge_into, + cwd=repo_root, + ) + try: + try: + do_merge(cwd=baseline_worktree) + except GitCommandError as merge_err: + if merge_already_up_to_date(cwd=baseline_worktree): + result_sha = self._run_git("rev-parse", "HEAD", cwd=baseline_worktree) + self._run_git("branch", "-f", merge_into, result_sha, cwd=repo_root) + return self.commit_sha(merge_into) + raise merge_err + result_sha = self._run_git("rev-parse", "HEAD", cwd=baseline_worktree) + self._run_git("branch", "-f", merge_into, result_sha, cwd=repo_root) + return self.commit_sha(merge_into) + finally: + try: + self._run_git("worktree", "remove", "--force", str(baseline_worktree)) + except GitCommandError: + pass + try: + self._run_git("branch", "-D", temp_branch) + except GitCommandError: + pass + + +class WorkflowService: + def __init__( + self, + seed_repo: SeedRepository | None = None, + run_repo: RunRepository | None = None, + branch_map_repo: BaselineBranchMapRepository | None = None, + metrics_repo: BaselineMetricsRepository | None = None, + git_service: GitService | None = None, + ) -> None: + self.seed_repo = seed_repo or SeedRepository() + self.run_repo = run_repo or RunRepository() + self.branch_map_repo = branch_map_repo or BaselineBranchMapRepository() + self.metrics_repo = metrics_repo or BaselineMetricsRepository() + self.git_service = git_service or GitService() + + @staticmethod + def _seed_worktree_path(seed_id: str) -> str: + return str(WORKTREE_ROOT / seed_id) + + @staticmethod + def _baseline_worktree_path() -> str: + return str(WORKTREE_ROOT / BASELINE_SEED_ID) + + def _normalize_seed_runtime_state(self, seed: SeedRecord) -> SeedRecord: + """Ensure baseline seed worktree_path matches the canonical path.""" + if seed.seed_id != BASELINE_SEED_ID: + return seed + expected_worktree = self._baseline_worktree_path() + if seed.worktree_path == expected_worktree: + return seed + seed.worktree_path = expected_worktree + seed.updated_at = now_ts() + self.seed_repo.save(seed) + return seed + + def ensure_seed_worktree_ready(self, seed_id: str) -> SeedRecord: + """Ensure the runtime seed worktree exists; recreate only when missing.""" + seed = self.require_seed(seed_id) + if seed.seed_id == BASELINE_SEED_ID: + expected_worktree = self._baseline_worktree_path() + if Path(expected_worktree).is_dir(): + if seed.worktree_path != expected_worktree: + seed.worktree_path = expected_worktree + seed.updated_at = now_ts() + self.seed_repo.save(seed) + return seed + seed = self.git_service.ensure_seed_worktrees(seed) + seed.updated_at = now_ts() + self.seed_repo.save(seed) + commit_sha = "" + try: + commit_sha = self.git_service.commit_sha(seed.baseline_branch) + except GitCommandError: + pass + self.seed_repo.append_event( + seed.seed_id, + "seed.worktree_ready", + "Recreated missing baseline worktree before the run started.", + commit_sha=commit_sha or None, + ) + return seed + expected_worktree = self._seed_worktree_path(seed.seed_id) + if Path(expected_worktree).is_dir(): + if seed.worktree_path != expected_worktree: + seed.worktree_path = expected_worktree + seed.updated_at = now_ts() + self.seed_repo.save(seed) + return seed + seed = self.git_service.ensure_seed_worktrees(seed) + seed.updated_at = now_ts() + self.seed_repo.save(seed) + commit_sha = "" + try: + commit_sha = self.git_service.commit_sha(seed.seed_id) + except GitCommandError: + pass + self.seed_repo.append_event( + seed.seed_id, + "seed.worktree_ready", + "Recreated missing seed worktree before the run started.", + commit_sha=commit_sha or None, + ) + return seed + + def _preferred_baseline_branch(self) -> str: + setup_error = self.git_service.setup_error() + if setup_error is not None: + return DEFAULT_BASELINE_BRANCH + try: + branches = [ + branch + for branch in self.git_service.list_branches() + if not self.git_service.is_seed_specific_branch(branch) + ] + except GitCommandError: + return DEFAULT_BASELINE_BRANCH + if branches and DEFAULT_BASELINE_BRANCH in branches: + return DEFAULT_BASELINE_BRANCH + return branches[0] if branches else DEFAULT_BASELINE_BRANCH + + def _first_user_seed_baseline_branch(self) -> str | None: + """Return the baseline_branch of the earliest-created user seed (excluding __baseline__), or None.""" + user_seeds = [s for s in self.seed_repo.list() if s.seed_id != BASELINE_SEED_ID] + if not user_seeds: + return None + first = min(user_seeds, key=lambda s: s.created_at) + return first.baseline_branch or None + + def _enqueue_plan_run(self, seed: SeedRecord, event_kind: str = "p.queued", event_message: str = "Queued Plan stage for the seed.") -> StageRun: + run = StageRun( + run_id=new_run_id("p"), + seed_id=seed.seed_id, + stage=StageName.p, + status=RunStatus.queued, + task_id=new_run_id("task-p"), + created_at=now_ts(), + updated_at=now_ts(), + ) + seed.status = SeedStatus.queued + seed.updated_at = now_ts() + seed.latest_run_id = run.run_id + seed.last_error = None + self.seed_repo.save(seed) + self.run_repo.save(run) + self.seed_repo.append_event(seed.seed_id, event_kind, event_message) + write_task( + "p", + { + "seed_id": seed.seed_id, + "run_id": run.run_id, + "prompt": seed.prompt, + "worktree_path": seed.worktree_path, + }, + task_id=run.task_id, + ) + return run + + def _release_seeds_waiting_for_baseline(self, branch: str) -> None: + """Release seeds that were waiting for baseline result on the given branch.""" + branch_metrics = self.metrics_repo.get_for_branch(branch) + if not branch_metrics or branch_metrics.get("best_val_bpb") is None: + return + waiting_seeds = sorted(self.seed_repo.list(), key=lambda item: item.created_at) + for seed in waiting_seeds: + if seed.seed_id == BASELINE_SEED_ID: + continue + if seed.baseline_branch != branch: + continue + if seed.status is not SeedStatus.queued or seed.latest_run_id is not None: + continue + self._enqueue_plan_run( + seed, + event_kind="p.released", + event_message="Baseline is ready; queued Plan stage for the waiting seed.", + ) + + def _commit_sha_for_branch(self, branch: str) -> str: + """Return current commit SHA for branch, or 'unknown' if unavailable (ensures baseline_metrics never has null commit_sha).""" + try: + sha = self.git_service.commit_sha(branch) + return sha if (isinstance(sha, str) and sha.strip()) else "unknown" + except GitCommandError: + return "unknown" + + @staticmethod + def _status_from_dca_signal(signal: str) -> SeedStatus: + """Centralized mapping from DCA signal to terminal seed status.""" + if signal == "positive_signal": + return SeedStatus.promoted + if signal == "error": + return SeedStatus.failed + return SeedStatus.passed + + def _reconcile_seed_status_signal(self, seed: SeedRecord) -> bool: + """ + Auto-heal known inconsistent terminal combinations from historical data. + + Returns True when the seed was updated and persisted. + """ + if seed.status is SeedStatus.passed and seed.latest_signal == "error": + seed.status = SeedStatus.failed + seed.updated_at = now_ts() + self.seed_repo.save(seed) + self.seed_repo.append_event( + seed.seed_id, + "seed.reconciled", + "Reconciled inconsistent terminal state (passed + error) to failed.", + ) + return True + return False + + def create_seed( + self, + prompt: str, + baseline_branch: str | None = None, + ralph_loop_enabled: bool = False, + ) -> SeedRecord: + seed_id = new_seed_id() + selected_baseline = (baseline_branch or DEFAULT_BASELINE_BRANCH).strip() + seed = SeedRecord( + seed_id=seed_id, + prompt=prompt.strip(), + status=SeedStatus.draft, + created_at=now_ts(), + updated_at=now_ts(), + baseline_branch=selected_baseline, + worktree_path=self._seed_worktree_path(seed_id), + ralph_loop_enabled=ralph_loop_enabled, + ) + self.seed_repo.save(seed) + self.branch_map_repo.set_branch_for_seed(seed_id, selected_baseline) + try: + pass # branch seed_id is created when Plan is queued (ensure_seed_worktrees) + except GitCommandError: + # Keep seed creation non-blocking; branch creation will be retried at P queue time. + pass + self.seed_repo.append_event(seed.seed_id, "seed.created", "Seed created from prompt.") + if ralph_loop_enabled: + self.seed_repo.append_event( + seed.seed_id, + "ralph.enabled", + "Ralph loop enabled; Plan will auto-requeue after each DCA completion.", + ) + return seed + + def create_direct_code_seed(self, prompt: str) -> tuple[SeedRecord, StageRun]: + cleaned_prompt = prompt.strip() + if not cleaned_prompt: + raise RuntimeError("Prompt cannot be empty.") + baseline_branch = self._preferred_baseline_branch() + seed_id = new_seed_id("direct") + now = now_ts() + run = StageRun( + run_id=new_run_id("direct"), + seed_id=seed_id, + stage=StageName.direct, + status=RunStatus.queued, + task_id=new_run_id("task-direct"), + created_at=now, + updated_at=now, + ) + seed = SeedRecord( + seed_id=seed_id, + prompt=cleaned_prompt, + status=SeedStatus.adapting, + created_at=now, + updated_at=now, + baseline_branch=baseline_branch, + worktree_path=str(COMPONENT_SYSTEM_ROOT.parent), + latest_run_id=run.run_id, + plan=PlanIdea( + title="Direct code agent", + target_component="project_root", + description="Direct code agent run requested from the dashboard and executed from the project root.", + ), + ) + self.seed_repo.save(seed) + self.branch_map_repo.set_branch_for_seed(seed_id, baseline_branch) + self.run_repo.save(run) + self.seed_repo.append_event(seed.seed_id, "seed.created", "Seed created from direct code agent prompt.") + self.seed_repo.append_event( + seed.seed_id, + "direct_code.queued", + "Queued direct code agent run from the project root.", + run_id=run.run_id, + ) + write_task( + "direct", + { + "seed_id": seed.seed_id, + "run_id": run.run_id, + "prompt": seed.prompt, + "worktree_path": None, + }, + task_id=run.task_id, + ) + return seed, run + + def _get_or_create_baseline_seed(self, baseline_branch: str | None = None) -> SeedRecord: + """Return the baseline seed for establishing initial val_bpb; create with baseline_branch if missing.""" + seed = self.seed_repo.get(BASELINE_SEED_ID) + if seed is not None: + return self._normalize_seed_runtime_state(seed) + branch = baseline_branch if baseline_branch is not None else ( + self._first_user_seed_baseline_branch() or DEFAULT_BASELINE_BRANCH + ) + seed = SeedRecord( + seed_id=BASELINE_SEED_ID, + prompt="Baseline measurement: run training on current code without changes.", + status=SeedStatus.draft, + created_at=now_ts(), + updated_at=now_ts(), + baseline_branch=branch, + worktree_path=self._baseline_worktree_path(), + ralph_loop_enabled=False, + ) + self.seed_repo.save(seed) + self.branch_map_repo.set_branch_for_seed(BASELINE_SEED_ID, branch) + self.seed_repo.append_event( + seed.seed_id, + "seed.created", + "Baseline seed created for initial measurement.", + ) + return seed + + def ensure_baseline_result(self, baseline_branch: str) -> None: + """ + If there is no baseline result (best_val_bpb) for the given branch, ensure a baseline seed exists for that + branch, ensure its worktree is checked out from baseline_branch, then queue DCA to establish baseline. + Idempotent; safe to call before queue_p for any user seed. Call with seed.baseline_branch. + """ + seed = self._get_or_create_baseline_seed(baseline_branch) + branch_metrics = self.metrics_repo.get_for_branch(baseline_branch) + if branch_metrics and branch_metrics.get("best_val_bpb") is not None: + return + if seed.baseline_branch != baseline_branch: + return + if seed.status in (SeedStatus.dca_queued, SeedStatus.adapting, SeedStatus.running): + return + if seed.status in (SeedStatus.passed, SeedStatus.failed, SeedStatus.promoted): + branch_metrics = self.metrics_repo.get_for_branch(baseline_branch) + if branch_metrics and branch_metrics.get("best_val_bpb") is not None: + return + setup_error = self.git_service.setup_error() + if setup_error is not None: + return + try: + self.git_service.ensure_branch(baseline_branch, self.git_service.current_head()) + except GitCommandError: + return + setup_error = self.git_service.setup_error_for_branches(baseline_branch) + if setup_error is not None: + return + self.ensure_seed_worktree_ready(BASELINE_SEED_ID) + seed = self.require_seed(BASELINE_SEED_ID) + seed.status = SeedStatus.generated + seed.plan = PlanIdea(title="Baseline", description="No changes; measure current baseline.") + seed.updated_at = now_ts() + self.seed_repo.save(seed) + self.seed_repo.append_event( + seed.seed_id, + "baseline.queued", + "Queued DCA to establish baseline result before first seed.", + ) + self.queue_dca(seed.seed_id) + + def set_ralph_loop(self, seed_id: str, enabled: bool) -> SeedRecord: + seed = self.require_seed(seed_id) + if seed.ralph_loop_enabled == enabled: + return seed + seed.ralph_loop_enabled = enabled + seed.updated_at = now_ts() + self.seed_repo.save(seed) + if enabled: + self.seed_repo.append_event( + seed.seed_id, + "ralph.enabled", + "Ralph loop enabled; Plan will auto-requeue after each DCA completion.", + ) + else: + self.seed_repo.append_event(seed.seed_id, "ralph.disabled", "Ralph loop disabled by user.") + return seed + + def can_edit_seed_prompt(self, seed: SeedRecord) -> bool: + return seed.status in {SeedStatus.draft, SeedStatus.queued} + + def update_seed_prompt(self, seed_id: str, prompt: str) -> SeedRecord: + seed = self.require_seed(seed_id) + if not self.can_edit_seed_prompt(seed): + raise RuntimeError("Seed prompt can only be edited before Plan starts.") + updated_prompt = prompt.strip() + if not updated_prompt: + raise RuntimeError("Prompt cannot be empty.") + if updated_prompt == seed.prompt: + return seed + seed.prompt = updated_prompt + seed.updated_at = now_ts() + self.seed_repo.save(seed) + self.seed_repo.append_event(seed.seed_id, "seed.updated", "Seed prompt was edited before execution.") + return seed + + def queue_p(self, seed_id: str) -> StageRun | None: + seed = self.require_seed(seed_id) + branch_metrics = self.metrics_repo.get_for_branch(seed.baseline_branch) if seed_id != BASELINE_SEED_ID else None + has_baseline = branch_metrics is not None and branch_metrics.get("best_val_bpb") is not None + if seed_id != BASELINE_SEED_ID and not has_baseline: + self.ensure_baseline_result(seed.baseline_branch) + branch_metrics = self.metrics_repo.get_for_branch(seed.baseline_branch) + has_baseline = branch_metrics is not None and branch_metrics.get("best_val_bpb") is not None + if not has_baseline: + baseline_seed = self.seed_repo.get(BASELINE_SEED_ID) + # Only wait for baseline when the baseline seed is for this branch (e.g. master). + # For another branch (e.g. dev), no baseline run is queued for it, so allow planning; + # the first DCA completion on this branch will establish baseline metrics. + if baseline_seed is not None and baseline_seed.baseline_branch == seed.baseline_branch: + if not (seed.status is SeedStatus.queued and seed.latest_run_id is None): + seed.status = SeedStatus.queued + seed.updated_at = now_ts() + seed.latest_run_id = None + seed.last_error = None + self.seed_repo.save(seed) + self.seed_repo.append_event( + seed.seed_id, + "p.waiting_for_baseline", + "Baseline run is still in progress; Plan will queue after baseline finishes.", + ) + return None + # Branch has no baseline and is not the baseline seed's branch: proceed with planning. + setup_error = self.git_service.setup_error() + if setup_error is not None: + raise RuntimeError(setup_error) + try: + self.git_service.ensure_branch(seed.baseline_branch, self.git_service.current_head()) + except GitCommandError: + pass + setup_error = self.git_service.setup_error_for_branches(seed.baseline_branch) + if setup_error is not None: + raise RuntimeError(setup_error) + return self._enqueue_plan_run(seed) + + def queue_dca( + self, + seed_id: str, + merge_resolution: bool = False, + metrics_recovery: bool = False, + source_run_id: str | None = None, + source_stdout_log_path: str | None = None, + source_stderr_log_path: str | None = None, + last_metrics: dict[str, Any] | None = None, + last_summary: dict[str, Any] | None = None, + commit_sha_before_p: str | None = None, + ) -> StageRun: + seed = self.require_seed(seed_id) + if not metrics_recovery and seed.status in {SeedStatus.draft, SeedStatus.queued, SeedStatus.planning}: + raise RuntimeError("Run Plan first. Do-Check-Action is available after code is generated into the seed branch.") + if not metrics_recovery: + setup_error = self.git_service.setup_error_for_branches(seed.baseline_branch) + if setup_error is not None: + raise RuntimeError(setup_error) + run = StageRun( + run_id=new_run_id("dca"), + seed_id=seed.seed_id, + stage=StageName.dca, + status=RunStatus.queued, + task_id=new_run_id("task-dca"), + created_at=now_ts(), + updated_at=now_ts(), + ) + if seed.seed_id != BASELINE_SEED_ID: + try: + # Ref to restore worktree to on negative signal (commit before P when from finish_p_run, else baseline). + run.summary["commit_sha_before_p"] = ( + commit_sha_before_p + if commit_sha_before_p is not None + else self.git_service.commit_sha(seed.baseline_branch) + ) + except GitCommandError: + pass + seed.status = SeedStatus.dca_queued + seed.updated_at = now_ts() + seed.latest_run_id = run.run_id + seed.last_error = None + self.seed_repo.save(seed) + self.run_repo.save(run) + self.seed_repo.append_event( + seed.seed_id, + "dca.queued", + "Queued DCA for merge conflict resolution." + if merge_resolution + else "Queued DCA for metrics recovery from saved logs." + if metrics_recovery + else "Queued DCA stage for the seed.", + ) + payload = { + "seed_id": seed.seed_id, + "run_id": run.run_id, + "prompt": seed.prompt, + "worktree_path": seed.worktree_path, + "merge_resolution": merge_resolution, + "metrics_recovery": metrics_recovery, + } + if merge_resolution: + payload["baseline_branch"] = seed.baseline_branch + if last_metrics is not None: + payload["last_metrics"] = last_metrics + if last_summary is not None: + payload["last_summary"] = last_summary + if metrics_recovery: + payload["source_run_id"] = source_run_id + payload["source_stdout_log_path"] = source_stdout_log_path + payload["source_stderr_log_path"] = source_stderr_log_path + payload["worktree_path"] = None + write_task("dca", payload, task_id=run.task_id) + return run + + def queue_sync_resolution(self, seed_id: str) -> StageRun: + """Queue a merge-resolution run to resolve 'merge baseline into seed' in the seed worktree (e.g. after sync failed before P).""" + seed = self.require_seed(seed_id) + if seed.seed_id == BASELINE_SEED_ID: + raise RuntimeError("Sync resolution is not used for the baseline seed.") + setup_error = self.git_service.setup_error_for_branches(seed.baseline_branch) + if setup_error is not None: + raise RuntimeError(setup_error) + run = StageRun( + run_id=new_run_id("dca"), + seed_id=seed.seed_id, + stage=StageName.dca, + status=RunStatus.queued, + task_id=new_run_id("task-dca"), + created_at=now_ts(), + updated_at=now_ts(), + ) + seed.status = SeedStatus.dca_queued + seed.updated_at = now_ts() + seed.latest_run_id = run.run_id + seed.last_error = None + self.seed_repo.save(seed) + self.run_repo.save(run) + self.seed_repo.append_event( + seed.seed_id, + "p.sync_resolution_queued", + "Worktree sync with baseline failed; queued merge-resolution to resolve and re-run Plan.", + ) + payload = { + "seed_id": seed.seed_id, + "run_id": run.run_id, + "prompt": seed.prompt, + "worktree_path": seed.worktree_path, + "baseline_branch": seed.baseline_branch, + "sync_resolution": True, + } + write_task("dca", payload, task_id=run.task_id) + return run + + def finish_sync_resolution(self, seed_id: str, run_id: str) -> None: + """Mark sync-resolution run completed and re-queue Plan for the seed.""" + seed = self.require_seed(seed_id) + run = self.require_run(run_id) + run.status = RunStatus.succeeded + run.updated_at = now_ts() + self.run_repo.save(run) + seed.status = SeedStatus.queued + seed.updated_at = now_ts() + self.seed_repo.save(seed) + self.seed_repo.append_event( + seed.seed_id, + "p.sync_resolution_done", + "Sync resolution completed; Plan re-queued.", + run_id=run_id, + ) + self._enqueue_plan_run( + seed, + event_kind="p.queued", + event_message="Re-queued Plan after sync resolution.", + ) + + def require_seed(self, seed_id: str) -> SeedRecord: + seed = self.seed_repo.get(seed_id) + if seed is None: + raise KeyError(f"Unknown seed_id={seed_id}") + return self._normalize_seed_runtime_state(seed) + + def require_run(self, run_id: str) -> StageRun: + run = self.run_repo.get(run_id) + if run is None: + raise KeyError(f"Unknown run_id={run_id}") + return run + + def is_seed_eligible_for_stage(self, seed_id: str | None, stage: str) -> bool: + """True if this seed is in a state that allows the given stage to run (used at claim time to avoid P/DCA races).""" + if not seed_id: + return False + seed = self.seed_repo.get(seed_id) + if seed is None: + return False + seed = self._normalize_seed_runtime_state(seed) + if stage == "p": + return seed.status not in (SeedStatus.adapting, SeedStatus.running, SeedStatus.dca_queued) + if stage == "dca": + return seed.status is not SeedStatus.planning + if stage == "direct": + return True + return False + + def mark_run_started(self, seed_id: str, run_id: str) -> tuple[SeedRecord, StageRun]: + seed = self.require_seed(seed_id) + run = self.require_run(run_id) + if run.status != RunStatus.queued: + raise DuplicateRunStartError( + f"Run {run_id} already started (status={run.status}); possible restored in-progress task." + ) + run.status = RunStatus.running + run.updated_at = now_ts() + if run.stage is StageName.p: + setup_error = self.git_service.setup_error() + if setup_error is not None: + raise RuntimeError(setup_error) + try: + self.git_service.ensure_branch(seed.baseline_branch, self.git_service.current_head()) + except GitCommandError: + pass + setup_error = self.git_service.setup_error_for_branches(seed.baseline_branch) + if setup_error is not None: + raise RuntimeError(setup_error) + seed = self.ensure_seed_worktree_ready(seed.seed_id) + # Sync seed worktree with baseline branch before P so Plan runs from latest baseline. + try: + self.git_service.sync_seed_worktree_with_baseline(seed) + except GitCommandError as sync_err: + run.status = RunStatus.failed + run.error = f"Worktree sync with baseline failed: {sync_err}" + self.run_repo.save(run) + self.queue_sync_resolution(seed.seed_id) + raise SyncResolutionQueued( + f"Worktree sync with baseline failed: {sync_err}. Queued merge-resolution." + ) from sync_err + # Record baseline val_bpb at sync time for positive/negative/neutral judgement in DCA. + branch_metrics = self.metrics_repo.get_for_branch(seed.baseline_branch) + former = branch_metrics.get("best_val_bpb") if branch_metrics else None + if run.summary is None: + run.summary = {} + run.summary["former_val_bpb"] = former + seed.former_val_bpb = float(former) if former is not None else None + if seed.worktree_path: + worktree_path = Path(seed.worktree_path) + if worktree_path.is_dir(): + try: + run.summary["commit_sha_before_p"] = self.git_service.head_sha_at( + worktree_path + ) + except GitCommandError: + pass + seed.status = SeedStatus.planning + event_kind = "p.started" + event_message = "Plan stage started in the candidate worktree." + else: + seed.status = SeedStatus.adapting + event_kind = "dca.started" + event_message = ( + "Baseline measurement started in the baseline worktree." + if seed.seed_id == BASELINE_SEED_ID + else "DCA stage started in the seed worktree." + ) + seed.updated_at = now_ts() + self.run_repo.save(run) + self.seed_repo.save(seed) + self.seed_repo.append_event(seed.seed_id, event_kind, event_message, run_id=run_id) + return seed, run + + def mark_direct_code_run_started(self, seed_id: str, run_id: str) -> tuple[SeedRecord, StageRun]: + seed = self.require_seed(seed_id) + run = self.require_run(run_id) + run.status = RunStatus.running + run.updated_at = now_ts() + seed.status = SeedStatus.adapting + seed.updated_at = now_ts() + self.run_repo.save(run) + self.seed_repo.save(seed) + self.seed_repo.append_event( + seed.seed_id, + "direct_code.started", + "Direct code agent started from the project root.", + run_id=run_id, + ) + return seed, run + + def mark_direct_code_run_failed( + self, + seed_id: str, + run_id: str, + error: str, + task_path: Path | None = None, + prompt_path: str | None = None, + log_path: str | None = None, + stderr_log_path: str | None = None, + ) -> None: + seed = self.require_seed(seed_id) + run = self.require_run(run_id) + run.status = RunStatus.failed + run.updated_at = now_ts() + run.error = error + if prompt_path is not None: + run.prompt_path = prompt_path + if log_path is not None: + run.log_path = log_path + if stderr_log_path is not None: + run.stderr_log_path = stderr_log_path + seed.status = SeedStatus.failed + seed.updated_at = now_ts() + seed.last_error = error + self.run_repo.save(run) + self.seed_repo.save(seed) + self.seed_repo.append_event(seed.seed_id, "direct_code.failed", error, run_id=run_id) + if task_path is not None and task_path.exists(): + move_to_error(task_path) + + def _ralph_try_restore_worktree(self, seed: SeedRecord, ref: str | None) -> None: + """Reset seed worktree to ref (e.g. commit before P) and log result. No-op if ref missing or baseline seed.""" + if not ref or not str(ref).strip() or seed.seed_id == BASELINE_SEED_ID: + return + try: + self.git_service.reset_seed_branch_to(seed, ref) + self.seed_repo.append_event( + seed.seed_id, + "ralph.worktree_restored", + "Restored seed worktree to commit before P for next Plan.", + commit_sha=ref, + ) + except GitCommandError as exc: + self.seed_repo.append_event( + seed.seed_id, + "ralph.worktree_restore_failed", + f"Could not restore seed worktree to commit before P: {exc}", + commit_sha=ref, + ) + + def mark_run_failed( + self, + seed_id: str, + run_id: str, + error: str, + task_path: Path | None = None, + prompt_path: str | None = None, + log_path: str | None = None, + stderr_log_path: str | None = None, + ) -> None: + seed = self.require_seed(seed_id) + run = self.require_run(run_id) + task_payload: dict[str, Any] = {} + if task_path is not None and task_path.exists(): + task_payload = read_task(task_path) + run.status = RunStatus.failed + run.updated_at = now_ts() + run.error = error + if prompt_path is not None: + run.prompt_path = prompt_path + if log_path is not None: + run.log_path = log_path + if stderr_log_path is not None: + run.stderr_log_path = stderr_log_path + seed.status = SeedStatus.failed + seed.updated_at = now_ts() + seed.last_error = error + self.run_repo.save(run) + self.seed_repo.save(seed) + self.seed_repo.append_event(seed.seed_id, f"{run.stage.value}.failed", error, run_id=run_id) + if ( + run.stage is StageName.dca + and seed.ralph_loop_enabled + and seed.seed_id != BASELINE_SEED_ID + and task_payload.get("merge_resolution") is not True + and task_payload.get("metrics_recovery") is not True + ): + self._ralph_try_restore_worktree(seed, run.summary.get("commit_sha_before_p")) + try: + self.queue_p(seed.seed_id) + self.seed_repo.append_event( + seed.seed_id, + "ralph.requeued", + "Ralph loop queued the next Plan run after failed DCA.", + ) + except (RuntimeError, GitCommandError) as exc: + self.seed_repo.append_event( + seed.seed_id, + "ralph.requeue_failed", + f"Ralph loop could not queue the next Plan run after failed DCA: {exc}", + ) + if task_path is not None and task_path.exists(): + move_to_error(task_path) + + def finish_direct_code_run( + self, + seed_id: str, + run_id: str, + stdout: str, + stderr: str | None = None, + log_path: str | None = None, + stderr_log_path: str | None = None, + prompt_path: str | None = None, + ) -> StageRun: + seed = self.require_seed(seed_id) + run = self.require_run(run_id) + run.status = RunStatus.succeeded + run.updated_at = now_ts() + run.log_path = log_path + run.stderr_log_path = stderr_log_path + run.prompt_path = prompt_path + run.summary = { + "mode": "direct_code_agent", + "cwd": str(COMPONENT_SYSTEM_ROOT.parent), + "stdout_bytes": len(stdout.encode("utf-8", errors="replace")), + "stderr_bytes": len((stderr or "").encode("utf-8", errors="replace")), + } + run.signal = "direct_code_completed" + seed.status = SeedStatus.passed + seed.updated_at = now_ts() + seed.latest_signal = run.signal + seed.last_error = None + self.run_repo.save(run) + self.seed_repo.save(seed) + self.seed_repo.append_event( + seed.seed_id, + "direct_code.completed", + "Direct code agent completed from the project root.", + run_id=run_id, + ) + return run + + def finish_p_run( + self, + seed_id: str, + run_id: str, + stdout: str, + log_path: str | None = None, + stderr_log_path: str | None = None, + prompt_path: str | None = None, + ) -> StageRun: + seed = self.require_seed(seed_id) + run = self.require_run(run_id) + summary = self.extract_summary(stdout, StageName.p) or {} + seed.plan = PlanIdea( + title=summary.get("idea", "Generated plan"), + target_component=summary.get("target_component", "model"), + description=summary.get("description", ""), + source_refs=summary.get("source_refs", []), + commit_sha=summary.get("commit_sha"), + ) + # Single branch per seed (SSOT): worktree is already on seed_id branch. + commit_sha = self.git_service.commit_sha(seed.seed_id) + run.status = RunStatus.succeeded + run.updated_at = now_ts() + run.log_path = log_path + run.stderr_log_path = stderr_log_path + run.prompt_path = prompt_path + # Preserve run.summary fields set earlier (e.g. commit_sha_before_p) when merging P output. + run.summary = run.summary | summary | {"commit_sha": commit_sha} + seed.status = SeedStatus.generated + seed.updated_at = now_ts() + self.run_repo.save(run) + self.seed_repo.save(seed) + self.seed_repo.append_event( + seed.seed_id, + "p.completed", + "Plan completed on seed branch.", + commit_sha=commit_sha, + ) + self.queue_dca( + seed.seed_id, + commit_sha_before_p=run.summary.get("commit_sha_before_p"), + ) + return run + + @staticmethod + def combine_output(stdout: str, stderr: str | None = None) -> str: + if stdout and stderr: + return f"{stdout}\n{stderr}" + return stdout or stderr or "" + + def finish_dca_run( + self, + seed_id: str, + run_id: str, + stdout: str, + stderr: str | None = None, + log_path: str | None = None, + stderr_log_path: str | None = None, + prompt_path: str | None = None, + metrics_recovery: bool = False, + merge_resolution: bool = False, + ) -> StageRun: + seed = self.require_seed(seed_id) + run = self.require_run(run_id) + branch_metrics = self.metrics_repo.get_for_branch(seed.baseline_branch) + best_val_bpb = float(branch_metrics["best_val_bpb"]) if branch_metrics and branch_metrics.get("best_val_bpb") is not None else None + # Use baseline at sync-before-P time (former_val_bpb) when available; else branch best for baseline seed. + baseline_for_signal = seed.former_val_bpb if (seed.former_val_bpb is not None and seed.seed_id != BASELINE_SEED_ID) else best_val_bpb + output_text = self.combine_output(stdout, stderr) + summary = self.extract_summary(output_text, StageName.dca) or {} + metrics = self.extract_dca_metrics(output_text, summary) + signal = self.evaluate_signal(metrics, baseline_for_signal, PROMOTION_THRESHOLD) + commit_sha = summary.get("commit_sha") + if not (isinstance(commit_sha, str) and commit_sha.strip()): + try: + commit_sha = self.git_service.commit_sha(seed.seed_id) + except GitCommandError: + commit_sha = "" + run.status = RunStatus.succeeded + run.updated_at = now_ts() + run.log_path = log_path + run.stderr_log_path = stderr_log_path + run.prompt_path = prompt_path + # Preserve runner-set keys (e.g. commit_sha_before_p, former_val_bpb) for restore and comparison. + preserved = {k: run.summary[k] for k in ("commit_sha_before_p", "former_val_bpb") if run.summary and k in run.summary} + if seed.former_val_bpb is not None and "former_val_bpb" not in preserved: + preserved["former_val_bpb"] = seed.former_val_bpb + run.summary = summary | {"commit_sha": commit_sha} | preserved + run.metrics = metrics + run.signal = signal + seed.updated_at = now_ts() + if signal == "error" and not metrics_recovery: + run.summary = run.summary | {"metrics_recovery_queued": True} + self.run_repo.save(run) + self.seed_repo.save(seed) + self.seed_repo.append_event( + seed.seed_id, + "dca.metrics_recovery_queued", + "DCA completed without recoverable metrics in the structured report; queued a follow-up DCA to inspect saved logs.", + run_id=run_id, + ) + self.queue_dca( + seed.seed_id, + metrics_recovery=True, + source_run_id=run_id, + source_stdout_log_path=log_path, + source_stderr_log_path=stderr_log_path, + ) + if ( + seed.ralph_loop_enabled + and seed.seed_id != BASELINE_SEED_ID + ): + self._ralph_try_restore_worktree(seed, run.summary.get("commit_sha_before_p")) + return run + seed.latest_metrics = metrics + seed.latest_signal = signal + terminal_status = self._status_from_dca_signal(signal) + merge_commit_sha = None # set when seed branch is successfully merged into baseline + if seed.seed_id == BASELINE_SEED_ID and best_val_bpb is None: + if "val_bpb" not in metrics: + seed.status = SeedStatus.failed + event_message = ( + "Baseline metrics recovery could not recover metrics; marked as failed." + if metrics_recovery + else "Baseline measurement completed without metrics; marked as failed." + ) + self.run_repo.save(run) + self.seed_repo.save(seed) + self.seed_repo.append_event( + seed.seed_id, + "dca.completed", + event_message, + signal=signal, + metrics=metrics, + ) + return run + target_branch = self._first_user_seed_baseline_branch() or seed.baseline_branch + _idea = summary.get("idea") or summary.get("notes") + if isinstance(_idea, str) and _idea.strip(): + baseline_promoted_idea = _idea[:80] + elif _idea: + baseline_promoted_idea = str(_idea)[:80] + else: + baseline_promoted_idea = "Initial baseline adaptation" + # Only positive_signal is merged into the per-seed baseline branch; record baseline value otherwise. + if signal != "positive_signal": + self.metrics_repo.append_baseline_run(target_branch, metrics["val_bpb"]) + seed.status = terminal_status + self.run_repo.save(run) + self.seed_repo.save(seed) + self.seed_repo.append_event( + seed.seed_id, + "dca.completed", + "Baseline measurement completed (no promotion); not merged into baseline branch.", + signal=signal, + metrics=metrics, + ) + return run + # Merge-resolution DCA: agent already ran merge (or "Already up to date"). Treat as pass; do not run promote_seed_branch again. + if merge_resolution: + effective_sha = self._commit_sha_for_branch(target_branch) + self.metrics_repo.append_promotion_for_branch( + target_branch, + { + "val_bpb": metrics["val_bpb"], + "promoted_branch": seed.seed_id, + "promoted_idea": baseline_promoted_idea, + "promoted_at": summary.get("completed_at"), + "commit_sha": effective_sha, + }, + ) + seed.status = SeedStatus.passed + self.run_repo.save(run) + self.seed_repo.save(seed) + self.seed_repo.append_event( + seed.seed_id, + "dca.completed", + f"Merge resolution DCA completed; __baseline__ merged or already up to date with {target_branch}.", + signal=signal, + metrics=metrics, + commit_sha=effective_sha, + ) + self._release_seeds_waiting_for_baseline(target_branch) + return run + try: + merge_commit_sha = self.git_service.promote_seed_branch(seed, target_branch=target_branch) + effective_sha = ( + merge_commit_sha + if (isinstance(merge_commit_sha, str) and merge_commit_sha.strip()) + else self._commit_sha_for_branch(target_branch) + ) + self.metrics_repo.append_promotion_for_branch( + target_branch, + { + "val_bpb": metrics["val_bpb"], + "promoted_branch": seed.seed_id, + "promoted_idea": baseline_promoted_idea, + "promoted_at": summary.get("completed_at"), + "commit_sha": effective_sha, + }, + ) + seed.status = SeedStatus.passed + event_message = f"Baseline measurement completed and __baseline__ was merged into {target_branch}; waiting seeds can now start Plan." + self.run_repo.save(run) + self.seed_repo.save(seed) + self.seed_repo.append_event( + seed.seed_id, + "dca.completed", + event_message, + signal=signal, + metrics=metrics, + commit_sha=merge_commit_sha, + ) + self._release_seeds_waiting_for_baseline(target_branch) + return run + except GitCommandError as merge_err: + tried_sha = commit_sha or "" + try: + tried_sha = self.git_service.commit_sha(seed.seed_id) + except GitCommandError: + pass + self.seed_repo.append_event( + seed.seed_id, + "dca.merge_failed", + f"Merge into baseline failed: {merge_err}. Queued a new DCA run to resolve conflicts.", + commit_sha=tried_sha or None, + target_branch=target_branch, + ) + if not merge_resolution: + self.queue_dca( + seed.seed_id, + merge_resolution=True, + last_metrics=metrics, + last_summary=summary, + ) + seed.status = SeedStatus.dca_queued + seed.updated_at = now_ts() + self.seed_repo.save(seed) + self.run_repo.save(run) + self.seed_repo.append_event( + seed.seed_id, + "dca.completed", + "Baseline measurement completed but merge failed; conflict-resolution DCA queued.", + signal=signal, + metrics=metrics, + ) + return run + effective_sha = self._commit_sha_for_branch(target_branch) + self.metrics_repo.append_promotion_for_branch( + target_branch, + { + "val_bpb": metrics["val_bpb"], + "promoted_branch": seed.seed_id, + "promoted_idea": baseline_promoted_idea, + "promoted_at": summary.get("completed_at"), + "commit_sha": effective_sha, + }, + ) + seed.status = SeedStatus.passed + self.seed_repo.save(seed) + self.run_repo.save(run) + self.seed_repo.append_event( + seed.seed_id, + "dca.completed", + "Baseline measurement completed; merge into baseline branch failed again after resolution run (loop avoided). Baseline metrics recorded; manual merge may be needed.", + signal=signal, + metrics=metrics, + ) + self._release_seeds_waiting_for_baseline(target_branch) + return run + if terminal_status is SeedStatus.promoted: + # Merge seed into baseline first on positive signal; then update metrics/state. + # Merge-resolution DCA: agent already ran merge (or "Already up to date"). Treat as pass; do not run promote_seed_branch again. + if merge_resolution: + effective_sha = self._commit_sha_for_branch(seed.baseline_branch) + self.metrics_repo.append_promotion_for_branch( + seed.baseline_branch, + { + "val_bpb": metrics["val_bpb"], + "promoted_branch": seed.seed_id, + "promoted_idea": seed.plan.title if seed.plan else seed.prompt[:80], + "promoted_at": summary.get("completed_at"), + "commit_sha": effective_sha, + }, + ) + seed.status = terminal_status + event_message = "Merge resolution DCA completed; seed merged or already up to date with baseline." + else: + try: + merge_commit_sha = self.git_service.promote_seed_branch(seed) + effective_sha = ( + merge_commit_sha + if (isinstance(merge_commit_sha, str) and merge_commit_sha.strip()) + else self._commit_sha_for_branch(seed.baseline_branch) + ) + self.metrics_repo.append_promotion_for_branch( + seed.baseline_branch, + { + "val_bpb": metrics["val_bpb"], + "promoted_branch": seed.seed_id, + "promoted_idea": seed.plan.title if seed.plan else seed.prompt[:80], + "promoted_at": summary.get("completed_at"), + "commit_sha": effective_sha, + }, + ) + seed.status = terminal_status + event_message = "DCA succeeded and seed branch was promoted into baseline." + except GitCommandError as merge_err: + tried_sha = commit_sha or "" + try: + tried_sha = self.git_service.commit_sha(seed.seed_id) + except GitCommandError: + pass + self.seed_repo.append_event( + seed.seed_id, + "dca.merge_failed", + ( + f"Merge into baseline failed: {merge_err}. Queued a new DCA run to resolve conflicts." + if not merge_resolution + else f"Merge into baseline failed again after a conflict-resolution DCA: {merge_err}. " + "Ralph can proceed with the next Plan run." + ), + commit_sha=tried_sha or None, + target_branch=seed.baseline_branch, + ) + if not merge_resolution: + self.queue_dca( + seed.seed_id, + merge_resolution=True, + last_metrics=metrics, + last_summary=summary, + ) + seed.status = SeedStatus.dca_queued + seed.updated_at = now_ts() + self.seed_repo.save(seed) + self.run_repo.save(run) + self.seed_repo.append_event( + seed.seed_id, + "dca.completed", + "DCA run completed but merge failed; conflict-resolution DCA queued.", + signal=signal, + metrics=metrics, + ) + return run + # Resolution run also failed to merge; avoid infinite resolution loop and continue Ralph. + seed.status = SeedStatus.generated + seed.updated_at = now_ts() + self.seed_repo.save(seed) + self.run_repo.save(run) + self.seed_repo.append_event( + seed.seed_id, + "dca.completed", + "Conflict-resolution DCA completed but merge still failed; proceeding to next Plan run.", + signal=signal, + metrics=metrics, + ) + if seed.ralph_loop_enabled: + try: + self.queue_p(seed.seed_id) + self.seed_repo.append_event( + seed.seed_id, + "ralph.requeued", + "Ralph loop queued the next Plan run after unresolved merge conflict.", + ) + except (RuntimeError, GitCommandError) as exc: + self.seed_repo.append_event( + seed.seed_id, + "ralph.requeue_failed", + f"Ralph loop could not queue the next Plan run after unresolved merge conflict: {exc}", + ) + return run + elif terminal_status is SeedStatus.failed: + seed.status = terminal_status + event_message = ( + "DCA metrics recovery could not recover metrics; marked as failed." + if metrics_recovery + else "DCA completed but metrics were missing; marked as failed." + ) + else: + seed.status = terminal_status + event_message = "DCA completed without promotion." + self.run_repo.save(run) + self.seed_repo.save(seed) + event_commit_sha = merge_commit_sha if merge_commit_sha else run.summary.get("commit_sha") + self.seed_repo.append_event( + seed.seed_id, + "dca.completed", + event_message, + signal=signal, + metrics=metrics, + **({"commit_sha": event_commit_sha} if event_commit_sha else {}), + ) + if ( + seed.ralph_loop_enabled + and signal in ("negative_signal", "neutral", "error") + and not merge_resolution + and not metrics_recovery + and seed.seed_id != BASELINE_SEED_ID + ): + self._ralph_try_restore_worktree(seed, run.summary.get("commit_sha_before_p")) + if seed.ralph_loop_enabled: + try: + self.queue_p(seed.seed_id) + self.seed_repo.append_event( + seed.seed_id, + "ralph.requeued", + "Ralph loop queued the next Plan run.", + ) + except (RuntimeError, GitCommandError) as exc: + self.seed_repo.append_event( + seed.seed_id, + "ralph.requeue_failed", + f"Ralph loop could not queue the next Plan run: {exc}", + ) + return run + + def build_dashboard(self, selected_seed_id: str | None = None) -> DashboardViewModel: + seeds = self.seed_repo.list() + selected_seed = self.seed_repo.get(selected_seed_id) if selected_seed_id else None + baseline_metrics_by_branch = self.metrics_repo.get_all() + available_branches: list[str] = [] + setup_error = self.git_service.setup_error() + if setup_error is None: + try: + all_branches = self.git_service.list_branches() + if not all_branches: + setup_error = "No local branches found yet. Create an initial commit/branch, then reload." + else: + available_branches = [ + b for b in all_branches + if not self.git_service.is_seed_specific_branch(b) + ] + # Use only branches that exist in the repo; do not add DEFAULT_BASELINE_BRANCH + # if it does not exist, so the dropdown never shows a non-existent branch. + except GitCommandError as exc: + setup_error = str(exc) + # Default to first existing branch so the selected value is always valid. + default_baseline_branch = (available_branches[0] if available_branches else DEFAULT_BASELINE_BRANCH) or "master" + status_column_map = { + SeedStatus.draft: "seedInbox", + SeedStatus.queued: "seedInbox", + SeedStatus.planning: "generated", + SeedStatus.generated: "generated", + SeedStatus.dca_queued: "generated", + SeedStatus.adapting: "activeDca", + SeedStatus.running: "activeDca", + SeedStatus.passed: "completed", + SeedStatus.failed: "completed", + SeedStatus.promoted: "completed", + } + seeds_by_column: dict[str, list[SeedRecord]] = { + "seedInbox": [], + "generated": [], + "activeDca": [], + "completed": [], + } + for seed in seeds: + self._reconcile_seed_status_signal(seed) + column_id = status_column_map.get(seed.status, "seedInbox") + seeds_by_column[column_id].append(seed) + columns = [ + DashboardColumn( + id="seedInbox", + title="Seed", + description="New prompts and queued planning work.", + seeds=seeds_by_column["seedInbox"], + ), + DashboardColumn( + id="generated", + title="Plan", + description="Planning and generated code ready for Do-Check-Action.", + seeds=seeds_by_column["generated"], + ), + DashboardColumn( + id="activeDca", + title="Do-Check-Action", + description="Adapting, fixing, and running the seed run.", + seeds=seeds_by_column["activeDca"], + ), + DashboardColumn( + id="completed", + title="Completed", + description="Finished runs; promoted seeds merged into baseline.", + seeds=seeds_by_column["completed"], + ), + ] + return DashboardViewModel( + setup_error=setup_error, + baseline_metrics_by_branch=baseline_metrics_by_branch, + default_baseline_branch=default_baseline_branch, + available_branches=available_branches, + seed_count=len(seeds), + columns=columns, + selected_seed=selected_seed, + daemon_status=get_daemon_status(), + ) + + def seed_detail(self, seed_id: str) -> dict[str, object]: + seed = self.require_seed(seed_id) + expected_worktree = ( + self._baseline_worktree_path() + if seed.seed_id == BASELINE_SEED_ID + else self._seed_worktree_path(seed.seed_id) + ) + needs_save = False + if expected_worktree is not None and not seed.worktree_path: + seed.worktree_path = expected_worktree + needs_save = True + if needs_save: + seed.updated_at = now_ts() + self.seed_repo.save(seed) + self._reconcile_seed_status_signal(seed) + raw_events = self.seed_repo.events(seed_id) + return { + "seed": seed, + "can_edit_prompt": self.can_edit_seed_prompt(seed), + "runs": self.run_repo.list(seed_id), + "events": _timeline_display_events(raw_events), + "baseline_metrics_for_branch": self.metrics_repo.get_for_branch(seed.baseline_branch), + "setup_error": self.git_service.setup_error_for_branches(seed.baseline_branch), + } + + def seed_detail_versions(self, seed_id: str) -> dict[str, str]: + """Return version fingerprints for runs and timeline so the client can skip refresh when unchanged.""" + self.require_seed(seed_id) + runs = self.run_repo.list(seed_id) + events = self.seed_repo.events(seed_id) + runs_version = ( + ",".join(f"{r.run_id}:{r.status.value}:{r.updated_at}" for r in runs) + if runs + else "0" + ) + timeline_version = ( + ",".join(str(e.get("created_at", "")) for e in events[-20:]) + if events + else "0" + ) + return { + "runs_version": runs_version, + "timeline_version": timeline_version, + } + + def extract_summary(self, output_text: str, stage: StageName) -> dict[str, object] | None: + start_marker, end_marker = SUMMARY_MARKERS[stage.value] + pattern = rf"{start_marker}\s*(\{{.*?\}})\s*{end_marker}" + match = re.search(pattern, output_text, flags=re.DOTALL) + if not match: + return None + try: + return json.loads(match.group(1)) + except json.JSONDecodeError: + return {"raw_summary": match.group(1)} + + def extract_metrics(self, output_text: str) -> dict[str, float | int]: + patterns = { + "val_bpb": r"^val_bpb:\s+([0-9.]+)", + "training_seconds": r"^training_seconds:\s+([0-9.]+)", + "total_seconds": r"^total_seconds:\s+([0-9.]+)", + "startup_seconds": r"^startup_seconds:\s+([0-9.]+)", + "peak_vram_mb": r"^peak_vram_mb:\s+([0-9.]+)", + "mfu_percent": r"^mfu_percent:\s+([0-9.]+)", + "total_tokens_M": r"^total_tokens_M:\s+([0-9.]+)", + "num_steps": r"^num_steps:\s+([0-9]+)", + "num_params_M": r"^num_params_M:\s+([0-9.]+)", + "depth": r"^depth:\s+([0-9]+)", + } + metrics: dict[str, float | int] = {} + for key, pattern in patterns.items(): + match = re.search(pattern, output_text, flags=re.MULTILINE) + if not match: + continue + metrics[key] = int(match.group(1)) if key in {"num_steps", "depth"} else float(match.group(1)) + return metrics + + def extract_dca_metrics( + self, output_text: str, summary: dict[str, object] | None = None + ) -> dict[str, float | int]: + if summary: + summary_metrics = summary.get("metrics") + if isinstance(summary_metrics, dict): + parsed: dict[str, float | int] = {} + int_keys = {"num_steps", "depth"} + float_keys = { + "val_bpb", + "training_seconds", + "total_seconds", + "startup_seconds", + "peak_vram_mb", + "mfu_percent", + "total_tokens_M", + "num_params_M", + } + for key in int_keys | float_keys: + value = summary_metrics.get(key) + if value is None: + continue + try: + parsed[key] = int(value) if key in int_keys else float(value) + except (TypeError, ValueError): + continue + if parsed: + return parsed + return self.extract_metrics(output_text) + + @staticmethod + def evaluate_signal( + metrics: dict[str, float | int], + baseline_val_bpb: float | None, + promotion_threshold: float = PROMOTION_THRESHOLD, + ) -> str: + val_bpb = metrics.get("val_bpb") + if val_bpb is None: + return "error" + if baseline_val_bpb is None: + return "positive_signal" + delta = float(baseline_val_bpb) - float(val_bpb) + if delta >= promotion_threshold: + return "positive_signal" + if delta <= -promotion_threshold: + return "negative_signal" + return "neutral" + + +def default_workflow_service() -> WorkflowService: + return WorkflowService() diff --git a/component_system/tailwind.config.js b/component_system/tailwind.config.js new file mode 100644 index 00000000..ea1a7a37 --- /dev/null +++ b/component_system/tailwind.config.js @@ -0,0 +1,11 @@ +/** @type {import('tailwindcss').Config} */ +module.exports = { + content: [ + "./web/templates/**/*.html", + "./web/static/**/*.js" + ], + theme: { + extend: {} + }, + plugins: [] +}; diff --git a/component_system/task.py b/component_system/task.py new file mode 100644 index 00000000..ea6f40dc --- /dev/null +++ b/component_system/task.py @@ -0,0 +1,310 @@ +"""Shared queue and JSON state helpers for the component-system web app.""" +from __future__ import annotations + +import json +import os +import shutil +import time +import uuid +from pathlib import Path +from typing import Any, Callable, Literal + +COMPONENT_SYSTEM_ROOT = Path(__file__).resolve().parent +HISTORY_ROOT = COMPONENT_SYSTEM_ROOT / "history" +QUEUE_ROOT = HISTORY_ROOT / "queue" +STATE_ROOT = HISTORY_ROOT / "state" +SEEDS_ROOT = STATE_ROOT / "seeds" +RUNS_ROOT = STATE_ROOT / "runs" +EVENTS_ROOT = STATE_ROOT / "events" +BASELINE_BRANCHES_PATH = COMPONENT_SYSTEM_ROOT / "baseline_branches.json" +BASELINE_METRICS_PATH = COMPONENT_SYSTEM_ROOT / "baseline_metrics.json" +WORKTREE_ROOT = HISTORY_ROOT / "worktrees" +LOG_ROOT = HISTORY_ROOT / "logs" + +STAGE_DIRS = { + "p": QUEUE_ROOT / "p", + "dca": QUEUE_ROOT / "dca", + "direct": QUEUE_ROOT / "direct", +} +IN_PROGRESS_DIR = QUEUE_ROOT / "in_progress" +DONE_DIR = QUEUE_ROOT / "done" +ERROR_DIR = QUEUE_ROOT / "error" +DAEMON_HEARTBEAT_PATH = STATE_ROOT / "daemon_heartbeat.json" +DAEMON_HEARTBEAT_STALE_SECONDS = 5 + +def _read_json(path: Path, default: Any) -> Any: + if not path.exists(): + return default + return json.loads(path.read_text(encoding="utf-8")) + + +def _write_json(path: Path, payload: Any) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, ensure_ascii=False), encoding="utf-8") + return path + + +def now_ts() -> float: + return time.time() + + +def now_iso() -> str: + return time.strftime("%Y-%m-%d %H:%M:%S") + + +def daemon_heartbeat() -> None: + """Write the daemon heartbeat file (call from the daemon process).""" + ensure_queue_layout() + _write_json( + DAEMON_HEARTBEAT_PATH, + {"timestamp": now_ts(), "pid": os.getpid()}, + ) + + +def get_daemon_status() -> str: + """Return 'running' if the daemon heartbeat is recent, else 'stopped'.""" + if not DAEMON_HEARTBEAT_PATH.exists(): + return "stopped" + try: + data = _read_json(DAEMON_HEARTBEAT_PATH, {}) + ts = data.get("timestamp") + if ts is None: + return "stopped" + if (now_ts() - float(ts)) <= DAEMON_HEARTBEAT_STALE_SECONDS: + return "running" + except Exception: + pass + return "stopped" + + +def ensure_queue_layout() -> None: + HISTORY_ROOT.mkdir(parents=True, exist_ok=True) + for d in STAGE_DIRS.values(): + d.mkdir(parents=True, exist_ok=True) + IN_PROGRESS_DIR.mkdir(parents=True, exist_ok=True) + DONE_DIR.mkdir(parents=True, exist_ok=True) + ERROR_DIR.mkdir(parents=True, exist_ok=True) + SEEDS_ROOT.mkdir(parents=True, exist_ok=True) + RUNS_ROOT.mkdir(parents=True, exist_ok=True) + EVENTS_ROOT.mkdir(parents=True, exist_ok=True) + WORKTREE_ROOT.mkdir(parents=True, exist_ok=True) + LOG_ROOT.mkdir(parents=True, exist_ok=True) + # Auto-create baseline JSON files if missing (like results.tsv for recording run data) + if not BASELINE_METRICS_PATH.exists(): + _write_json(BASELINE_METRICS_PATH, {}) + if not BASELINE_BRANCHES_PATH.exists(): + _write_json(BASELINE_BRANCHES_PATH, {}) + + +def new_task_id(prefix: str | None = None) -> str: + ts = time.strftime("%Y%m%d-%H%M%S") + short = uuid.uuid4().hex[:8] + task_id = f"{ts}-{short}" + return f"{prefix}-{task_id}" if prefix else task_id + + +def new_seed_id(prefix: str = "seed") -> str: + return f"{prefix}-{uuid.uuid4().hex[:6]}" + + +def new_run_id(stage: str) -> str: + return new_task_id(stage) + + +def write_task(stage: str, payload: dict[str, Any], task_id: str | None = None) -> Path: + ensure_queue_layout() + if stage not in STAGE_DIRS: + raise KeyError(f"Unknown stage {stage!r}") + tid = task_id or new_task_id(stage) + path = STAGE_DIRS[stage] / f"{tid}.json" + payload_with_meta = {"task_id": tid, "stage": stage, "created_at": now_ts(), **payload} + return _write_json(path, payload_with_meta) + + +def read_task(path: Path) -> dict[str, Any]: + return _read_json(path, {}) + + +def move_to_done(path: Path) -> Path: + ensure_queue_layout() + dest = DONE_DIR / path.name + if not path.exists(): + raise FileNotFoundError( + f"Task file already moved: {path}; possible duplicate daemon or double completion." + ) + if dest.exists(): + dest.unlink() + path.rename(dest) + return dest + + +def move_to_error(path: Path) -> Path: + ensure_queue_layout() + dest = ERROR_DIR / path.name + if dest.exists(): + dest.unlink() + path.rename(dest) + return dest + + +def list_pending(stage: str) -> list[Path]: + ensure_queue_layout() + if stage not in STAGE_DIRS: + raise KeyError(f"Unknown stage {stage!r}") + return sorted(STAGE_DIRS[stage].glob("*.json")) + + +def _is_aux_dca_task(payload: dict[str, Any]) -> bool: + return payload.get("metrics_recovery") is True or payload.get("merge_resolution") is True + + +def claim_pending( + stage: str, + lane: Literal["any", "gpu", "aux"] = "any", + eligible_fn: Callable[[dict[str, Any]], bool] | None = None, +) -> Path | None: + """Atomically claim the oldest pending task for a stage/lane. If eligible_fn is set, only claim tasks for which it returns True (avoids P/DCA races).""" + ensure_queue_layout() + if stage not in STAGE_DIRS: + raise KeyError(f"Unknown stage {stage!r}") + if lane not in {"any", "gpu", "aux"}: + raise KeyError(f"Unknown lane {lane!r}") + for path in sorted(STAGE_DIRS[stage].glob("*.json")): + payload = _read_json(path, {}) + if eligible_fn is not None and not eligible_fn(payload): + continue + if stage == "dca" and lane != "any": + is_aux = _is_aux_dca_task(payload) + if lane == "aux" and not is_aux: + continue + if lane == "gpu" and is_aux: + continue + claimed_path = IN_PROGRESS_DIR / path.name + try: + path.rename(claimed_path) + return claimed_path + except FileNotFoundError: + continue + except OSError: + # Another worker likely claimed the task first. + continue + return None + + +def restore_in_progress_tasks() -> dict[str, int]: + """Move stranded in-progress tasks back to their stage queue.""" + ensure_queue_layout() + restored = {stage: 0 for stage in STAGE_DIRS} + for path in sorted(IN_PROGRESS_DIR.glob("*.json")): + payload = _read_json(path, {}) + stage = payload.get("stage") + if stage not in STAGE_DIRS: + continue + dest = STAGE_DIRS[stage] / path.name + if dest.exists(): + dest.unlink() + path.rename(dest) + restored[stage] += 1 + return restored + + +def seed_path(seed_id: str) -> Path: + return SEEDS_ROOT / f"{seed_id}.json" + + +def run_path(run_id: str) -> Path: + return RUNS_ROOT / f"{run_id}.json" + + +def event_path(seed_id: str) -> Path: + return EVENTS_ROOT / f"{seed_id}.json" + + +def save_seed(seed: dict[str, Any]) -> Path: + seed_id = seed["seed_id"] + return _write_json(seed_path(seed_id), seed) + + +def load_seed(seed_id: str) -> dict[str, Any]: + return _read_json(seed_path(seed_id), {}) + + +def list_seeds() -> list[dict[str, Any]]: + ensure_queue_layout() + seeds = [_read_json(path, {}) for path in SEEDS_ROOT.glob("*.json")] + return sorted(seeds, key=lambda item: item.get("updated_at", item.get("created_at", 0)), reverse=True) + + +def save_run(run: dict[str, Any]) -> Path: + return _write_json(run_path(run["run_id"]), run) + + +def load_run(run_id: str) -> dict[str, Any]: + return _read_json(run_path(run_id), {}) + + +def list_runs(seed_id: str | None = None) -> list[dict[str, Any]]: + ensure_queue_layout() + runs = [_read_json(path, {}) for path in RUNS_ROOT.glob("*.json")] + if seed_id is not None: + runs = [run for run in runs if run.get("seed_id") == seed_id] + return sorted(runs, key=lambda item: item.get("updated_at", item.get("created_at", 0)), reverse=True) + + +def append_event(seed_id: str, event: dict[str, Any]) -> list[dict[str, Any]]: + ensure_queue_layout() + payload = _read_json(event_path(seed_id), []) + payload.append({"created_at": now_ts(), "created_at_human": now_iso(), **event}) + _write_json(event_path(seed_id), payload) + return payload + + +def load_events(seed_id: str) -> list[dict[str, Any]]: + return _read_json(event_path(seed_id), []) + + +def delete_seed(seed_id: str) -> None: + for path in (seed_path(seed_id), event_path(seed_id)): + if path.exists(): + path.unlink() + for run in list_runs(seed_id): + path = run_path(run["run_id"]) + if path.exists(): + path.unlink() + + +def load_baseline_branch_map() -> dict[str, str]: + """Load seed_id -> baseline_branch mapping (for agent lookup and workflow).""" + ensure_queue_layout() + return _read_json(BASELINE_BRANCHES_PATH, {}) + + +def save_baseline_branch_map(mapping: dict[str, str]) -> None: + """Persist seed_id -> baseline_branch mapping.""" + ensure_queue_layout() + _write_json(BASELINE_BRANCHES_PATH, mapping) + + +def load_baseline_metrics() -> dict[str, list[dict[str, Any]]]: + """Load baseline_branch -> list of promotion/measurement records. Each record: val_bpb, promoted_branch?, promoted_idea?, promoted_at?, commit_sha?.""" + ensure_queue_layout() + raw = _read_json(BASELINE_METRICS_PATH, {}) + result: dict[str, list[dict[str, Any]]] = {} + for branch, value in raw.items(): + if isinstance(value, list): + result[branch] = value + else: + result[branch] = [] + return result + + +def save_baseline_metrics(metrics_by_branch: dict[str, list[dict[str, Any]]]) -> None: + """Persist per-branch baseline metrics (branch -> list of records).""" + ensure_queue_layout() + _write_json(BASELINE_METRICS_PATH, metrics_by_branch) + + +def reset_worktree(path: str | Path) -> None: + worktree = Path(path) + if worktree.exists(): + shutil.rmtree(worktree) diff --git a/component_system/training/mainline.py b/component_system/training/mainline.py new file mode 100644 index 00000000..e91771d8 --- /dev/null +++ b/component_system/training/mainline.py @@ -0,0 +1,82 @@ +"""Mainline assembler: reads static config, dynamically loads components, runs training.""" +from __future__ import annotations + +if __package__ in {None, ""}: + import sys + from pathlib import Path + + sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +import importlib +import os +from dataclasses import asdict +from typing import Any + +import torch + +from prepare import Tokenizer + +from component_system.config import get_training_binding + + +def _prepare_environment() -> None: + os.environ["PYTORCH_ALLOC_CONF"] = "expandable_segments:True" + os.environ["HF_HUB_DISABLE_PROGRESS_BARS"] = "1" + torch.manual_seed(42) + if torch.cuda.is_available(): + torch.cuda.manual_seed(42) + torch.set_float32_matmul_precision("high") + torch.cuda.reset_peak_memory_stats() + + +def _import_module(path: str) -> Any: + return importlib.import_module(path) + + +def run_mainline_training(binding_path: str | None = None) -> dict[str, Any]: + _prepare_environment() + binding = get_training_binding() + + tokenizer = Tokenizer.from_directory() + vocab_size = tokenizer.get_vocab_size() + + model_module = _import_module(binding["model_module"]) + optimizer_module = _import_module(binding["optimizer_module"]) + training_step_module = _import_module(binding["training_step_module"]) + + settings = training_step_module.default_training_settings() + config = model_module.build_model_config( + depth=settings.depth, + vocab_size=vocab_size, + aspect_ratio=settings.aspect_ratio, + head_dim=settings.head_dim, + window_pattern=settings.window_pattern, + ) + + print("Loaded training binding from config") + print(f"Model config: {asdict(config)}") + + model, param_counts, num_flops_per_token = model_module.create_model( + config, + compile_model=settings.compile_model, + ) + + print("Parameter counts:") + for key, value in param_counts.items(): + print(f" {key:24s}: {value:,}") + print(f"Estimated FLOPs per token: {num_flops_per_token:e}") + + optimizer = optimizer_module.create_optimizer(model, settings) + return training_step_module.run_training_session( + model=model, + optimizer=optimizer, + tokenizer=tokenizer, + settings=settings, + param_counts=param_counts, + num_flops_per_token=num_flops_per_token, + baseline_binding=binding, + ) + + +if __name__ == "__main__": + run_mainline_training() diff --git a/component_system/web/app.py b/component_system/web/app.py new file mode 100644 index 00000000..971b666e --- /dev/null +++ b/component_system/web/app.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +import time +from datetime import datetime, timezone +from pathlib import Path + +from fastapi import FastAPI +from fastapi.responses import RedirectResponse, Response +from fastapi.staticfiles import StaticFiles +from fastapi.templating import Jinja2Templates + +from component_system.services.workflow import default_workflow_service +from component_system.task import ensure_queue_layout +from component_system.web.routes import router + +WEB_ROOT = Path(__file__).resolve().parent +TEMPLATE_ROOT = WEB_ROOT / "templates" +STATIC_ROOT = WEB_ROOT / "static" + + +def _static_version() -> str: + """Cache-busting version from app.js mtime so browsers load fresh static assets after changes.""" + app_js = STATIC_ROOT / "app.js" + if app_js.exists(): + return str(int(app_js.stat().st_mtime)) + return str(int(time.time())) + + +def create_app() -> FastAPI: + ensure_queue_layout() + app = FastAPI(title="Component System", version="0.1.0") + app.state.workflow = default_workflow_service() + app.state.static_version = _static_version() + app.state.templates = Jinja2Templates(directory=str(TEMPLATE_ROOT)) + + def _format_ts(ts: float | None) -> str: + if ts is None: + return "" + try: + return datetime.fromtimestamp(ts, tz=timezone.utc).strftime("%Y-%m-%d %H:%M UTC") + except (TypeError, OSError): + return "" + + app.state.templates.env.filters["format_ts"] = _format_ts + app.mount("/static", StaticFiles(directory=str(STATIC_ROOT)), name="static") + app.include_router(router, prefix="/component-system") + + @app.get("/", include_in_schema=False) + def root() -> RedirectResponse: + return RedirectResponse(url="/component-system", status_code=307) + + @app.get("/favicon.ico", include_in_schema=False) + def favicon() -> Response: + return Response(status_code=204) + + @app.get("/.well-known/appspecific/com.chrome.devtools.json", include_in_schema=False) + def chrome_devtools_probe() -> Response: + # Chrome DevTools probes this endpoint; return 204 to avoid log spam. + return Response(status_code=204) + + return app + + +app = create_app() diff --git a/component_system/web/routes.py b/component_system/web/routes.py new file mode 100644 index 00000000..b1d7c01a --- /dev/null +++ b/component_system/web/routes.py @@ -0,0 +1,376 @@ +from __future__ import annotations + +from pathlib import Path + +from fastapi import APIRouter, Form, HTTPException, Query, Request +from fastapi.responses import HTMLResponse, RedirectResponse, Response + +from component_system.domain.models import SeedStatus +from component_system.services.workflow import GitCommandError, WorkflowService +from component_system.task import COMPONENT_SYSTEM_ROOT, get_daemon_status, LOG_ROOT + +router = APIRouter() + + +def _templates(request: Request): + return request.app.state.templates + + +def _workflow(request: Request) -> WorkflowService: + return request.app.state.workflow + + +def _is_htmx(request: Request) -> bool: + return request.headers.get("hx-request", "").lower() == "true" + + +def _render(request: Request, template_name: str, context: dict, status_code: int = 200) -> HTMLResponse: + templates = _templates(request) + return templates.TemplateResponse(request, template_name, {"request": request, **context}, status_code=status_code) + + +def _resolve_log_path(run_id: str, stream: str, run_log_path: str | None) -> Path | None: + # Primary source: persisted run metadata path. + if run_log_path: + candidate = Path(run_log_path) + if candidate.exists() and candidate.is_file(): + return candidate + + # Deterministic run-id naming (new format). + run_named = LOG_ROOT / f"{run_id}.{stream}.log" + if run_named.exists() and run_named.is_file(): + return run_named + + return None + + +def _resolve_prompt_path(run_id: str, run_prompt_path: str | None) -> Path | None: + if run_prompt_path: + candidate = Path(run_prompt_path) + if candidate.exists() and candidate.is_file(): + return candidate + prompt_named = LOG_ROOT / f"{run_id}.prompt.txt" + if prompt_named.exists() and prompt_named.is_file(): + return prompt_named + return None + + +@router.get("/", response_class=HTMLResponse) +def dashboard(request: Request, seed_id: str | None = None) -> HTMLResponse: + workflow = _workflow(request) + viewmodel = workflow.build_dashboard(selected_seed_id=seed_id) + context = { + "dashboard": viewmodel, + "selected_seed_id": seed_id, + "detail": workflow.seed_detail(seed_id) if seed_id else None, + } + return _render(request, "dashboard.html", context) + + +@router.get("/partials/dashboard", response_class=HTMLResponse) +def dashboard_board(request: Request, seed_id: str | None = None) -> HTMLResponse: + workflow = _workflow(request) + viewmodel = workflow.build_dashboard(selected_seed_id=seed_id) + return _render(request, "partials/dashboard_board.html", {"dashboard": viewmodel, "selected_seed_id": seed_id}) + + +@router.get("/partials/daemon-status", response_class=HTMLResponse) +def daemon_status_partial(request: Request) -> HTMLResponse: + return _render(request, "partials/daemon_status.html", {"daemon_status": get_daemon_status()}) + + +@router.get("/partials/seeds/{seed_id}", response_class=HTMLResponse) +def seed_detail_partial(request: Request, seed_id: str) -> HTMLResponse: + workflow = _workflow(request) + try: + detail = workflow.seed_detail(seed_id) + except KeyError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + dashboard = workflow.build_dashboard(selected_seed_id=seed_id) + context = { + **detail, + "dashboard": dashboard, + "selected_seed_id": seed_id, + "oob": True, + "daemon_status": get_daemon_status(), + } + return _render(request, "partials/seed_detail_response.html", context) + + +@router.get("/api/seeds/{seed_id}/versions") +def seed_versions(request: Request, seed_id: str) -> dict[str, str]: + workflow = _workflow(request) + try: + return workflow.seed_detail_versions(seed_id) + except KeyError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + + +@router.get("/partials/seeds/{seed_id}/runs", response_class=HTMLResponse) +def seed_runs_partial(request: Request, seed_id: str) -> HTMLResponse: + workflow = _workflow(request) + try: + detail = workflow.seed_detail(seed_id) + except KeyError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + return _render( + request, + "partials/seed_runs_inner.html", + {"seed": detail["seed"], "runs": detail["runs"]}, + ) + + +@router.get("/partials/seeds/{seed_id}/timeline", response_class=HTMLResponse) +def seed_timeline_partial(request: Request, seed_id: str) -> HTMLResponse: + workflow = _workflow(request) + try: + detail = workflow.seed_detail(seed_id) + except KeyError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + return _render( + request, + "partials/seed_timeline_inner.html", + {"seed": detail["seed"], "events": detail["events"]}, + ) + + +@router.get("/api/runs/{run_id}/prompt") +def run_prompt(request: Request, run_id: str) -> dict[str, object]: + workflow = _workflow(request) + run = workflow.run_repo.get(run_id) + run_prompt_path = run.prompt_path if run is not None else None + prompt_path = _resolve_prompt_path(run_id, run_prompt_path) + if prompt_path is None: + raise HTTPException(status_code=404, detail=f"Prompt for run '{run_id}' not found.") + content = prompt_path.read_text(encoding="utf-8", errors="replace") + return {"content": content} + + +@router.get("/api/runs/{run_id}/log") +def run_log_chunk( + request: Request, + run_id: str, + stream: str = Query("stdout"), + offset: int = Query(0, ge=0), + limit: int = Query(64 * 1024, ge=1024, le=512 * 1024), +) -> dict[str, object]: + workflow = _workflow(request) + run = workflow.run_repo.get(run_id) + + complete_status = bool(run is not None and run.status.value in {"succeeded", "failed"}) + if stream not in {"stdout", "stderr"}: + raise HTTPException(status_code=400, detail="stream must be one of: stdout, stderr") + + run_log_path = None + if run is not None: + run_log_path = run.log_path if stream == "stdout" else run.stderr_log_path + if not run_log_path and stream == "stderr" and run.log_path and run.log_path.endswith(".stdout.log"): + run_log_path = run.log_path.replace(".stdout.log", ".stderr.log") + + log_path = _resolve_log_path(run_id, stream, run_log_path) + if log_path is None and run is not None and not complete_status: + # During queued/running phases metadata may not yet include paths and files may appear slightly later. + return { + "chunk": "", + "next_offset": offset, + "size": 0, + "complete": False, + } + + if log_path is None: + raise HTTPException(status_code=404, detail=f"Log for run '{run_id}' ({stream}) not found.") + + if not log_path.exists() or not log_path.is_file(): + return { + "chunk": "", + "next_offset": offset, + "size": 0, + "complete": complete_status, + } + + file_size = log_path.stat().st_size + if offset > file_size: + offset = file_size + + with open(log_path, "rb") as handle: + handle.seek(offset) + payload = handle.read(limit) + + next_offset = offset + len(payload) + return { + "chunk": payload.decode("utf-8", errors="replace"), + "next_offset": next_offset, + "size": file_size, + "complete": bool(complete_status and next_offset >= file_size), + } + + +@router.get("/seeds/{seed_id}", response_class=HTMLResponse) +def seed_detail_page(request: Request, seed_id: str) -> HTMLResponse: + workflow = _workflow(request) + try: + detail = workflow.seed_detail(seed_id) + except KeyError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + return _render(request, "seed_detail_page.html", {**detail, "daemon_status": get_daemon_status()}) + + +@router.post("/actions/seeds", response_class=HTMLResponse) +def create_seed( + request: Request, + prompt: str = Form(...), + baseline_branch: str = Form(...), + seed_mode: str = Form("manual"), +) -> Response: + workflow = _workflow(request) + seed = workflow.create_seed( + prompt, + baseline_branch=baseline_branch, + ralph_loop_enabled=seed_mode == "ralph", + ) + if seed_mode == "ralph": + try: + workflow.queue_p(seed.seed_id) + except (RuntimeError, GitCommandError) as exc: + workflow.seed_repo.append_event( + seed.seed_id, + "ralph.start_failed", + f"Ralph loop could not queue the initial Plan run: {exc}", + ) + target_url = str(request.url_for("dashboard")) + f"?seed_id={seed.seed_id}" + if _is_htmx(request): + response = Response(status_code=204) + response.headers["HX-Redirect"] = target_url + return response + return RedirectResponse(target_url, status_code=303) + + +@router.post("/actions/direct-code-agent", response_class=HTMLResponse) +def start_direct_code_agent(request: Request, prompt: str = Form(...)) -> Response: + workflow = _workflow(request) + try: + seed, _run = workflow.create_direct_code_seed(prompt) + except RuntimeError as exc: + if _is_htmx(request): + return _render(request, "partials/action_error.html", {"message": str(exc)}, status_code=400) + raise HTTPException(status_code=400, detail=str(exc)) from exc + target_url = str(request.url_for("dashboard")) + f"?seed_id={seed.seed_id}" + if _is_htmx(request): + response = Response(status_code=204) + response.headers["HX-Redirect"] = target_url + return response + return RedirectResponse(target_url, status_code=303) + + +@router.post("/actions/seeds/{seed_id}/p", response_class=HTMLResponse) +def queue_p(request: Request, seed_id: str) -> Response: + workflow = _workflow(request) + try: + workflow.queue_p(seed_id) + except KeyError as exc: + if _is_htmx(request): + return _render(request, "partials/action_error.html", {"message": str(exc)}, status_code=404) + raise HTTPException(status_code=404, detail=str(exc)) from exc + except (RuntimeError, GitCommandError) as exc: + if _is_htmx(request): + return _render(request, "partials/action_error.html", {"message": str(exc)}, status_code=400) + raise HTTPException(status_code=400, detail=str(exc)) from exc + target_url = str(request.url_for("dashboard")) + f"?seed_id={seed_id}" + if _is_htmx(request): + response = Response(status_code=204) + response.headers["HX-Redirect"] = target_url + return response + return RedirectResponse(target_url, status_code=303) + + +@router.post("/actions/seeds/{seed_id}/prompt", response_class=HTMLResponse) +def update_seed_prompt(request: Request, seed_id: str, prompt: str = Form(...)) -> Response: + workflow = _workflow(request) + try: + workflow.update_seed_prompt(seed_id, prompt) + except KeyError as exc: + if _is_htmx(request): + return _render(request, "partials/action_error.html", {"message": str(exc)}, status_code=404) + raise HTTPException(status_code=404, detail=str(exc)) from exc + except RuntimeError as exc: + if _is_htmx(request): + return _render(request, "partials/action_error.html", {"message": str(exc)}, status_code=400) + raise HTTPException(status_code=400, detail=str(exc)) from exc + + if _is_htmx(request): + detail = workflow.seed_detail(seed_id) + dashboard = workflow.build_dashboard(selected_seed_id=seed_id) + context = { + **detail, + "dashboard": dashboard, + "selected_seed_id": seed_id, + "oob": True, + "daemon_status": get_daemon_status(), + } + return _render(request, "partials/seed_detail_response.html", context) + + target_url = str(request.url_for("dashboard")) + f"?seed_id={seed_id}" + return RedirectResponse(target_url, status_code=303) + + +@router.post("/actions/seeds/{seed_id}/dca", response_class=HTMLResponse) +def queue_dca(request: Request, seed_id: str) -> Response: + workflow = _workflow(request) + try: + workflow.queue_dca(seed_id) + except (KeyError, RuntimeError) as exc: + if _is_htmx(request): + return _render(request, "partials/action_error.html", {"message": str(exc)}, status_code=400) + raise HTTPException(status_code=400, detail=str(exc)) from exc + target_url = str(request.url_for("dashboard")) + f"?seed_id={seed_id}" + if _is_htmx(request): + response = Response(status_code=204) + response.headers["HX-Redirect"] = target_url + return response + return RedirectResponse(target_url, status_code=303) + + +@router.post("/actions/seeds/{seed_id}/ralph/start", response_class=HTMLResponse) +def start_ralph_loop(request: Request, seed_id: str) -> Response: + workflow = _workflow(request) + try: + seed = workflow.set_ralph_loop(seed_id, True) + if seed.status in { + SeedStatus.draft, + SeedStatus.generated, + SeedStatus.passed, + SeedStatus.failed, + SeedStatus.promoted, + }: + workflow.queue_p(seed_id) + except KeyError as exc: + if _is_htmx(request): + return _render(request, "partials/action_error.html", {"message": str(exc)}, status_code=404) + raise HTTPException(status_code=404, detail=str(exc)) from exc + except (RuntimeError, GitCommandError) as exc: + if _is_htmx(request): + return _render(request, "partials/action_error.html", {"message": str(exc)}, status_code=400) + raise HTTPException(status_code=400, detail=str(exc)) from exc + target_url = str(request.url_for("dashboard")) + f"?seed_id={seed_id}" + if _is_htmx(request): + response = Response(status_code=204) + response.headers["HX-Redirect"] = target_url + return response + return RedirectResponse(target_url, status_code=303) + + +@router.post("/actions/seeds/{seed_id}/ralph/stop", response_class=HTMLResponse) +def stop_ralph_loop(request: Request, seed_id: str) -> Response: + workflow = _workflow(request) + try: + workflow.set_ralph_loop(seed_id, False) + except KeyError as exc: + if _is_htmx(request): + return _render(request, "partials/action_error.html", {"message": str(exc)}, status_code=404) + raise HTTPException(status_code=404, detail=str(exc)) from exc + target_url = str(request.url_for("dashboard")) + f"?seed_id={seed_id}" + if _is_htmx(request): + response = Response(status_code=204) + response.headers["HX-Redirect"] = target_url + return response + return RedirectResponse(target_url, status_code=303) diff --git a/component_system/web/static/app.css b/component_system/web/static/app.css new file mode 100644 index 00000000..7edabb7c --- /dev/null +++ b/component_system/web/static/app.css @@ -0,0 +1,137 @@ +:root { + color-scheme: dark; + --card-bg: rgb(15 23 42 / 0.6); + --card-border: rgb(51 65 85); + --muted: rgb(148 163 184); +} + +body { + font-family: + Inter, + ui-sans-serif, + system-ui, + -apple-system, + BlinkMacSystemFont, + "Segoe UI", + sans-serif; + -webkit-font-smoothing: antialiased; +} + +/* IDs and branch names */ +.font-mono-id { + font-family: ui-monospace, "Cascadia Code", "Source Code Pro", Menlo, Consolas, monospace; + font-size: 0.9em; + letter-spacing: 0.02em; +} + +.line-clamp-3 { + display: -webkit-box; + overflow: hidden; + -webkit-box-orient: vertical; + -webkit-line-clamp: 3; +} + +/* Card hover for clickable seed cards */ +.seed-card { + transition: border-color 0.15s ease, background-color 0.15s ease; +} +.seed-card:hover { + border-color: rgb(56 189 248 / 0.5); + background-color: rgb(15 23 42 / 0.9); +} +.seed-card.is-selected { + border-color: rgb(14 165 233); + background-color: rgb(14 165 233 / 0.14); + box-shadow: inset 0 0 0 1px rgb(14 165 233 / 0.35); +} + +/* Status pills */ +.status-pill { + display: inline-flex; + align-items: center; + border: 1px solid transparent; + font-size: 0.625rem; + font-weight: 600; + letter-spacing: 0.04em; + text-transform: uppercase; + line-height: 1; + padding: 0.2rem 0.5rem; + border-radius: 9999px; + white-space: nowrap; +} +.status-draft { background: rgb(51 65 85 / 0.62); border-color: rgb(148 163 184 / 0.4); color: rgb(226 232 240); } +.status-queued { background: rgb(146 64 14 / 0.45); border-color: rgb(245 158 11 / 0.45); color: rgb(253 230 138); } +.status-planning { background: rgb(30 64 175 / 0.4); border-color: rgb(96 165 250 / 0.45); color: rgb(191 219 254); } +.status-generated { background: rgb(15 118 110 / 0.38); border-color: rgb(45 212 191 / 0.4); color: rgb(153 246 228); } +.status-dca_queued { background: rgb(8 145 178 / 0.33); border-color: rgb(34 211 238 / 0.38); color: rgb(165 243 252); } +.status-adapting, +.status-running { background: rgb(109 40 217 / 0.35); border-color: rgb(192 132 252 / 0.42); color: rgb(233 213 255); } +.status-passed { background: rgb(21 128 61 / 0.28); border-color: rgb(74 222 128 / 0.4); color: rgb(187 247 208); } +.status-failed { background: rgb(153 27 27 / 0.34); border-color: rgb(248 113 113 / 0.42); color: rgb(254 202 202); } +.status-promoted { background: rgb(22 163 74 / 0.28); border-color: rgb(74 222 128 / 0.42); color: rgb(187 247 208); } + +/* Empty state placeholder */ +.empty-value { + color: rgb(100 116 139); + font-style: normal; +} + +/* Section headers */ +.section-label { + font-size: 11px; + letter-spacing: 0.2em; + text-transform: uppercase; + color: rgb(100 116 139); +} + +/* Scroll containers for long dashboard lists */ +.scroll-pane { + min-height: 0; + overflow-y: auto; + scrollbar-gutter: stable; +} + +.scroll-pane-stage { + max-height: min(32rem, 68vh); +} + +.scroll-pane-detail { + max-height: min(30rem, 62vh); +} + +.run-log-grid { + display: grid; + gap: 0.75rem; + grid-template-columns: minmax(0, 2fr) minmax(0, 4fr) minmax(0, 4fr); +} + +.run-log-pane { + min-width: 0; + min-height: 0; + display: flex; + flex-direction: column; +} + +.run-log-pre { + min-width: 0; + min-height: 0; + flex: 1 1 auto; + overflow: auto; + white-space: pre-wrap; + overflow-wrap: anywhere; + word-break: break-word; + border: 1px solid rgb(30 41 59); + border-radius: 0.25rem; + background: rgb(0 0 0 / 0.3); + padding: 0.5rem; + font-family: ui-monospace, "Cascadia Code", "Source Code Pro", Menlo, Consolas, monospace; + font-size: 11px; + line-height: 1.25rem; + color: rgb(226 232 240); +} + +@media (max-width: 1024px) { + .run-log-grid { + grid-template-columns: 1fr; + } +} diff --git a/component_system/web/static/app.js b/component_system/web/static/app.js new file mode 100644 index 00000000..34de6c3a --- /dev/null +++ b/component_system/web/static/app.js @@ -0,0 +1,513 @@ +document.body.addEventListener("htmx:responseError", (event) => { + const target = event.detail.target; + if (!target) { + return; + } + target.innerHTML = `
Request failed.
`; +}); + +function selectedSeedIdFromUrl() { + const params = new URLSearchParams(window.location.search); + return params.get("seed_id"); +} + +function applySelectedSeed(seedId) { + const cards = document.querySelectorAll(".seed-card[data-seed-id]"); + cards.forEach((card) => { + const isSelected = seedId !== null && card.dataset.seedId === seedId; + card.classList.toggle("is-selected", isSelected); + card.setAttribute("aria-current", isSelected ? "true" : "false"); + }); +} + +let dashboardPollInFlight = false; +let seedDetailPollInFlight = false; +let seedVersionsPollInFlight = false; +const lastSeedVersions = {}; +const savedScrollPositions = { runs: null, timeline: null }; +const INTERACTION_DEBOUNCE_MS = 3000; +let lastRunsInteraction = 0; +let lastTimelineInteraction = 0; + +function seedDetailUrl(seedId) { + const detail = document.getElementById("seed-detail"); + const template = detail?.dataset.seedDetailUrlTemplate; + if (!template || !seedId) { + return null; + } + return template.replace("__SEED_ID__", encodeURIComponent(seedId)); +} + +function seedVersionsUrl(seedId) { + const detail = document.getElementById("seed-detail"); + const template = detail?.dataset.seedVersionsUrlTemplate; + if (!template || !seedId) return null; + return template.replace("__SEED_ID__", encodeURIComponent(seedId)); +} + +function seedRunsUrl(seedId) { + const detail = document.getElementById("seed-detail"); + const template = detail?.dataset.seedRunsUrlTemplate; + if (!template || !seedId) return null; + return template.replace("__SEED_ID__", encodeURIComponent(seedId)); +} + +function seedTimelineUrl(seedId) { + const detail = document.getElementById("seed-detail"); + const template = detail?.dataset.seedTimelineUrlTemplate; + if (!template || !seedId) return null; + return template.replace("__SEED_ID__", encodeURIComponent(seedId)); +} + +function isLogViewerOpen() { + const target = document.getElementById("seed-detail"); + if (!target) { + return false; + } + if (target.querySelector('[data-log-viewer-open="true"]')) { + return true; + } + if (target.querySelector("[data-log-stream]")) { + return true; + } + const seedId = selectedSeedIdFromUrl(); + return Boolean(seedId && localStorage.getItem(`seed-active-run-${seedId}`)); +} + +function dashboardBoardUrl() { + const board = document.getElementById("dashboard-board"); + const base = board?.dataset.dashboardPartialUrl; + if (!base) { + return null; + } + const seedId = selectedSeedIdFromUrl(); + if (!seedId) { + return base; + } + const separator = base.includes("?") ? "&" : "?"; + return `${base}${separator}seed_id=${encodeURIComponent(seedId)}`; +} + +function pollDashboardBoard() { + const target = document.getElementById("dashboard-board"); + const url = dashboardBoardUrl(); + if (!target || !url || dashboardPollInFlight) { + return; + } + dashboardPollInFlight = true; + htmx + .ajax("GET", url, { target: "#dashboard-board", swap: "outerHTML" }) + .finally(() => { + dashboardPollInFlight = false; + }); +} + +function pollSeedDetail() { + const seedId = selectedSeedIdFromUrl(); + const target = document.getElementById("seed-detail"); + const url = seedDetailUrl(seedId); + if (!target || !url || seedDetailPollInFlight) { + return; + } + if (isLogViewerOpen()) { + return; + } + seedDetailPollInFlight = true; + htmx.ajax("GET", url, { target: "#seed-detail", swap: "innerHTML" }).finally(() => { + seedDetailPollInFlight = false; + }); +} + +function applyRunsPartial(seedId) { + const listEl = document.getElementById("seed-runs-list"); + const paneEl = document.getElementById("seed-runs-scroll-pane"); + const url = seedRunsUrl(seedId); + if (!listEl || !url) return Promise.resolve(); + savedScrollPositions.runs = paneEl ? paneEl.scrollTop : null; + return htmx.ajax("GET", url, { target: "#seed-runs-list", swap: "innerHTML" }); +} + +function applyTimelinePartial(seedId) { + const listEl = document.getElementById("seed-timeline-list"); + const paneEl = document.getElementById("seed-timeline-scroll-pane"); + const url = seedTimelineUrl(seedId); + if (!listEl || !url) return Promise.resolve(); + savedScrollPositions.timeline = paneEl ? paneEl.scrollTop : null; + return htmx.ajax("GET", url, { target: "#seed-timeline-list", swap: "innerHTML" }); +} + +function pollSeedDetailSections() { + const seedId = selectedSeedIdFromUrl(); + if (!seedId || isLogViewerOpen()) return; + const versionsUrl = seedVersionsUrl(seedId); + if (!versionsUrl || seedVersionsPollInFlight) return; + seedVersionsPollInFlight = true; + fetch(versionsUrl) + .then((r) => (r.ok ? r.json() : null)) + .then((data) => { + if (!data) return; + const prev = lastSeedVersions[seedId] || {}; + const runsChanged = data.runs_version !== prev.runs_version; + const timelineChanged = data.timeline_version !== prev.timeline_version; + lastSeedVersions[seedId] = { + runs_version: data.runs_version, + timeline_version: data.timeline_version, + }; + const now = Date.now(); + const runsIdle = now - lastRunsInteraction >= INTERACTION_DEBOUNCE_MS; + const timelineIdle = now - lastTimelineInteraction >= INTERACTION_DEBOUNCE_MS; + const promises = []; + if (runsChanged && runsIdle) promises.push(applyRunsPartial(seedId)); + if (timelineChanged && timelineIdle) promises.push(applyTimelinePartial(seedId)); + return Promise.all(promises); + }) + .finally(() => { + seedVersionsPollInFlight = false; + }); +} + +function attachScrollPaneInteractionGuards() { + const runsPane = document.getElementById("seed-runs-scroll-pane"); + const timelinePane = document.getElementById("seed-timeline-scroll-pane"); + function onRunsActivity() { + lastRunsInteraction = Date.now(); + } + function onTimelineActivity() { + lastTimelineInteraction = Date.now(); + } + runsPane?.addEventListener("scroll", onRunsActivity, { passive: true }); + runsPane?.addEventListener("mouseenter", onRunsActivity); + runsPane?.addEventListener("focusin", onRunsActivity); + timelinePane?.addEventListener("scroll", onTimelineActivity, { passive: true }); + timelinePane?.addEventListener("mouseenter", onTimelineActivity); + timelinePane?.addEventListener("focusin", onTimelineActivity); +} + +function pollDashboard() { + if (document.hidden) return; + if (isLogViewerOpen()) return; + pollDashboardBoard(); + const seedId = selectedSeedIdFromUrl(); + if (seedId && document.getElementById("seed-runs-list")) { + pollSeedDetailSections(); + } else if (seedId && !document.getElementById("seed-runs-list")) { + pollSeedDetail(); + } +} + +document.body.addEventListener("htmx:beforeRequest", (event) => { + const target = event.detail?.target; + if (!target || !isLogViewerOpen()) { + return; + } + // Pause daemon status auto-refresh while viewing logs. + if (target.id === "daemon-status-panel") { + event.preventDefault(); + } +}); + +document.body.addEventListener("click", (event) => { + const card = event.target.closest(".seed-card[data-seed-id]"); + if (!card) { + return; + } + applySelectedSeed(card.dataset.seedId); +}); + +document.body.addEventListener("htmx:afterSettle", (event) => { + const target = event.detail?.target; + if (!target) return; + if (target.id === "seed-detail") { + applySelectedSeed(selectedSeedIdFromUrl()); + attachScrollPaneInteractionGuards(); + return; + } + if (target.id === "seed-runs-list") { + const pane = document.getElementById("seed-runs-scroll-pane"); + if (pane && savedScrollPositions.runs != null) { + pane.scrollTop = savedScrollPositions.runs; + savedScrollPositions.runs = null; + } + initializeLogStreams(target.closest("#seed-detail") || document); + return; + } + if (target.id === "seed-timeline-list") { + const pane = document.getElementById("seed-timeline-scroll-pane"); + if (pane && savedScrollPositions.timeline != null) { + pane.scrollTop = savedScrollPositions.timeline; + savedScrollPositions.timeline = null; + } + return; + } +}); + +window.addEventListener("popstate", () => { + applySelectedSeed(selectedSeedIdFromUrl()); +}); + +applySelectedSeed(selectedSeedIdFromUrl()); +attachScrollPaneInteractionGuards(); +window.setInterval(pollDashboard, 5000); + +const logStreamIntervals = new Map(); +const logStreamState = new Map(); +const ansiCtor = window.AnsiUp || window.ansi_up?.AnsiUp || null; +const ansiRenderer = ansiCtor ? new ansiCtor() : null; + +if (ansiRenderer && Object.prototype.hasOwnProperty.call(ansiRenderer, "escape_html")) { + ansiRenderer.escape_html = true; +} + +function stripAnsiSequences(value) { + // CSI: \x1b[...m, OSC: \x1b]...\x07 or \x1b\ ; then any remaining ESC controls. + return (value || "") + .replace(/\u001b\][^\u0007]*(?:\u0007|\u001b\\)/g, "") + .replace(/\u001b\[[0-?]*[ -/]*[@-~]/g, "") + .replace(/\u001b[@-_]/g, ""); +} + +function isRunComplete(status) { + return status === "succeeded" || status === "failed"; +} + +function updateLogStatus(runId, text) { + const nodes = document.querySelectorAll(`[data-log-status][data-run-id="${runId}"]`); + nodes.forEach((node) => { + node.textContent = text; + }); +} + +function updateCopyButtonState(runId, stream, enabled) { + const buttons = document.querySelectorAll( + `[data-log-copy][data-run-id="${runId}"][data-stream="${stream}"]` + ); + buttons.forEach((button) => { + button.disabled = !enabled; + }); +} + +function appendLogContent(pre, chunk) { + const currentRaw = pre.dataset.rawLog || ""; + const nextRaw = currentRaw + (chunk || ""); + + // Keep the viewer responsive for very large logs. + const maxChars = 200_000; + const trimmedRaw = + nextRaw.length > maxChars ? nextRaw.slice(nextRaw.length - maxChars) : nextRaw; + + pre.dataset.rawLog = trimmedRaw; + if (ansiRenderer) { + pre.innerHTML = ansiRenderer.ansi_to_html(trimmedRaw); + } else { + pre.textContent = stripAnsiSequences(trimmedRaw); + } + + pre.scrollTop = pre.scrollHeight; +} + +async function pollLogStream(pre) { + const runId = pre.dataset.runId; + const stream = pre.dataset.stream || "stdout"; + if (!runId) { + return; + } + + const state = logStreamState.get(pre) || { offset: 0, complete: false }; + const response = await fetch( + `/component-system/api/runs/${encodeURIComponent(runId)}/log?stream=${encodeURIComponent(stream)}&offset=${state.offset}` + ); + if (!response.ok) { + throw new Error(`Failed to fetch logs for ${runId}: ${response.status}`); + } + + const payload = await response.json(); + const chunk = payload.chunk || ""; + const nextOffset = Number(payload.next_offset || 0); + const complete = Boolean(payload.complete); + + appendLogContent(pre, chunk); + updateCopyButtonState(runId, stream, pre.textContent.length > 0); + logStreamState.set(pre, { offset: nextOffset, complete }); + + if (complete) { + updateLogStatus(runId, "Completed"); + const intervalId = logStreamIntervals.get(pre); + if (intervalId) { + clearInterval(intervalId); + logStreamIntervals.delete(pre); + } + return; + } + + if (chunk) { + updateLogStatus(runId, "Streaming..."); + } else { + updateLogStatus(runId, "Waiting for log output..."); + } +} + +function cleanupDetachedLogStreams() { + for (const [pre, intervalId] of logStreamIntervals.entries()) { + if (!document.body.contains(pre)) { + clearInterval(intervalId); + logStreamIntervals.delete(pre); + logStreamState.delete(pre); + } + } +} + +function initializeLogCopyButtons(root) { + root.querySelectorAll("[data-log-copy]").forEach((button) => { + if (button.dataset.logCopyReady === "true") { + return; + } + button.dataset.logCopyReady = "true"; + button.addEventListener("click", async () => { + const runId = button.dataset.runId; + if (!runId) { + return; + } + const stream = button.dataset.stream || "stdout"; + const pre = root.querySelector( + `[data-log-stream][data-run-id="${runId}"][data-stream="${stream}"]` + ); + if (!pre || !pre.textContent) { + return; + } + try { + await navigator.clipboard.writeText(pre.textContent); + const labelBefore = button.textContent; + button.textContent = "Copied!"; + setTimeout(() => { + button.textContent = labelBefore || "Copy"; + }, 1200); + } catch (error) { + console.error("Failed to copy log output", error); + } + }); + }); +} + +async function loadPromptContent(pre) { + const runId = pre.dataset.runId; + if (!runId) return; + try { + const response = await fetch( + `/component-system/api/runs/${encodeURIComponent(runId)}/prompt` + ); + if (!response.ok) return; + const payload = await response.json(); + const content = payload.content ?? ""; + pre.textContent = content; + const copyBtn = document.querySelector( + `[data-prompt-copy][data-run-id="${runId}"]` + ); + if (copyBtn) copyBtn.disabled = false; + } catch (err) { + console.error("Failed to load prompt for run", runId, err); + } +} + +function initializePromptDisplays(root) { + root.querySelectorAll("[data-prompt-content]").forEach((pre) => { + if (pre.dataset.promptLoaded === "true") return; + pre.dataset.promptLoaded = "true"; + loadPromptContent(pre); + }); + root.querySelectorAll("[data-prompt-copy]").forEach((button) => { + if (button.dataset.promptCopyReady === "true") return; + button.dataset.promptCopyReady = "true"; + button.addEventListener("click", async () => { + const runId = button.dataset.runId; + if (!runId) return; + const pre = root.querySelector( + `[data-prompt-content][data-run-id="${runId}"]` + ); + if (!pre || !pre.textContent) return; + try { + await navigator.clipboard.writeText(pre.textContent); + const labelBefore = button.textContent; + button.textContent = "Copied!"; + setTimeout(() => { + button.textContent = labelBefore || "Copy"; + }, 1200); + } catch (err) { + console.error("Failed to copy prompt", err); + } + }); + }); +} + +function initializeLogStreams(root = document) { + cleanupDetachedLogStreams(); + initializeLogCopyButtons(root); + initializePromptDisplays(root); + + root.querySelectorAll("[data-log-stream]").forEach((pre) => { + if (pre.dataset.logStreamReady === "true") { + return; + } + pre.dataset.logStreamReady = "true"; + const runStatus = pre.dataset.runStatus || ""; + const runId = pre.dataset.runId; + if (!runId) { + return; + } + + if (isRunComplete(runStatus)) { + updateLogStatus(runId, "Completed"); + } else { + updateLogStatus(runId, "Connecting..."); + } + + const runPoll = async () => { + try { + await pollLogStream(pre); + } catch (error) { + updateLogStatus(runId, "Log fetch failed"); + console.error(error); + } + }; + + runPoll(); + const intervalId = window.setInterval(runPoll, 2000); + logStreamIntervals.set(pre, intervalId); + }); +} + +function observeLogStreamMounts() { + const observer = new MutationObserver((mutations) => { + for (const mutation of mutations) { + if (mutation.type !== "childList" || mutation.addedNodes.length === 0) { + continue; + } + for (const node of mutation.addedNodes) { + if (!(node instanceof Element)) { + continue; + } + if ( + node.matches?.("[data-log-stream], [data-log-copy], [data-prompt-content], [data-prompt-copy]") || + node.querySelector?.("[data-log-stream], [data-log-copy], [data-prompt-content], [data-prompt-copy]") + ) { + initializeLogStreams(node); + return; + } + } + } + }); + + observer.observe(document.body, { childList: true, subtree: true }); +} + +document.body.addEventListener("htmx:afterSettle", (event) => { + const target = event.detail?.target; + if (!target) { + return; + } + if (target.id === "seed-detail") { + initializeLogStreams(target); + } +}); + +initializeLogStreams(document); +observeLogStreamMounts(); diff --git a/component_system/web/static/tailwind.input.css b/component_system/web/static/tailwind.input.css new file mode 100644 index 00000000..a563500f --- /dev/null +++ b/component_system/web/static/tailwind.input.css @@ -0,0 +1,27 @@ +@tailwind base; +@tailwind components; +@tailwind utilities; + +@layer base { + :root { + color-scheme: dark; + } + + body { + @apply min-h-screen bg-slate-950 text-slate-100; + font-family: + Inter, + ui-sans-serif, + system-ui, + -apple-system, + BlinkMacSystemFont, + "Segoe UI", + sans-serif; + } +} + +@layer utilities { + .card-panel { + @apply rounded-2xl border border-slate-800 bg-slate-900; + } +} diff --git a/component_system/web/templates/base.html b/component_system/web/templates/base.html new file mode 100644 index 00000000..ef14a634 --- /dev/null +++ b/component_system/web/templates/base.html @@ -0,0 +1,32 @@ + + + + + + {% block title %}Component System{% endblock %} + + + + + + + + +
+
+
+ + Component System + +

Seed -> Plan -> Do-Check-Action orchestration with FastAPI, HTMX, Alpine, and Tailwind.

+
+ +
+
+
+ {% block content %}{% endblock %} +
+ + diff --git a/component_system/web/templates/dashboard.html b/component_system/web/templates/dashboard.html new file mode 100644 index 00000000..103f87f3 --- /dev/null +++ b/component_system/web/templates/dashboard.html @@ -0,0 +1,124 @@ +{% extends "base.html" %} +{% block title %}Component System Dashboard{% endblock %} +{% block content %} +
+
+
+

Create Seed

+

Start a new seed from a prompt. Baseline branch is selected here; each seed has one branch (seed id).

+
+
+
+ + +
+
+

One branch per seed: the seed id is the branch name (e.g. seed-a1b2c3).

+ + + +
+ {% if dashboard.setup_error %} +
+

Git setup required

+

{{ dashboard.setup_error }}

+
+ {% endif %} + {% with daemon_status=dashboard.daemon_status %} + {% include "partials/daemon_status.html" %} + {% endwith %} +
+

Baseline branches

+

Per-branch best val_bpb from baseline_metrics.json. Workflow-managed, read-only: component_system/baseline_branches.json (per-branch mapping), component_system/baseline_metrics.json (baseline run metrics).

+ {% if dashboard.baseline_metrics_by_branch %} +
+ {% for branch, m in dashboard.baseline_metrics_by_branch.items() %} +
+
{{ branch }}
+
val_bpb {{ "%.6f"|format(m.get('best_val_bpb')) if m.get('best_val_bpb') is not none else "—" }} · {{ m.get('promoted_branch') or "—" }}{% if m.get('commit_sha') %} · {{ m.get('commit_sha')[:7] }}{% endif %}
+
+ {% endfor %} +
+ {% else %} +

No baseline metrics yet. Run the first DCA to establish baseline for a branch.

+ {% endif %} +
+
+

Direct Code Agent

+

Run the configured code agent from the project root with a dedicated single-worker executor. New runs appear in the Do-Check-Action column.

+
+ + + +
+
+
+
+ {% include "partials/dashboard_board.html" %} +
+ {% if detail %} + {% with + seed=detail.seed, + runs=detail.runs, + events=detail.events, + baseline_metrics_for_branch=detail.baseline_metrics_for_branch, + setup_error=detail.setup_error, + daemon_status=dashboard.daemon_status + %} + {% include "partials/seed_detail.html" %} + {% endwith %} + {% else %} +
+ Select a seed to inspect its worktree, plan, runs, logs, and promotion history. +
+ {% endif %} +
+
+
+
+{% endblock %} diff --git a/component_system/web/templates/partials/action_error.html b/component_system/web/templates/partials/action_error.html new file mode 100644 index 00000000..8a856804 --- /dev/null +++ b/component_system/web/templates/partials/action_error.html @@ -0,0 +1,3 @@ +
+ {{ message }} +
diff --git a/component_system/web/templates/partials/daemon_status.html b/component_system/web/templates/partials/daemon_status.html new file mode 100644 index 00000000..75a0d5e2 --- /dev/null +++ b/component_system/web/templates/partials/daemon_status.html @@ -0,0 +1,14 @@ +
+

Daemon: {% if daemon_status == 'running' %}running{% else %}not running{% endif %}

+

Plan and Do-Check-Action runs are executed by the daemon.

+ {% if daemon_status != 'running' %} +

Start it in a terminal:

+

uv run component_system/run.py

+ {% endif %} +
diff --git a/component_system/web/templates/partials/dashboard_board.html b/component_system/web/templates/partials/dashboard_board.html new file mode 100644 index 00000000..df82ab6a --- /dev/null +++ b/component_system/web/templates/partials/dashboard_board.html @@ -0,0 +1,58 @@ +
+
+

+ Dashboard {{ dashboard.seed_count }} seed{{ 's' if dashboard.seed_count != 1 else '' }} across all stages +

+
+
+ {% for column in dashboard.columns %} +
+
+

{{ column.title }}

+

{{ column.description }}

+
+
+ {% if column.seeds %} + {% for seed in column.seeds %} + {% set is_selected = selected_seed_id == seed.seed_id %} + {% set is_promoted = column.id == 'completed' and seed.status.value == 'promoted' %} + +
+

{{ seed.seed_id }}

+ {{ seed.status.value|replace('_', ' ')|title }} +
+

{{ seed.prompt }}

+ {% if seed.plan %} +

{{ seed.plan.title }}

+ {% endif %} + {% if seed.latest_metrics and seed.latest_metrics.get('val_bpb') is not none %} +

val_bpb {{ "%.4f"|format(seed.latest_metrics.val_bpb) }}{% if seed.latest_signal %} · {{ seed.latest_signal }}{% endif %}

+ {% endif %} + + {% endfor %} + {% else %} +
+ No seeds in this stage. +
+ {% endif %} +
+
+ {% endfor %} +
+
diff --git a/component_system/web/templates/partials/seed_detail.html b/component_system/web/templates/partials/seed_detail.html new file mode 100644 index 00000000..89e81a0c --- /dev/null +++ b/component_system/web/templates/partials/seed_detail.html @@ -0,0 +1,171 @@ +
+
+
+

Seed detail

+

{{ seed.seed_id }}

+ {% if can_edit_prompt %} +
+ + + +
+ {% else %} +

{{ seed.prompt }}

+ {% endif %} +
+
+ {% if seed.ralph_loop_enabled %} +
+ +
+ {% else %} +
+ +
+ {% endif %} +
+ +
+
+ +
+
+
+ + {% if setup_error %} +
+ {{ setup_error }} +
+ {% endif %} + +
+
+
+

Status

+ {{ seed.status.value|replace('_', ' ')|title }} +
+

Ralph loop: {% if seed.ralph_loop_enabled %}enabled{% else %}disabled{% endif %}

+

Latest signal: {% if seed.latest_signal %}{{ seed.latest_signal }}{% else %}—{% endif %}

+
+
+

Branches

+
+
Baseline
{{ seed.baseline_branch }}
+
Branch
{{ seed.seed_id }}
+
+
+
+

Worktrees

+
+
Seed worktree
{{ seed.worktree_path or "—" }}
+
+
+
+ +
+
+
+

Plan

+ {% if seed.plan %} +
+
+

Idea

+

{{ seed.plan.title }}

+
+
+

Target component

+

{{ seed.plan.target_component }}

+
+
+

Description

+

{{ seed.plan.description }}

+
+ {% if seed.plan.commit_sha %} +
+

Generated commit

+

{{ seed.plan.commit_sha }}

+
+ {% endif %} +
+ {% else %} +

No plan yet. Click Run Plan to queue the task; the plan is generated when the daemon runs it.

+ {% endif %} +
+ +
+
+

Runs

+
+ {% if runs and seed.status.value in ['queued', 'planning'] and (daemon_status|default('')) != 'running' %} +

Runs stay queued until the daemon is running. Start: uv run component_system/run.py

+ {% endif %} +
+
+ {% include "partials/seed_runs_inner.html" %} +
+
+
+
+ +
+
+

Latest Metrics

+ {% if seed.latest_metrics %} +
+ {% for key, value in seed.latest_metrics.items() %} +
+
{{ key }}
+
{{ value }}
+
+ {% endfor %} +
+ {% else %} +

Metrics appear here after Do-Check-Action runs the training entrypoint.

+ {% endif %} +
+ +
+
+

Timeline

+ +
+
+
+ {% include "partials/seed_timeline_inner.html" %} +
+
+
+
+
+
diff --git a/component_system/web/templates/partials/seed_detail_response.html b/component_system/web/templates/partials/seed_detail_response.html new file mode 100644 index 00000000..c2b27e70 --- /dev/null +++ b/component_system/web/templates/partials/seed_detail_response.html @@ -0,0 +1 @@ +{% include "partials/seed_detail.html" %} diff --git a/component_system/web/templates/partials/seed_detail_runs_content.html b/component_system/web/templates/partials/seed_detail_runs_content.html new file mode 100644 index 00000000..278a12ba --- /dev/null +++ b/component_system/web/templates/partials/seed_detail_runs_content.html @@ -0,0 +1,148 @@ +{% if runs and seed.status.value in ['queued', 'planning'] %} +

Runs stay queued until the daemon is running. Start: uv run component_system/run.py

+{% endif %} +{% if runs %} + {% for run in runs %} +
+
+
+

{% if run.stage.value == 'p' %}Plan{% else %}{{ run.stage.value|upper }}{% endif %} · {{ run.status.value }}

+

{{ run.run_id }}

+
+
+ {% if run.signal %} + {{ run.signal }} + {% endif %} + +
+
+ {% if run.metrics %} +
+ {% for key, value in run.metrics.items() %} +
+
{{ key }}
+
{{ value }}
+
+ {% endfor %} +
+ {% endif %} +
+ + {% endfor %} +{% else %} +

No runs yet. Use Run Plan to start.

+{% endif %} diff --git a/component_system/web/templates/partials/seed_detail_timeline_content.html b/component_system/web/templates/partials/seed_detail_timeline_content.html new file mode 100644 index 00000000..8fabcdd5 --- /dev/null +++ b/component_system/web/templates/partials/seed_detail_timeline_content.html @@ -0,0 +1,16 @@ +{% if events %} + {% for event in events %} +
+

{{ event.message }}

+ {% if event.commit_sha %} +

commit: {{ event.commit_sha }}

+ {% endif %} + {% if event.target_branch %} +

target branch: {{ event.target_branch }}

+ {% endif %} +

{{ event.kind }} · {{ event.created_at_human }}

+
+ {% endfor %} +{% else %} +

No events yet.

+{% endif %} diff --git a/component_system/web/templates/partials/seed_runs_inner.html b/component_system/web/templates/partials/seed_runs_inner.html new file mode 100644 index 00000000..3d4c3c67 --- /dev/null +++ b/component_system/web/templates/partials/seed_runs_inner.html @@ -0,0 +1,164 @@ +{% if runs %} + {% for run in runs %} +
+
+
+ {% if run.created_at %} +

{{ run.created_at|format_ts }}

+ {% endif %} +

{% if run.stage.value == 'p' %}Plan{% else %}{{ run.stage.value|upper }}{% endif %} · {{ run.status.value }}

+

{{ run.run_id }}

+
+
+ {% if run.signal %} + {{ run.signal }} + {% endif %} + +
+
+ {% if run.stage.value == 'p' and run.summary and (run.summary.get('idea') or run.summary.get('description')) %} +
+ {% if run.summary.get('idea') %} +
+

Idea

+

{{ run.summary.get('idea', '') }}

+
+ {% endif %} + {% if run.summary.get('description') %} +
+

Description

+

{{ run.summary.get('description', '') }}

+
+ {% endif %} +
+ {% endif %} + {% if run.metrics %} +
+ {% for key, value in run.metrics.items() %} +
+
{{ key }}
+
{{ value }}
+
+ {% endfor %} +
+ {% endif %} +
+ + {% endfor %} +{% else %} +

No runs yet. Use Run Plan to start.

+{% endif %} diff --git a/component_system/web/templates/partials/seed_timeline_inner.html b/component_system/web/templates/partials/seed_timeline_inner.html new file mode 100644 index 00000000..d545f0e1 --- /dev/null +++ b/component_system/web/templates/partials/seed_timeline_inner.html @@ -0,0 +1,10 @@ +{% if events %} + {% for event in events %} +
+

{{ event.display_message | default(event.message) }}

+

{{ event.kind }} · {{ event.created_at_human }}

+
+ {% endfor %} +{% else %} +

No events yet.

+{% endif %} diff --git a/component_system/web/templates/seed_detail_page.html b/component_system/web/templates/seed_detail_page.html new file mode 100644 index 00000000..4fa3d6f0 --- /dev/null +++ b/component_system/web/templates/seed_detail_page.html @@ -0,0 +1,18 @@ +{% extends "base.html" %} +{% block title %}Seed {{ seed.seed_id }}{% endblock %} +{% block content %} +
+ + Back to dashboard + +
+
+ {% include "partials/seed_detail.html" %} +
+{% endblock %} diff --git a/prepare.py b/prepare.py index 06bea916..7568f05a 100644 --- a/prepare.py +++ b/prepare.py @@ -38,7 +38,8 @@ CACHE_DIR = os.path.join(os.path.expanduser("~"), ".cache", "autoresearch") DATA_DIR = os.path.join(CACHE_DIR, "data") TOKENIZER_DIR = os.path.join(CACHE_DIR, "tokenizer") -BASE_URL = "https://huggingface.co/datasets/karpathy/climbmix-400b-shuffle/resolve/main" +HF_ENDPOINT = os.environ.get("HF_ENDPOINT", "https://huggingface.co").rstrip("/") +BASE_URL = f"{HF_ENDPOINT}/datasets/karpathy/climbmix-400b-shuffle/resolve/main" MAX_SHARD = 6542 # the last datashard is shard_06542.parquet VAL_SHARD = MAX_SHARD # pinned validation shard (shard_06542) VAL_FILENAME = f"shard_{VAL_SHARD:05d}.parquet" diff --git a/pyproject.toml b/pyproject.toml index 94ae3298..8882b6fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,15 +5,20 @@ description = "Autonomous pretraining research swarm" readme = "README.md" requires-python = ">=3.10" dependencies = [ + "arxiv>=2.4.1", + "fastapi>=0.116.0", + "jinja2>=3.1.6", "kernels>=0.11.7", "matplotlib>=3.10.8", "numpy>=2.2.6", "pandas>=2.3.3", "pyarrow>=21.0.0", + "python-multipart>=0.0.20", "requests>=2.32.0", "rustbpe>=0.1.0", "tiktoken>=0.11.0", "torch==2.9.1", + "uvicorn>=0.35.0", ] [tool.uv.sources] diff --git a/scripts/clean_history.py b/scripts/clean_history.py new file mode 100644 index 00000000..13b4b1f7 --- /dev/null +++ b/scripts/clean_history.py @@ -0,0 +1,371 @@ +#!/usr/bin/env python3 +"""Reset local autoresearch history/runtime artifacts. + +Actions: +1) Checkout main branch (configurable) +2) Remove all extra git worktrees +3) Delete only local branches that match seed-* or equal __baseline__ (other branches are left intact) +4) Clear component_system runtime state/history folders +5) Remove .pytest_cache, __pycache__, and results.tsv + +With --preserve-seeds SEED_IDS: keep everything for those seeds (state, events, runs, +queue tasks, worktrees, branches, logs, baseline mappings); remove only other seeds' data. +SEED_IDS can be comma-separated, e.g. --preserve-seeds seed-a,seed-b,seed-c. +""" + +from __future__ import annotations + +import argparse +import json +import shutil +import subprocess +from pathlib import Path + + +def _read_json(path: Path, default: object) -> object: + if not path.exists(): + return default + return json.loads(path.read_text(encoding="utf-8")) + + +def _write_json(path: Path, payload: object) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, ensure_ascii=False), encoding="utf-8") + + +def run_git(args: list[str], cwd: Path, dry_run: bool = False) -> list[str]: + cmd = ["git", *args] + if dry_run: + print(f"[dry-run] {' '.join(cmd)}") + return [] + proc = subprocess.run(cmd, cwd=cwd, text=True, capture_output=True) + if proc.returncode != 0: + raise RuntimeError( + f"Command failed: {' '.join(cmd)}\n" + f"stdout:\n{proc.stdout}\n" + f"stderr:\n{proc.stderr}" + ) + return [line for line in proc.stdout.splitlines() if line.strip()] + + +def is_broken_worktree_remove_error(error: RuntimeError) -> bool: + msg = str(error) + return ( + "worktree remove --force" in msg + and "validation failed, cannot remove working tree" in msg + and ".git' does not exist" in msg + ) + + +def remove_children(path: Path, dry_run: bool = False) -> None: + if not path.exists(): + return + for child in path.iterdir(): + if dry_run: + print(f"[dry-run] remove {child}") + continue + if child.is_dir(): + shutil.rmtree(child, ignore_errors=True) + else: + child.unlink(missing_ok=True) + + +def remove_pycache_dirs(repo_root: Path, dry_run: bool = False) -> None: + for pycache in repo_root.rglob("__pycache__"): + parts = set(pycache.parts) + if ".venv" in parts or ".git" in parts: + continue + if pycache.is_dir(): + if dry_run: + print(f"[dry-run] remove {pycache}") + else: + shutil.rmtree(pycache, ignore_errors=True) + + +def _gather_preserved_seed_info( + repo_root: Path, seed_ids: list[str] +) -> tuple[set[str], set[str]]: + """Return (preserved_run_ids, baseline_branches). Exits if any seed not found.""" + comp = repo_root / "component_system" + history = comp / "history" + state = history / "state" + seeds_dir = state / "seeds" + runs_dir = state / "runs" + preserved_ids = set(seed_ids) + baseline_branches: set[str] = set() + run_ids: set[str] = set() + + for seed_id in seed_ids: + seed_file = seeds_dir / f"{seed_id}.json" + if not seed_file.exists(): + raise SystemExit(f"Seed not found: {seed_id} (no {seed_file})") + seed_data = _read_json(seed_file, {}) + if isinstance(seed_data, dict): + bl = seed_data.get("baseline_branch") + if isinstance(bl, str): + baseline_branches.add(bl) + + for path in runs_dir.glob("*.json"): + data = _read_json(path, {}) + if isinstance(data, dict) and data.get("seed_id") in preserved_ids: + rid = data.get("run_id") + if isinstance(rid, str): + run_ids.add(rid) + return run_ids, baseline_branches + + +def _clean_state_preserving_seeds( + repo_root: Path, + preserved_seed_ids: set[str], + preserved_run_ids: set[str], + dry_run: bool, +) -> None: + comp = repo_root / "component_system" + history = comp / "history" + state = history / "state" + seeds_dir = state / "seeds" + events_dir = state / "events" + runs_dir = state / "runs" + + for path in seeds_dir.glob("*.json"): + if path.stem not in preserved_seed_ids: + if dry_run: + print(f"[dry-run] remove {path}") + else: + path.unlink(missing_ok=True) + + for path in events_dir.glob("*.json"): + if path.stem not in preserved_seed_ids: + if dry_run: + print(f"[dry-run] remove {path}") + else: + path.unlink(missing_ok=True) + + for path in runs_dir.glob("*.json"): + data = _read_json(path, {}) + rid = data.get("run_id") if isinstance(data, dict) else None + if rid not in preserved_run_ids: + if dry_run: + print(f"[dry-run] remove {path}") + else: + path.unlink(missing_ok=True) + + +def _clean_queue_preserving_seeds( + repo_root: Path, preserved_seed_ids: set[str], dry_run: bool +) -> None: + history = repo_root / "component_system" / "history" / "queue" + stage_dirs = [ + history / "p", + history / "dca", + history / "direct", + history / "in_progress", + history / "done", + history / "error", + ] + for stage_dir in stage_dirs: + if not stage_dir.exists(): + continue + for path in stage_dir.glob("*.json"): + data = _read_json(path, {}) + task_seed = data.get("seed_id") if isinstance(data, dict) else None + if task_seed not in preserved_seed_ids: + if dry_run: + print(f"[dry-run] remove {path}") + else: + path.unlink(missing_ok=True) + + +def _clean_worktrees_preserving_seeds( + repo_root: Path, + preserved_seed_ids: set[str], + dry_run: bool, +) -> None: + worktrees_dir = repo_root / "component_system" / "history" / "worktrees" + if not worktrees_dir.exists(): + return + keep_names = preserved_seed_ids | {"baseline"} + for child in worktrees_dir.iterdir(): + if child.is_dir() and child.name not in keep_names: + if dry_run: + print(f"[dry-run] remove {child}") + else: + shutil.rmtree(child, ignore_errors=True) + + +def _clean_logs_preserving_seed( + repo_root: Path, + preserved_run_ids: set[str], + dry_run: bool, +) -> None: + logs_dir = repo_root / "component_system" / "history" / "logs" + if not logs_dir.exists(): + return + for path in logs_dir.iterdir(): + if not path.is_file(): + continue + # logs: {run_id}.stdout.log, {run_id}.stderr.log, {run_id}.prompt.txt + run_id = path.stem + if path.suffix in (".log", ".txt"): + run_id = run_id.rsplit(".", 1)[0] if "." in run_id else run_id + if run_id not in preserved_run_ids: + if dry_run: + print(f"[dry-run] remove {path}") + else: + path.unlink(missing_ok=True) + + +def _filter_baseline_jsons_preserving_seeds( + repo_root: Path, + preserved_seed_ids: set[str], + baseline_branches: set[str], + dry_run: bool, +) -> None: + comp = repo_root / "component_system" + branches_path = comp / "baseline_branches.json" + metrics_path = comp / "baseline_metrics.json" + + if branches_path.exists(): + data = _read_json(branches_path, {}) + if isinstance(data, dict): + new_data = {k: v for k, v in data.items() if k in preserved_seed_ids} + if dry_run: + print(f"[dry-run] write {branches_path} (keep {preserved_seed_ids})") + else: + _write_json(branches_path, new_data) + + if metrics_path.exists(): + data = _read_json(metrics_path, {}) + if isinstance(data, dict): + keep_branches = preserved_seed_ids | baseline_branches + new_data = {k: v for k, v in data.items() if k in keep_branches} + if dry_run: + print(f"[dry-run] write {metrics_path} (keep branches {keep_branches})") + else: + _write_json(metrics_path, new_data) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Clean local branches/worktrees and runtime history.") + parser.add_argument("--main-branch", default="master", help="Branch to keep. Default: master") + parser.add_argument( + "--preserve-seeds", + metavar="SEED_IDS", + help="Comma-separated seed IDs to keep (e.g. seed-a,seed-b). Keep their state, events, runs, queue, worktrees, branches, logs, baseline mappings; remove only other seeds.", + ) + parser.add_argument("--dry-run", action="store_true", help="Print actions without changing anything") + args = parser.parse_args() + + repo_root = Path.cwd().resolve() + print(f"Repository: {repo_root}") + + raw_preserve = getattr(args, "preserve_seeds", None) + preserve_seeds: list[str] = ( + [s.strip() for s in raw_preserve.split(",") if s.strip()] if raw_preserve else [] + ) + preserved_run_ids: set[str] = set() + baseline_branches: set[str] = set() + preserved_seed_ids: set[str] = set() + if preserve_seeds: + preserved_seed_ids = set(preserve_seeds) + print(f"Preserving everything for seeds: {', '.join(sorted(preserved_seed_ids))}") + preserved_run_ids, baseline_branches = _gather_preserved_seed_info(repo_root, preserve_seeds) + print(f" runs to keep: {len(preserved_run_ids)}") + + print("Verifying git repository...") + run_git(["rev-parse", "--is-inside-work-tree"], cwd=repo_root, dry_run=args.dry_run) + + print(f"Checking out '{args.main_branch}'...") + run_git(["checkout", args.main_branch], cwd=repo_root, dry_run=args.dry_run) + + print("Removing extra worktrees...") + run_git(["worktree", "prune"], cwd=repo_root, dry_run=args.dry_run) + wt_lines = run_git(["worktree", "list", "--porcelain"], cwd=repo_root, dry_run=args.dry_run) + worktrees: list[Path] = [] + for line in wt_lines: + if line.startswith("worktree "): + worktrees.append(Path(line[len("worktree ") :]).resolve()) + + branches_to_keep = {args.main_branch} | preserved_seed_ids | baseline_branches + worktree_keep_names = preserved_seed_ids | {"baseline"} if preserved_seed_ids else set() + + def is_clearable_branch(name: str) -> bool: + """Only branches matching seed-xxx or exactly __baseline__ may be cleared.""" + return name.startswith("seed-") or name == "__baseline__" + + for wt in worktrees: + if wt == repo_root: + continue + if worktree_keep_names and wt.name in worktree_keep_names: + print(f" - keeping worktree {wt} (preserved: {wt.name})") + continue + print(f" - removing worktree {wt}") + try: + run_git(["worktree", "remove", "--force", str(wt)], cwd=repo_root, dry_run=args.dry_run) + except RuntimeError as error: + if not is_broken_worktree_remove_error(error): + raise + print(f" ! stale/broken worktree metadata detected, deleting directory: {wt}") + if args.dry_run: + print(f"[dry-run] remove {wt}") + else: + shutil.rmtree(wt, ignore_errors=True) + run_git(["worktree", "prune"], cwd=repo_root, dry_run=args.dry_run) + + branches = run_git( + ["for-each-ref", "--format=%(refname:short)", "refs/heads"], + cwd=repo_root, + dry_run=args.dry_run, + ) + clearable = [b for b in branches if is_clearable_branch(b) and b not in branches_to_keep] + print(f"Deleting clearable branches (seed-* or __baseline__): {sorted(clearable)}") + for branch in clearable: + print(f" - deleting branch {branch}") + run_git(["branch", "-D", branch], cwd=repo_root, dry_run=args.dry_run) + + history_root = repo_root / "component_system" / "history" + if preserved_seed_ids: + print("Clearing component-system state (keeping preserved seeds)...") + _clean_state_preserving_seeds( + repo_root, preserved_seed_ids, preserved_run_ids, args.dry_run + ) + print("Clearing queue (keeping tasks for preserved seeds)...") + _clean_queue_preserving_seeds(repo_root, preserved_seed_ids, args.dry_run) + print("Clearing worktrees (keeping preserved seeds + baseline)...") + _clean_worktrees_preserving_seeds(repo_root, preserved_seed_ids, args.dry_run) + print("Clearing logs (keeping logs for preserved seed runs)...") + _clean_logs_preserving_seed(repo_root, preserved_run_ids, args.dry_run) + print("Filtering baseline_branches.json and baseline_metrics.json...") + _filter_baseline_jsons_preserving_seeds( + repo_root, preserved_seed_ids, baseline_branches, args.dry_run + ) + else: + print("Clearing component-system runtime/history artifacts...") + for name in ("state", "queue", "worktrees", "logs"): + remove_children(history_root / name, dry_run=args.dry_run) + + pytest_cache = repo_root / ".pytest_cache" + if pytest_cache.exists(): + if args.dry_run: + print(f"[dry-run] remove {pytest_cache}") + else: + shutil.rmtree(pytest_cache, ignore_errors=True) + + results_tsv = repo_root / "results.tsv" + if results_tsv.exists(): + if args.dry_run: + print(f"[dry-run] remove {results_tsv}") + else: + results_tsv.unlink(missing_ok=True) + + print("Removing __pycache__ directories...") + remove_pycache_dirs(repo_root, dry_run=args.dry_run) + + print("Done.") + print("Remaining branches:") + for branch in run_git(["branch", "--format=%(refname:short)"], cwd=repo_root, dry_run=args.dry_run): + print(f" {branch}") + + +if __name__ == "__main__": + main() diff --git a/uv.lock b/uv.lock index c840d62f..02a5e851 100644 --- a/uv.lock +++ b/uv.lock @@ -27,6 +27,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1e/d3/26bf1008eb3d2daa8ef4cacc7f3bfdc11818d111f7e2d0201bc6e3b49d45/annotated_doc-0.0.4-py3-none-any.whl", hash = "sha256:571ac1dc6991c450b25a9c2d84a3705e2ae7a53467b5d111c24fa8baabbed320", size = 5303, upload-time = "2025-11-10T22:07:40.673Z" }, ] +[[package]] +name = "annotated-types" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ee/67/531ea369ba64dcff5ec9c3402f9f51bf748cec26dde048a2f973a4eea7f5/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89", size = 16081, upload-time = "2024-05-20T21:33:25.928Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, +] + [[package]] name = "anyio" version = "4.12.1" @@ -41,11 +50,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/38/0e/27be9fdef66e72d64c0cdc3cc2823101b80585f8119b5c112c2e8f5f7dab/anyio-4.12.1-py3-none-any.whl", hash = "sha256:d405828884fc140aa80a3c667b8beed277f1dfedec42ba031bd6ac3db606ab6c", size = 113592, upload-time = "2026-01-06T11:45:19.497Z" }, ] +[[package]] +name = "arxiv" +version = "2.4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "feedparser" }, + { name = "requests" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/24/6e/647dd134e66d3ea6ff8aba2a177a37c74245625cfc58184e3aff99c8d8ec/arxiv-2.4.1.tar.gz", hash = "sha256:691606c1069bcca8316fcb082f5d15e65f1f24a021b0b87f01b9fa56347f63c8", size = 74975, upload-time = "2026-03-04T03:05:33.991Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3d/6a/297380dc42fa25dff095feda66d46f7abba77ba54579d079071a2459e8d3/arxiv-2.4.1-py3-none-any.whl", hash = "sha256:060d678410ffc224ada01089f877b7676f250e37f96c140bad6c287afadb15d8", size = 12106, upload-time = "2026-03-04T03:05:33.029Z" }, +] + [[package]] name = "autoresearch" version = "0.1.0" source = { virtual = "." } dependencies = [ + { name = "arxiv" }, + { name = "fastapi" }, + { name = "jinja2" }, { name = "kernels" }, { name = "matplotlib" }, { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, @@ -53,23 +79,30 @@ dependencies = [ { name = "pandas", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "pandas", version = "3.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, { name = "pyarrow" }, + { name = "python-multipart" }, { name = "requests" }, { name = "rustbpe" }, { name = "tiktoken" }, { name = "torch" }, + { name = "uvicorn" }, ] [package.metadata] requires-dist = [ + { name = "arxiv", specifier = ">=2.4.1" }, + { name = "fastapi", specifier = ">=0.116.0" }, + { name = "jinja2", specifier = ">=3.1.6" }, { name = "kernels", specifier = ">=0.11.7" }, { name = "matplotlib", specifier = ">=3.10.8" }, { name = "numpy", specifier = ">=2.2.6" }, { name = "pandas", specifier = ">=2.3.3" }, { name = "pyarrow", specifier = ">=21.0.0" }, + { name = "python-multipart", specifier = ">=0.0.20" }, { name = "requests", specifier = ">=2.32.0" }, { name = "rustbpe", specifier = ">=0.1.0" }, { name = "tiktoken", specifier = ">=0.11.0" }, { name = "torch", specifier = "==2.9.1", index = "https://download.pytorch.org/whl/cu128" }, + { name = "uvicorn", specifier = ">=0.35.0" }, ] [[package]] @@ -379,6 +412,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8a/0e/97c33bf5009bdbac74fd2beace167cab3f978feb69cc36f1ef79360d6c4e/exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598", size = 16740, upload-time = "2025-11-21T23:01:53.443Z" }, ] +[[package]] +name = "fastapi" +version = "0.135.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-doc" }, + { name = "pydantic" }, + { name = "starlette" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e7/7b/f8e0211e9380f7195ba3f3d40c292594fd81ba8ec4629e3854c353aaca45/fastapi-0.135.1.tar.gz", hash = "sha256:d04115b508d936d254cea545b7312ecaa58a7b3a0f84952535b4c9afae7668cd", size = 394962, upload-time = "2026-03-01T18:18:29.369Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e4/72/42e900510195b23a56bde950d26a51f8b723846bfcaa0286e90287f0422b/fastapi-0.135.1-py3-none-any.whl", hash = "sha256:46e2fc5745924b7c840f71ddd277382af29ce1cdb7d5eab5bf697e3fb9999c9e", size = 116999, upload-time = "2026-03-01T18:18:30.831Z" }, +] + +[[package]] +name = "feedparser" +version = "6.0.12" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "sgmllib3k" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/dc/79/db7edb5e77d6dfbc54d7d9df72828be4318275b2e580549ff45a962f6461/feedparser-6.0.12.tar.gz", hash = "sha256:64f76ce90ae3e8ef5d1ede0f8d3b50ce26bcce71dd8ae5e82b1cd2d4a5f94228", size = 286579, upload-time = "2025-09-10T13:33:59.486Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4e/eb/c96d64137e29ae17d83ad2552470bafe3a7a915e85434d9942077d7fd011/feedparser-6.0.12-py3-none-any.whl", hash = "sha256:6bbff10f5a52662c00a2e3f86a38928c37c48f77b3c511aedcd51de933549324", size = 81480, upload-time = "2025-09-10T13:33:58.022Z" }, +] + [[package]] name = "filelock" version = "3.24.3" @@ -1524,6 +1585,139 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/50/f2/c0e76a0b451ffdf0cf788932e182758eb7558953f4f27f1aff8e2518b653/pyarrow-23.0.1-cp314-cp314t-win_amd64.whl", hash = "sha256:527e8d899f14bd15b740cd5a54ad56b7f98044955373a17179d5956ddb93d9ce", size = 28365807, upload-time = "2026-02-16T10:14:03.892Z" }, ] +[[package]] +name = "pydantic" +version = "2.12.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/69/44/36f1a6e523abc58ae5f928898e4aca2e0ea509b5aa6f6f392a5d882be928/pydantic-2.12.5.tar.gz", hash = "sha256:4d351024c75c0f085a9febbb665ce8c0c6ec5d30e903bdb6394b7ede26aebb49", size = 821591, upload-time = "2025-11-26T15:11:46.471Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/87/b70ad306ebb6f9b585f114d0ac2137d792b48be34d732d60e597c2f8465a/pydantic-2.12.5-py3-none-any.whl", hash = "sha256:e561593fccf61e8a20fc46dfc2dfe075b8be7d0188df33f221ad1f0139180f9d", size = 463580, upload-time = "2025-11-26T15:11:44.605Z" }, +] + +[[package]] +name = "pydantic-core" +version = "2.41.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/71/70/23b021c950c2addd24ec408e9ab05d59b035b39d97cdc1130e1bce647bb6/pydantic_core-2.41.5.tar.gz", hash = "sha256:08daa51ea16ad373ffd5e7606252cc32f07bc72b28284b6bc9c6df804816476e", size = 460952, upload-time = "2025-11-04T13:43:49.098Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c6/90/32c9941e728d564b411d574d8ee0cf09b12ec978cb22b294995bae5549a5/pydantic_core-2.41.5-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:77b63866ca88d804225eaa4af3e664c5faf3568cea95360d21f4725ab6e07146", size = 2107298, upload-time = "2025-11-04T13:39:04.116Z" }, + { url = "https://files.pythonhosted.org/packages/fb/a8/61c96a77fe28993d9a6fb0f4127e05430a267b235a124545d79fea46dd65/pydantic_core-2.41.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:dfa8a0c812ac681395907e71e1274819dec685fec28273a28905df579ef137e2", size = 1901475, upload-time = "2025-11-04T13:39:06.055Z" }, + { url = "https://files.pythonhosted.org/packages/5d/b6/338abf60225acc18cdc08b4faef592d0310923d19a87fba1faf05af5346e/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5921a4d3ca3aee735d9fd163808f5e8dd6c6972101e4adbda9a4667908849b97", size = 1918815, upload-time = "2025-11-04T13:39:10.41Z" }, + { url = "https://files.pythonhosted.org/packages/d1/1c/2ed0433e682983d8e8cba9c8d8ef274d4791ec6a6f24c58935b90e780e0a/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e25c479382d26a2a41b7ebea1043564a937db462816ea07afa8a44c0866d52f9", size = 2065567, upload-time = "2025-11-04T13:39:12.244Z" }, + { url = "https://files.pythonhosted.org/packages/b3/24/cf84974ee7d6eae06b9e63289b7b8f6549d416b5c199ca2d7ce13bbcf619/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f547144f2966e1e16ae626d8ce72b4cfa0caedc7fa28052001c94fb2fcaa1c52", size = 2230442, upload-time = "2025-11-04T13:39:13.962Z" }, + { url = "https://files.pythonhosted.org/packages/fd/21/4e287865504b3edc0136c89c9c09431be326168b1eb7841911cbc877a995/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6f52298fbd394f9ed112d56f3d11aabd0d5bd27beb3084cc3d8ad069483b8941", size = 2350956, upload-time = "2025-11-04T13:39:15.889Z" }, + { url = "https://files.pythonhosted.org/packages/a8/76/7727ef2ffa4b62fcab916686a68a0426b9b790139720e1934e8ba797e238/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:100baa204bb412b74fe285fb0f3a385256dad1d1879f0a5cb1499ed2e83d132a", size = 2068253, upload-time = "2025-11-04T13:39:17.403Z" }, + { url = "https://files.pythonhosted.org/packages/d5/8c/a4abfc79604bcb4c748e18975c44f94f756f08fb04218d5cb87eb0d3a63e/pydantic_core-2.41.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:05a2c8852530ad2812cb7914dc61a1125dc4e06252ee98e5638a12da6cc6fb6c", size = 2177050, upload-time = "2025-11-04T13:39:19.351Z" }, + { url = "https://files.pythonhosted.org/packages/67/b1/de2e9a9a79b480f9cb0b6e8b6ba4c50b18d4e89852426364c66aa82bb7b3/pydantic_core-2.41.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:29452c56df2ed968d18d7e21f4ab0ac55e71dc59524872f6fc57dcf4a3249ed2", size = 2147178, upload-time = "2025-11-04T13:39:21Z" }, + { url = "https://files.pythonhosted.org/packages/16/c1/dfb33f837a47b20417500efaa0378adc6635b3c79e8369ff7a03c494b4ac/pydantic_core-2.41.5-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:d5160812ea7a8a2ffbe233d8da666880cad0cbaf5d4de74ae15c313213d62556", size = 2341833, upload-time = "2025-11-04T13:39:22.606Z" }, + { url = "https://files.pythonhosted.org/packages/47/36/00f398642a0f4b815a9a558c4f1dca1b4020a7d49562807d7bc9ff279a6c/pydantic_core-2.41.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:df3959765b553b9440adfd3c795617c352154e497a4eaf3752555cfb5da8fc49", size = 2321156, upload-time = "2025-11-04T13:39:25.843Z" }, + { url = "https://files.pythonhosted.org/packages/7e/70/cad3acd89fde2010807354d978725ae111ddf6d0ea46d1ea1775b5c1bd0c/pydantic_core-2.41.5-cp310-cp310-win32.whl", hash = "sha256:1f8d33a7f4d5a7889e60dc39856d76d09333d8a6ed0f5f1190635cbec70ec4ba", size = 1989378, upload-time = "2025-11-04T13:39:27.92Z" }, + { url = "https://files.pythonhosted.org/packages/76/92/d338652464c6c367e5608e4488201702cd1cbb0f33f7b6a85a60fe5f3720/pydantic_core-2.41.5-cp310-cp310-win_amd64.whl", hash = "sha256:62de39db01b8d593e45871af2af9e497295db8d73b085f6bfd0b18c83c70a8f9", size = 2013622, upload-time = "2025-11-04T13:39:29.848Z" }, + { url = "https://files.pythonhosted.org/packages/e8/72/74a989dd9f2084b3d9530b0915fdda64ac48831c30dbf7c72a41a5232db8/pydantic_core-2.41.5-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:a3a52f6156e73e7ccb0f8cced536adccb7042be67cb45f9562e12b319c119da6", size = 2105873, upload-time = "2025-11-04T13:39:31.373Z" }, + { url = "https://files.pythonhosted.org/packages/12/44/37e403fd9455708b3b942949e1d7febc02167662bf1a7da5b78ee1ea2842/pydantic_core-2.41.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7f3bf998340c6d4b0c9a2f02d6a400e51f123b59565d74dc60d252ce888c260b", size = 1899826, upload-time = "2025-11-04T13:39:32.897Z" }, + { url = "https://files.pythonhosted.org/packages/33/7f/1d5cab3ccf44c1935a359d51a8a2a9e1a654b744b5e7f80d41b88d501eec/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:378bec5c66998815d224c9ca994f1e14c0c21cb95d2f52b6021cc0b2a58f2a5a", size = 1917869, upload-time = "2025-11-04T13:39:34.469Z" }, + { url = "https://files.pythonhosted.org/packages/6e/6a/30d94a9674a7fe4f4744052ed6c5e083424510be1e93da5bc47569d11810/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e7b576130c69225432866fe2f4a469a85a54ade141d96fd396dffcf607b558f8", size = 2063890, upload-time = "2025-11-04T13:39:36.053Z" }, + { url = "https://files.pythonhosted.org/packages/50/be/76e5d46203fcb2750e542f32e6c371ffa9b8ad17364cf94bb0818dbfb50c/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6cb58b9c66f7e4179a2d5e0f849c48eff5c1fca560994d6eb6543abf955a149e", size = 2229740, upload-time = "2025-11-04T13:39:37.753Z" }, + { url = "https://files.pythonhosted.org/packages/d3/ee/fed784df0144793489f87db310a6bbf8118d7b630ed07aa180d6067e653a/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:88942d3a3dff3afc8288c21e565e476fc278902ae4d6d134f1eeda118cc830b1", size = 2350021, upload-time = "2025-11-04T13:39:40.94Z" }, + { url = "https://files.pythonhosted.org/packages/c8/be/8fed28dd0a180dca19e72c233cbf58efa36df055e5b9d90d64fd1740b828/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f31d95a179f8d64d90f6831d71fa93290893a33148d890ba15de25642c5d075b", size = 2066378, upload-time = "2025-11-04T13:39:42.523Z" }, + { url = "https://files.pythonhosted.org/packages/b0/3b/698cf8ae1d536a010e05121b4958b1257f0b5522085e335360e53a6b1c8b/pydantic_core-2.41.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c1df3d34aced70add6f867a8cf413e299177e0c22660cc767218373d0779487b", size = 2175761, upload-time = "2025-11-04T13:39:44.553Z" }, + { url = "https://files.pythonhosted.org/packages/b8/ba/15d537423939553116dea94ce02f9c31be0fa9d0b806d427e0308ec17145/pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:4009935984bd36bd2c774e13f9a09563ce8de4abaa7226f5108262fa3e637284", size = 2146303, upload-time = "2025-11-04T13:39:46.238Z" }, + { url = "https://files.pythonhosted.org/packages/58/7f/0de669bf37d206723795f9c90c82966726a2ab06c336deba4735b55af431/pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:34a64bc3441dc1213096a20fe27e8e128bd3ff89921706e83c0b1ac971276594", size = 2340355, upload-time = "2025-11-04T13:39:48.002Z" }, + { url = "https://files.pythonhosted.org/packages/e5/de/e7482c435b83d7e3c3ee5ee4451f6e8973cff0eb6007d2872ce6383f6398/pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c9e19dd6e28fdcaa5a1de679aec4141f691023916427ef9bae8584f9c2fb3b0e", size = 2319875, upload-time = "2025-11-04T13:39:49.705Z" }, + { url = "https://files.pythonhosted.org/packages/fe/e6/8c9e81bb6dd7560e33b9053351c29f30c8194b72f2d6932888581f503482/pydantic_core-2.41.5-cp311-cp311-win32.whl", hash = "sha256:2c010c6ded393148374c0f6f0bf89d206bf3217f201faa0635dcd56bd1520f6b", size = 1987549, upload-time = "2025-11-04T13:39:51.842Z" }, + { url = "https://files.pythonhosted.org/packages/11/66/f14d1d978ea94d1bc21fc98fcf570f9542fe55bfcc40269d4e1a21c19bf7/pydantic_core-2.41.5-cp311-cp311-win_amd64.whl", hash = "sha256:76ee27c6e9c7f16f47db7a94157112a2f3a00e958bc626e2f4ee8bec5c328fbe", size = 2011305, upload-time = "2025-11-04T13:39:53.485Z" }, + { url = "https://files.pythonhosted.org/packages/56/d8/0e271434e8efd03186c5386671328154ee349ff0354d83c74f5caaf096ed/pydantic_core-2.41.5-cp311-cp311-win_arm64.whl", hash = "sha256:4bc36bbc0b7584de96561184ad7f012478987882ebf9f9c389b23f432ea3d90f", size = 1972902, upload-time = "2025-11-04T13:39:56.488Z" }, + { url = "https://files.pythonhosted.org/packages/5f/5d/5f6c63eebb5afee93bcaae4ce9a898f3373ca23df3ccaef086d0233a35a7/pydantic_core-2.41.5-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f41a7489d32336dbf2199c8c0a215390a751c5b014c2c1c5366e817202e9cdf7", size = 2110990, upload-time = "2025-11-04T13:39:58.079Z" }, + { url = "https://files.pythonhosted.org/packages/aa/32/9c2e8ccb57c01111e0fd091f236c7b371c1bccea0fa85247ac55b1e2b6b6/pydantic_core-2.41.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:070259a8818988b9a84a449a2a7337c7f430a22acc0859c6b110aa7212a6d9c0", size = 1896003, upload-time = "2025-11-04T13:39:59.956Z" }, + { url = "https://files.pythonhosted.org/packages/68/b8/a01b53cb0e59139fbc9e4fda3e9724ede8de279097179be4ff31f1abb65a/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e96cea19e34778f8d59fe40775a7a574d95816eb150850a85a7a4c8f4b94ac69", size = 1919200, upload-time = "2025-11-04T13:40:02.241Z" }, + { url = "https://files.pythonhosted.org/packages/38/de/8c36b5198a29bdaade07b5985e80a233a5ac27137846f3bc2d3b40a47360/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ed2e99c456e3fadd05c991f8f437ef902e00eedf34320ba2b0842bd1c3ca3a75", size = 2052578, upload-time = "2025-11-04T13:40:04.401Z" }, + { url = "https://files.pythonhosted.org/packages/00/b5/0e8e4b5b081eac6cb3dbb7e60a65907549a1ce035a724368c330112adfdd/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:65840751b72fbfd82c3c640cff9284545342a4f1eb1586ad0636955b261b0b05", size = 2208504, upload-time = "2025-11-04T13:40:06.072Z" }, + { url = "https://files.pythonhosted.org/packages/77/56/87a61aad59c7c5b9dc8caad5a41a5545cba3810c3e828708b3d7404f6cef/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e536c98a7626a98feb2d3eaf75944ef6f3dbee447e1f841eae16f2f0a72d8ddc", size = 2335816, upload-time = "2025-11-04T13:40:07.835Z" }, + { url = "https://files.pythonhosted.org/packages/0d/76/941cc9f73529988688a665a5c0ecff1112b3d95ab48f81db5f7606f522d3/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eceb81a8d74f9267ef4081e246ffd6d129da5d87e37a77c9bde550cb04870c1c", size = 2075366, upload-time = "2025-11-04T13:40:09.804Z" }, + { url = "https://files.pythonhosted.org/packages/d3/43/ebef01f69baa07a482844faaa0a591bad1ef129253ffd0cdaa9d8a7f72d3/pydantic_core-2.41.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d38548150c39b74aeeb0ce8ee1d8e82696f4a4e16ddc6de7b1d8823f7de4b9b5", size = 2171698, upload-time = "2025-11-04T13:40:12.004Z" }, + { url = "https://files.pythonhosted.org/packages/b1/87/41f3202e4193e3bacfc2c065fab7706ebe81af46a83d3e27605029c1f5a6/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c23e27686783f60290e36827f9c626e63154b82b116d7fe9adba1fda36da706c", size = 2132603, upload-time = "2025-11-04T13:40:13.868Z" }, + { url = "https://files.pythonhosted.org/packages/49/7d/4c00df99cb12070b6bccdef4a195255e6020a550d572768d92cc54dba91a/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:482c982f814460eabe1d3bb0adfdc583387bd4691ef00b90575ca0d2b6fe2294", size = 2329591, upload-time = "2025-11-04T13:40:15.672Z" }, + { url = "https://files.pythonhosted.org/packages/cc/6a/ebf4b1d65d458f3cda6a7335d141305dfa19bdc61140a884d165a8a1bbc7/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:bfea2a5f0b4d8d43adf9d7b8bf019fb46fdd10a2e5cde477fbcb9d1fa08c68e1", size = 2319068, upload-time = "2025-11-04T13:40:17.532Z" }, + { url = "https://files.pythonhosted.org/packages/49/3b/774f2b5cd4192d5ab75870ce4381fd89cf218af999515baf07e7206753f0/pydantic_core-2.41.5-cp312-cp312-win32.whl", hash = "sha256:b74557b16e390ec12dca509bce9264c3bbd128f8a2c376eaa68003d7f327276d", size = 1985908, upload-time = "2025-11-04T13:40:19.309Z" }, + { url = "https://files.pythonhosted.org/packages/86/45/00173a033c801cacf67c190fef088789394feaf88a98a7035b0e40d53dc9/pydantic_core-2.41.5-cp312-cp312-win_amd64.whl", hash = "sha256:1962293292865bca8e54702b08a4f26da73adc83dd1fcf26fbc875b35d81c815", size = 2020145, upload-time = "2025-11-04T13:40:21.548Z" }, + { url = "https://files.pythonhosted.org/packages/f9/22/91fbc821fa6d261b376a3f73809f907cec5ca6025642c463d3488aad22fb/pydantic_core-2.41.5-cp312-cp312-win_arm64.whl", hash = "sha256:1746d4a3d9a794cacae06a5eaaccb4b8643a131d45fbc9af23e353dc0a5ba5c3", size = 1976179, upload-time = "2025-11-04T13:40:23.393Z" }, + { url = "https://files.pythonhosted.org/packages/87/06/8806241ff1f70d9939f9af039c6c35f2360cf16e93c2ca76f184e76b1564/pydantic_core-2.41.5-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:941103c9be18ac8daf7b7adca8228f8ed6bb7a1849020f643b3a14d15b1924d9", size = 2120403, upload-time = "2025-11-04T13:40:25.248Z" }, + { url = "https://files.pythonhosted.org/packages/94/02/abfa0e0bda67faa65fef1c84971c7e45928e108fe24333c81f3bfe35d5f5/pydantic_core-2.41.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:112e305c3314f40c93998e567879e887a3160bb8689ef3d2c04b6cc62c33ac34", size = 1896206, upload-time = "2025-11-04T13:40:27.099Z" }, + { url = "https://files.pythonhosted.org/packages/15/df/a4c740c0943e93e6500f9eb23f4ca7ec9bf71b19e608ae5b579678c8d02f/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cbaad15cb0c90aa221d43c00e77bb33c93e8d36e0bf74760cd00e732d10a6a0", size = 1919307, upload-time = "2025-11-04T13:40:29.806Z" }, + { url = "https://files.pythonhosted.org/packages/9a/e3/6324802931ae1d123528988e0e86587c2072ac2e5394b4bc2bc34b61ff6e/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:03ca43e12fab6023fc79d28ca6b39b05f794ad08ec2feccc59a339b02f2b3d33", size = 2063258, upload-time = "2025-11-04T13:40:33.544Z" }, + { url = "https://files.pythonhosted.org/packages/c9/d4/2230d7151d4957dd79c3044ea26346c148c98fbf0ee6ebd41056f2d62ab5/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dc799088c08fa04e43144b164feb0c13f9a0bc40503f8df3e9fde58a3c0c101e", size = 2214917, upload-time = "2025-11-04T13:40:35.479Z" }, + { url = "https://files.pythonhosted.org/packages/e6/9f/eaac5df17a3672fef0081b6c1bb0b82b33ee89aa5cec0d7b05f52fd4a1fa/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:97aeba56665b4c3235a0e52b2c2f5ae9cd071b8a8310ad27bddb3f7fb30e9aa2", size = 2332186, upload-time = "2025-11-04T13:40:37.436Z" }, + { url = "https://files.pythonhosted.org/packages/cf/4e/35a80cae583a37cf15604b44240e45c05e04e86f9cfd766623149297e971/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:406bf18d345822d6c21366031003612b9c77b3e29ffdb0f612367352aab7d586", size = 2073164, upload-time = "2025-11-04T13:40:40.289Z" }, + { url = "https://files.pythonhosted.org/packages/bf/e3/f6e262673c6140dd3305d144d032f7bd5f7497d3871c1428521f19f9efa2/pydantic_core-2.41.5-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b93590ae81f7010dbe380cdeab6f515902ebcbefe0b9327cc4804d74e93ae69d", size = 2179146, upload-time = "2025-11-04T13:40:42.809Z" }, + { url = "https://files.pythonhosted.org/packages/75/c7/20bd7fc05f0c6ea2056a4565c6f36f8968c0924f19b7d97bbfea55780e73/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:01a3d0ab748ee531f4ea6c3e48ad9dac84ddba4b0d82291f87248f2f9de8d740", size = 2137788, upload-time = "2025-11-04T13:40:44.752Z" }, + { url = "https://files.pythonhosted.org/packages/3a/8d/34318ef985c45196e004bc46c6eab2eda437e744c124ef0dbe1ff2c9d06b/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:6561e94ba9dacc9c61bce40e2d6bdc3bfaa0259d3ff36ace3b1e6901936d2e3e", size = 2340133, upload-time = "2025-11-04T13:40:46.66Z" }, + { url = "https://files.pythonhosted.org/packages/9c/59/013626bf8c78a5a5d9350d12e7697d3d4de951a75565496abd40ccd46bee/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:915c3d10f81bec3a74fbd4faebe8391013ba61e5a1a8d48c4455b923bdda7858", size = 2324852, upload-time = "2025-11-04T13:40:48.575Z" }, + { url = "https://files.pythonhosted.org/packages/1a/d9/c248c103856f807ef70c18a4f986693a46a8ffe1602e5d361485da502d20/pydantic_core-2.41.5-cp313-cp313-win32.whl", hash = "sha256:650ae77860b45cfa6e2cdafc42618ceafab3a2d9a3811fcfbd3bbf8ac3c40d36", size = 1994679, upload-time = "2025-11-04T13:40:50.619Z" }, + { url = "https://files.pythonhosted.org/packages/9e/8b/341991b158ddab181cff136acd2552c9f35bd30380422a639c0671e99a91/pydantic_core-2.41.5-cp313-cp313-win_amd64.whl", hash = "sha256:79ec52ec461e99e13791ec6508c722742ad745571f234ea6255bed38c6480f11", size = 2019766, upload-time = "2025-11-04T13:40:52.631Z" }, + { url = "https://files.pythonhosted.org/packages/73/7d/f2f9db34af103bea3e09735bb40b021788a5e834c81eedb541991badf8f5/pydantic_core-2.41.5-cp313-cp313-win_arm64.whl", hash = "sha256:3f84d5c1b4ab906093bdc1ff10484838aca54ef08de4afa9de0f5f14d69639cd", size = 1981005, upload-time = "2025-11-04T13:40:54.734Z" }, + { url = "https://files.pythonhosted.org/packages/ea/28/46b7c5c9635ae96ea0fbb779e271a38129df2550f763937659ee6c5dbc65/pydantic_core-2.41.5-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:3f37a19d7ebcdd20b96485056ba9e8b304e27d9904d233d7b1015db320e51f0a", size = 2119622, upload-time = "2025-11-04T13:40:56.68Z" }, + { url = "https://files.pythonhosted.org/packages/74/1a/145646e5687e8d9a1e8d09acb278c8535ebe9e972e1f162ed338a622f193/pydantic_core-2.41.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1d1d9764366c73f996edd17abb6d9d7649a7eb690006ab6adbda117717099b14", size = 1891725, upload-time = "2025-11-04T13:40:58.807Z" }, + { url = "https://files.pythonhosted.org/packages/23/04/e89c29e267b8060b40dca97bfc64a19b2a3cf99018167ea1677d96368273/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25e1c2af0fce638d5f1988b686f3b3ea8cd7de5f244ca147c777769e798a9cd1", size = 1915040, upload-time = "2025-11-04T13:41:00.853Z" }, + { url = "https://files.pythonhosted.org/packages/84/a3/15a82ac7bd97992a82257f777b3583d3e84bdb06ba6858f745daa2ec8a85/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:506d766a8727beef16b7adaeb8ee6217c64fc813646b424d0804d67c16eddb66", size = 2063691, upload-time = "2025-11-04T13:41:03.504Z" }, + { url = "https://files.pythonhosted.org/packages/74/9b/0046701313c6ef08c0c1cf0e028c67c770a4e1275ca73131563c5f2a310a/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4819fa52133c9aa3c387b3328f25c1facc356491e6135b459f1de698ff64d869", size = 2213897, upload-time = "2025-11-04T13:41:05.804Z" }, + { url = "https://files.pythonhosted.org/packages/8a/cd/6bac76ecd1b27e75a95ca3a9a559c643b3afcd2dd62086d4b7a32a18b169/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2b761d210c9ea91feda40d25b4efe82a1707da2ef62901466a42492c028553a2", size = 2333302, upload-time = "2025-11-04T13:41:07.809Z" }, + { url = "https://files.pythonhosted.org/packages/4c/d2/ef2074dc020dd6e109611a8be4449b98cd25e1b9b8a303c2f0fca2f2bcf7/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22f0fb8c1c583a3b6f24df2470833b40207e907b90c928cc8d3594b76f874375", size = 2064877, upload-time = "2025-11-04T13:41:09.827Z" }, + { url = "https://files.pythonhosted.org/packages/18/66/e9db17a9a763d72f03de903883c057b2592c09509ccfe468187f2a2eef29/pydantic_core-2.41.5-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2782c870e99878c634505236d81e5443092fba820f0373997ff75f90f68cd553", size = 2180680, upload-time = "2025-11-04T13:41:12.379Z" }, + { url = "https://files.pythonhosted.org/packages/d3/9e/3ce66cebb929f3ced22be85d4c2399b8e85b622db77dad36b73c5387f8f8/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:0177272f88ab8312479336e1d777f6b124537d47f2123f89cb37e0accea97f90", size = 2138960, upload-time = "2025-11-04T13:41:14.627Z" }, + { url = "https://files.pythonhosted.org/packages/a6/62/205a998f4327d2079326b01abee48e502ea739d174f0a89295c481a2272e/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:63510af5e38f8955b8ee5687740d6ebf7c2a0886d15a6d65c32814613681bc07", size = 2339102, upload-time = "2025-11-04T13:41:16.868Z" }, + { url = "https://files.pythonhosted.org/packages/3c/0d/f05e79471e889d74d3d88f5bd20d0ed189ad94c2423d81ff8d0000aab4ff/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:e56ba91f47764cc14f1daacd723e3e82d1a89d783f0f5afe9c364b8bb491ccdb", size = 2326039, upload-time = "2025-11-04T13:41:18.934Z" }, + { url = "https://files.pythonhosted.org/packages/ec/e1/e08a6208bb100da7e0c4b288eed624a703f4d129bde2da475721a80cab32/pydantic_core-2.41.5-cp314-cp314-win32.whl", hash = "sha256:aec5cf2fd867b4ff45b9959f8b20ea3993fc93e63c7363fe6851424c8a7e7c23", size = 1995126, upload-time = "2025-11-04T13:41:21.418Z" }, + { url = "https://files.pythonhosted.org/packages/48/5d/56ba7b24e9557f99c9237e29f5c09913c81eeb2f3217e40e922353668092/pydantic_core-2.41.5-cp314-cp314-win_amd64.whl", hash = "sha256:8e7c86f27c585ef37c35e56a96363ab8de4e549a95512445b85c96d3e2f7c1bf", size = 2015489, upload-time = "2025-11-04T13:41:24.076Z" }, + { url = "https://files.pythonhosted.org/packages/4e/bb/f7a190991ec9e3e0ba22e4993d8755bbc4a32925c0b5b42775c03e8148f9/pydantic_core-2.41.5-cp314-cp314-win_arm64.whl", hash = "sha256:e672ba74fbc2dc8eea59fb6d4aed6845e6905fc2a8afe93175d94a83ba2a01a0", size = 1977288, upload-time = "2025-11-04T13:41:26.33Z" }, + { url = "https://files.pythonhosted.org/packages/92/ed/77542d0c51538e32e15afe7899d79efce4b81eee631d99850edc2f5e9349/pydantic_core-2.41.5-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:8566def80554c3faa0e65ac30ab0932b9e3a5cd7f8323764303d468e5c37595a", size = 2120255, upload-time = "2025-11-04T13:41:28.569Z" }, + { url = "https://files.pythonhosted.org/packages/bb/3d/6913dde84d5be21e284439676168b28d8bbba5600d838b9dca99de0fad71/pydantic_core-2.41.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b80aa5095cd3109962a298ce14110ae16b8c1aece8b72f9dafe81cf597ad80b3", size = 1863760, upload-time = "2025-11-04T13:41:31.055Z" }, + { url = "https://files.pythonhosted.org/packages/5a/f0/e5e6b99d4191da102f2b0eb9687aaa7f5bea5d9964071a84effc3e40f997/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3006c3dd9ba34b0c094c544c6006cc79e87d8612999f1a5d43b769b89181f23c", size = 1878092, upload-time = "2025-11-04T13:41:33.21Z" }, + { url = "https://files.pythonhosted.org/packages/71/48/36fb760642d568925953bcc8116455513d6e34c4beaa37544118c36aba6d/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:72f6c8b11857a856bcfa48c86f5368439f74453563f951e473514579d44aa612", size = 2053385, upload-time = "2025-11-04T13:41:35.508Z" }, + { url = "https://files.pythonhosted.org/packages/20/25/92dc684dd8eb75a234bc1c764b4210cf2646479d54b47bf46061657292a8/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5cb1b2f9742240e4bb26b652a5aeb840aa4b417c7748b6f8387927bc6e45e40d", size = 2218832, upload-time = "2025-11-04T13:41:37.732Z" }, + { url = "https://files.pythonhosted.org/packages/e2/09/f53e0b05023d3e30357d82eb35835d0f6340ca344720a4599cd663dca599/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bd3d54f38609ff308209bd43acea66061494157703364ae40c951f83ba99a1a9", size = 2327585, upload-time = "2025-11-04T13:41:40Z" }, + { url = "https://files.pythonhosted.org/packages/aa/4e/2ae1aa85d6af35a39b236b1b1641de73f5a6ac4d5a7509f77b814885760c/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ff4321e56e879ee8d2a879501c8e469414d948f4aba74a2d4593184eb326660", size = 2041078, upload-time = "2025-11-04T13:41:42.323Z" }, + { url = "https://files.pythonhosted.org/packages/cd/13/2e215f17f0ef326fc72afe94776edb77525142c693767fc347ed6288728d/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d0d2568a8c11bf8225044aa94409e21da0cb09dcdafe9ecd10250b2baad531a9", size = 2173914, upload-time = "2025-11-04T13:41:45.221Z" }, + { url = "https://files.pythonhosted.org/packages/02/7a/f999a6dcbcd0e5660bc348a3991c8915ce6599f4f2c6ac22f01d7a10816c/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:a39455728aabd58ceabb03c90e12f71fd30fa69615760a075b9fec596456ccc3", size = 2129560, upload-time = "2025-11-04T13:41:47.474Z" }, + { url = "https://files.pythonhosted.org/packages/3a/b1/6c990ac65e3b4c079a4fb9f5b05f5b013afa0f4ed6780a3dd236d2cbdc64/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:239edca560d05757817c13dc17c50766136d21f7cd0fac50295499ae24f90fdf", size = 2329244, upload-time = "2025-11-04T13:41:49.992Z" }, + { url = "https://files.pythonhosted.org/packages/d9/02/3c562f3a51afd4d88fff8dffb1771b30cfdfd79befd9883ee094f5b6c0d8/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:2a5e06546e19f24c6a96a129142a75cee553cc018ffee48a460059b1185f4470", size = 2331955, upload-time = "2025-11-04T13:41:54.079Z" }, + { url = "https://files.pythonhosted.org/packages/5c/96/5fb7d8c3c17bc8c62fdb031c47d77a1af698f1d7a406b0f79aaa1338f9ad/pydantic_core-2.41.5-cp314-cp314t-win32.whl", hash = "sha256:b4ececa40ac28afa90871c2cc2b9ffd2ff0bf749380fbdf57d165fd23da353aa", size = 1988906, upload-time = "2025-11-04T13:41:56.606Z" }, + { url = "https://files.pythonhosted.org/packages/22/ed/182129d83032702912c2e2d8bbe33c036f342cc735737064668585dac28f/pydantic_core-2.41.5-cp314-cp314t-win_amd64.whl", hash = "sha256:80aa89cad80b32a912a65332f64a4450ed00966111b6615ca6816153d3585a8c", size = 1981607, upload-time = "2025-11-04T13:41:58.889Z" }, + { url = "https://files.pythonhosted.org/packages/9f/ed/068e41660b832bb0b1aa5b58011dea2a3fe0ba7861ff38c4d4904c1c1a99/pydantic_core-2.41.5-cp314-cp314t-win_arm64.whl", hash = "sha256:35b44f37a3199f771c3eaa53051bc8a70cd7b54f333531c59e29fd4db5d15008", size = 1974769, upload-time = "2025-11-04T13:42:01.186Z" }, + { url = "https://files.pythonhosted.org/packages/11/72/90fda5ee3b97e51c494938a4a44c3a35a9c96c19bba12372fb9c634d6f57/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:b96d5f26b05d03cc60f11a7761a5ded1741da411e7fe0909e27a5e6a0cb7b034", size = 2115441, upload-time = "2025-11-04T13:42:39.557Z" }, + { url = "https://files.pythonhosted.org/packages/1f/53/8942f884fa33f50794f119012dc6a1a02ac43a56407adaac20463df8e98f/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:634e8609e89ceecea15e2d61bc9ac3718caaaa71963717bf3c8f38bfde64242c", size = 1930291, upload-time = "2025-11-04T13:42:42.169Z" }, + { url = "https://files.pythonhosted.org/packages/79/c8/ecb9ed9cd942bce09fc888ee960b52654fbdbede4ba6c2d6e0d3b1d8b49c/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:93e8740d7503eb008aa2df04d3b9735f845d43ae845e6dcd2be0b55a2da43cd2", size = 1948632, upload-time = "2025-11-04T13:42:44.564Z" }, + { url = "https://files.pythonhosted.org/packages/2e/1b/687711069de7efa6af934e74f601e2a4307365e8fdc404703afc453eab26/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f15489ba13d61f670dcc96772e733aad1a6f9c429cc27574c6cdaed82d0146ad", size = 2138905, upload-time = "2025-11-04T13:42:47.156Z" }, + { url = "https://files.pythonhosted.org/packages/09/32/59b0c7e63e277fa7911c2fc70ccfb45ce4b98991e7ef37110663437005af/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:7da7087d756b19037bc2c06edc6c170eeef3c3bafcb8f532ff17d64dc427adfd", size = 2110495, upload-time = "2025-11-04T13:42:49.689Z" }, + { url = "https://files.pythonhosted.org/packages/aa/81/05e400037eaf55ad400bcd318c05bb345b57e708887f07ddb2d20e3f0e98/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:aabf5777b5c8ca26f7824cb4a120a740c9588ed58df9b2d196ce92fba42ff8dc", size = 1915388, upload-time = "2025-11-04T13:42:52.215Z" }, + { url = "https://files.pythonhosted.org/packages/6e/0d/e3549b2399f71d56476b77dbf3cf8937cec5cd70536bdc0e374a421d0599/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c007fe8a43d43b3969e8469004e9845944f1a80e6acd47c150856bb87f230c56", size = 1942879, upload-time = "2025-11-04T13:42:56.483Z" }, + { url = "https://files.pythonhosted.org/packages/f7/07/34573da085946b6a313d7c42f82f16e8920bfd730665de2d11c0c37a74b5/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76d0819de158cd855d1cbb8fcafdf6f5cf1eb8e470abe056d5d161106e38062b", size = 2139017, upload-time = "2025-11-04T13:42:59.471Z" }, + { url = "https://files.pythonhosted.org/packages/e6/b0/1a2aa41e3b5a4ba11420aba2d091b2d17959c8d1519ece3627c371951e73/pydantic_core-2.41.5-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:b5819cd790dbf0c5eb9f82c73c16b39a65dd6dd4d1439dcdea7816ec9adddab8", size = 2103351, upload-time = "2025-11-04T13:43:02.058Z" }, + { url = "https://files.pythonhosted.org/packages/a4/ee/31b1f0020baaf6d091c87900ae05c6aeae101fa4e188e1613c80e4f1ea31/pydantic_core-2.41.5-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:5a4e67afbc95fa5c34cf27d9089bca7fcab4e51e57278d710320a70b956d1b9a", size = 1925363, upload-time = "2025-11-04T13:43:05.159Z" }, + { url = "https://files.pythonhosted.org/packages/e1/89/ab8e86208467e467a80deaca4e434adac37b10a9d134cd2f99b28a01e483/pydantic_core-2.41.5-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ece5c59f0ce7d001e017643d8d24da587ea1f74f6993467d85ae8a5ef9d4f42b", size = 2135615, upload-time = "2025-11-04T13:43:08.116Z" }, + { url = "https://files.pythonhosted.org/packages/99/0a/99a53d06dd0348b2008f2f30884b34719c323f16c3be4e6cc1203b74a91d/pydantic_core-2.41.5-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:16f80f7abe3351f8ea6858914ddc8c77e02578544a0ebc15b4c2e1a0e813b0b2", size = 2175369, upload-time = "2025-11-04T13:43:12.49Z" }, + { url = "https://files.pythonhosted.org/packages/6d/94/30ca3b73c6d485b9bb0bc66e611cff4a7138ff9736b7e66bcf0852151636/pydantic_core-2.41.5-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:33cb885e759a705b426baada1fe68cbb0a2e68e34c5d0d0289a364cf01709093", size = 2144218, upload-time = "2025-11-04T13:43:15.431Z" }, + { url = "https://files.pythonhosted.org/packages/87/57/31b4f8e12680b739a91f472b5671294236b82586889ef764b5fbc6669238/pydantic_core-2.41.5-pp310-pypy310_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:c8d8b4eb992936023be7dee581270af5c6e0697a8559895f527f5b7105ecd36a", size = 2329951, upload-time = "2025-11-04T13:43:18.062Z" }, + { url = "https://files.pythonhosted.org/packages/7d/73/3c2c8edef77b8f7310e6fb012dbc4b8551386ed575b9eb6fb2506e28a7eb/pydantic_core-2.41.5-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:242a206cd0318f95cd21bdacff3fcc3aab23e79bba5cac3db5a841c9ef9c6963", size = 2318428, upload-time = "2025-11-04T13:43:20.679Z" }, + { url = "https://files.pythonhosted.org/packages/2f/02/8559b1f26ee0d502c74f9cca5c0d2fd97e967e083e006bbbb4e97f3a043a/pydantic_core-2.41.5-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d3a978c4f57a597908b7e697229d996d77a6d3c94901e9edee593adada95ce1a", size = 2147009, upload-time = "2025-11-04T13:43:23.286Z" }, + { url = "https://files.pythonhosted.org/packages/5f/9b/1b3f0e9f9305839d7e84912f9e8bfbd191ed1b1ef48083609f0dabde978c/pydantic_core-2.41.5-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:b2379fa7ed44ddecb5bfe4e48577d752db9fc10be00a6b7446e9663ba143de26", size = 2101980, upload-time = "2025-11-04T13:43:25.97Z" }, + { url = "https://files.pythonhosted.org/packages/a4/ed/d71fefcb4263df0da6a85b5d8a7508360f2f2e9b3bf5814be9c8bccdccc1/pydantic_core-2.41.5-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:266fb4cbf5e3cbd0b53669a6d1b039c45e3ce651fd5442eff4d07c2cc8d66808", size = 1923865, upload-time = "2025-11-04T13:43:28.763Z" }, + { url = "https://files.pythonhosted.org/packages/ce/3a/626b38db460d675f873e4444b4bb030453bbe7b4ba55df821d026a0493c4/pydantic_core-2.41.5-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58133647260ea01e4d0500089a8c4f07bd7aa6ce109682b1426394988d8aaacc", size = 2134256, upload-time = "2025-11-04T13:43:31.71Z" }, + { url = "https://files.pythonhosted.org/packages/83/d9/8412d7f06f616bbc053d30cb4e5f76786af3221462ad5eee1f202021eb4e/pydantic_core-2.41.5-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:287dad91cfb551c363dc62899a80e9e14da1f0e2b6ebde82c806612ca2a13ef1", size = 2174762, upload-time = "2025-11-04T13:43:34.744Z" }, + { url = "https://files.pythonhosted.org/packages/55/4c/162d906b8e3ba3a99354e20faa1b49a85206c47de97a639510a0e673f5da/pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:03b77d184b9eb40240ae9fd676ca364ce1085f203e1b1256f8ab9984dca80a84", size = 2143141, upload-time = "2025-11-04T13:43:37.701Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f2/f11dd73284122713f5f89fc940f370d035fa8e1e078d446b3313955157fe/pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:a668ce24de96165bb239160b3d854943128f4334822900534f2fe947930e5770", size = 2330317, upload-time = "2025-11-04T13:43:40.406Z" }, + { url = "https://files.pythonhosted.org/packages/88/9d/b06ca6acfe4abb296110fb1273a4d848a0bfb2ff65f3ee92127b3244e16b/pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f14f8f046c14563f8eb3f45f499cc658ab8d10072961e07225e507adb700e93f", size = 2316992, upload-time = "2025-11-04T13:43:43.602Z" }, + { url = "https://files.pythonhosted.org/packages/36/c7/cfc8e811f061c841d7990b0201912c3556bfeb99cdcb7ed24adc8d6f8704/pydantic_core-2.41.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:56121965f7a4dc965bff783d70b907ddf3d57f6eba29b6d2e5dabfaf07799c51", size = 2145302, upload-time = "2025-11-04T13:43:46.64Z" }, +] + [[package]] name = "pygments" version = "2.19.2" @@ -1554,6 +1748,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, ] +[[package]] +name = "python-multipart" +version = "0.0.22" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/01/979e98d542a70714b0cb2b6728ed0b7c46792b695e3eaec3e20711271ca3/python_multipart-0.0.22.tar.gz", hash = "sha256:7340bef99a7e0032613f56dc36027b959fd3b30a787ed62d310e951f7c3a3a58", size = 37612, upload-time = "2026-01-25T10:15:56.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1b/d0/397f9626e711ff749a95d96b7af99b9c566a9bb5129b8e4c10fc4d100304/python_multipart-0.0.22-py3-none-any.whl", hash = "sha256:2b2cd894c83d21bf49d702499531c7bafd057d730c201782048f7945d82de155", size = 24579, upload-time = "2026-01-25T10:15:54.811Z" }, +] + [[package]] name = "pytz" version = "2026.1.post1" @@ -1822,6 +2025,12 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e1/c6/76dc613121b793286a3f91621d7b75a2b493e0390ddca50f11993eadf192/setuptools-82.0.0-py3-none-any.whl", hash = "sha256:70b18734b607bd1da571d097d236cfcfacaf01de45717d59e6e04b96877532e0", size = 1003468, upload-time = "2026-02-08T15:08:38.723Z" }, ] +[[package]] +name = "sgmllib3k" +version = "1.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9e/bd/3704a8c3e0942d711c1299ebf7b9091930adae6675d7c8f476a7ce48653c/sgmllib3k-1.0.0.tar.gz", hash = "sha256:7868fb1c8bfa764c1ac563d3cf369c381d1325d36124933a726f29fcdaa812e9", size = 5750, upload-time = "2010-08-24T14:33:52.445Z" } + [[package]] name = "shellingham" version = "1.5.4" @@ -1840,6 +2049,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, ] +[[package]] +name = "starlette" +version = "0.52.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c4/68/79977123bb7be889ad680d79a40f339082c1978b5cfcf62c2d8d196873ac/starlette-0.52.1.tar.gz", hash = "sha256:834edd1b0a23167694292e94f597773bc3f89f362be6effee198165a35d62933", size = 2653702, upload-time = "2026-01-18T13:34:11.062Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/0d/13d1d239a25cbfb19e740db83143e95c772a1fe10202dda4b76792b114dd/starlette-0.52.1-py3-none-any.whl", hash = "sha256:0029d43eb3d273bc4f83a08720b4912ea4b071087a3b48db01b7c839f7954d74", size = 74272, upload-time = "2026-01-18T13:34:09.188Z" }, +] + [[package]] name = "sympy" version = "1.14.0" @@ -2078,6 +2300,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, ] +[[package]] +name = "typing-inspection" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949, upload-time = "2025-10-01T02:14:41.687Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, +] + [[package]] name = "tzdata" version = "2025.3" @@ -2095,3 +2329,17 @@ sdist = { url = "https://files.pythonhosted.org/packages/c7/24/5f1b3bdffd70275f6 wheels = [ { url = "https://files.pythonhosted.org/packages/39/08/aaaad47bc4e9dc8c725e68f9d04865dbcb2052843ff09c97b08904852d84/urllib3-2.6.3-py3-none-any.whl", hash = "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4", size = 131584, upload-time = "2026-01-07T16:24:42.685Z" }, ] + +[[package]] +name = "uvicorn" +version = "0.41.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "h11" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/32/ce/eeb58ae4ac36fe09e3842eb02e0eb676bf2c53ae062b98f1b2531673efdd/uvicorn-0.41.0.tar.gz", hash = "sha256:09d11cf7008da33113824ee5a1c6422d89fbc2ff476540d69a34c87fab8b571a", size = 82633, upload-time = "2026-02-16T23:07:24.1Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/83/e4/d04a086285c20886c0daad0e026f250869201013d18f81d9ff5eada73a88/uvicorn-0.41.0-py3-none-any.whl", hash = "sha256:29e35b1d2c36a04b9e180d4007ede3bcb32a85fbdfd6c6aeb3f26839de088187", size = 68783, upload-time = "2026-02-16T23:07:22.357Z" }, +]