feat: ci

.github/workflows/pr-task-eval.yml

@@ -0,0 +1,236 @@
+name: All PR Task Evaluation
+
+on:
+  pull_request:
+    branches: [main]
+  workflow_dispatch:
+    inputs:
+      task_ids:
+        description: Comma-separated task ids (empty = all)
+        required: false
+        type: string
+      litellm_model:
+        description: Optional LITELLM_MODEL override
+        required: false
+        type: string
+
+permissions:
+  contents: read
+  pull-requests: write
+
+concurrency:
+  group: pr-task-eval-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  discover-tasks:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.discover.outputs.matrix }}
+      task_count: ${{ steps.discover.outputs.task_count }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - name: Discover tasks
+        id: discover
+        env:
+          INPUT_TASK_IDS: ${{ github.event.inputs.task_ids }}
+        run: |
+          python - <<'PY'
+          import json
+          import os
+          from pathlib import Path
+
+          tasks = sorted(p.parent.name for p in Path("data/task").glob("*/task_config.json"))
+          raw = (os.getenv("INPUT_TASK_IDS") or "").strip()
+          if raw:
+              requested = [t.strip() for t in raw.split(",") if t.strip()]
+              missing = sorted(set(requested) - set(tasks))
+              if missing:
+                  raise SystemExit(f"Unknown task_ids: {', '.join(missing)}")
+              tasks = requested
+
+          matrix = json.dumps({"task_id": tasks})
+          with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as f:
+              f.write(f"matrix={matrix}\n")
+              f.write(f"task_count={len(tasks)}\n")
+          PY
+
+  run-task:
+    runs-on: ubuntu-latest
+    needs: discover-tasks
+    if: ${{ needs.discover-tasks.outputs.task_count != '0' }}
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.discover-tasks.outputs.matrix) }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v6
+      - name: Sync dependencies
+        run: uv sync
+      - name: Evaluate task
+        id: evaluate
+        continue-on-error: true
+        env:
+          LITELLM_PROXY_API_KEY: ${{ secrets.LITELLM_PROXY_API_KEY }}
+          LITELLM_PROXY_API_BASE: ${{ vars.LITELLM_PROXY_API_BASE || secrets.LITELLM_PROXY_API_BASE }}
+          LITELLM_MODEL: ${{ github.event.inputs.litellm_model || vars.LITELLM_MODEL || secrets.LITELLM_MODEL }}
+          JUDGMENT_ORG_ID: ${{ secrets.JUDGMENT_ORG_ID }}
+          JUDGMENT_API_KEY: ${{ secrets.JUDGMENT_API_KEY }}
+          JUDGMENT_PROJECT_NAME: ${{ secrets.JUDGMENT_PROJECT_NAME }}
+        run: |
+          [ -n "$LITELLM_PROXY_API_KEY" ] || { echo "Missing LITELLM_PROXY_API_KEY"; exit 1; }
+          [ -n "$LITELLM_PROXY_API_BASE" ] || { echo "Missing LITELLM_PROXY_API_BASE"; exit 1; }
+          [ -n "$JUDGMENT_ORG_ID" ] || { echo "Missing JUDGMENT_ORG_ID"; exit 1; }
+          [ -n "$JUDGMENT_API_KEY" ] || { echo "Missing JUDGMENT_API_KEY"; exit 1; }
+          [ -n "$JUDGMENT_PROJECT_NAME" ] || { echo "Missing JUDGMENT_PROJECT_NAME"; exit 1; }
+          uv run python main.py run-one "${{ matrix.task_id }}"
+      - name: Build score artifact
+        if: ${{ always() }}
+        env:
+          TASK_ID: ${{ matrix.task_id }}
+          EVALUATE_OUTCOME: ${{ steps.evaluate.outcome }}
+        run: |
+          mkdir -p artifacts
+          python - <<'PY'
+          import json
+          import os
+          from pathlib import Path
+
+          task_id = os.environ["TASK_ID"]
+          evaluate_outcome = os.environ["EVALUATE_OUTCOME"]
+          files = sorted(Path("results").glob(f"{task_id}_*.json"), key=lambda p: p.stat().st_mtime)
+          latest = files[-1] if files else None
+          payload = {
+              "task_id": task_id,
+              "status": "ok" if evaluate_outcome == "success" and latest else "error",
+              "score": 0,
+              "success": False,
+              "result_file": str(latest) if latest else None,
+          }
+          if latest:
+              data = json.loads(latest.read_text(encoding="utf-8"))
+              payload["score"] = float(data.get("score", 0))
+              payload["success"] = bool(data.get("success", False))
+          out = Path("artifacts") / f"score-{task_id}.json"
+          out.write_text(json.dumps(payload, indent=2), encoding="utf-8")
+          PY
+      - name: Upload score artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: score-${{ matrix.task_id }}
+          path: artifacts/score-${{ matrix.task_id }}.json
+      - name: Fail job if evaluation failed
+        if: ${{ steps.evaluate.outcome == 'failure' }}
+        run: exit 1
+
+  aggregate:
+    runs-on: ubuntu-latest
+    needs: [discover-tasks, run-task]
+    if: ${{ always() && needs.discover-tasks.outputs.task_count != '0' }}
+    steps:
+      - name: Download score artifacts
+        continue-on-error: true
+        uses: actions/download-artifact@v4
+        with:
+          pattern: score-*
+          merge-multiple: true
+          path: artifacts
+      - name: Aggregate scores
+        run: |
+          python - <<'PY'
+          import json
+          import os
+          from pathlib import Path
+
+          rows = []
+          for p in sorted(Path("artifacts").glob("score-*.json")):
+              rows.append(json.loads(p.read_text(encoding="utf-8")))
+
+          total_tasks = len(rows)
+          total_score = sum(float(r.get("score", 0)) for r in rows)
+          average_score = (total_score / total_tasks) if total_tasks else 0.0
+          success_count = sum(1 for r in rows if r.get("success"))
+          error_count = sum(1 for r in rows if r.get("status") != "ok")
+
+          aggregate = {
+              "total_tasks": total_tasks,
+              "total_score": total_score,
+              "average_score": average_score,
+              "success_count": success_count,
+              "error_count": error_count,
+              "results": rows,
+          }
+          Path("aggregate.json").write_text(json.dumps(aggregate, indent=2), encoding="utf-8")
+
+          lines = [
+              "<!-- pr-task-eval -->",
+              "## PR Task Evaluation",
+              "",
+              f"- Total tasks: {total_tasks}",
+              f"- Total score: {total_score:.2f}",
+              f"- Average score: {average_score:.2f}",
+              f"- Successful tasks: {success_count}",
+              f"- Error tasks: {error_count}",
+              "",
+              "| Task | Score | Success | Status |",
+              "|---|---:|:---:|:---:|",
+          ]
+          for r in rows:
+              success = "yes" if r.get("success") else "no"
+              status = r.get("status", "error")
+              lines.append(f"| {r['task_id']} | {float(r.get('score', 0)):.2f} | {success} | {status} |")
+
+          summary = "\n".join(lines) + "\n"
+          Path("summary.md").write_text(summary, encoding="utf-8")
+          with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as f:
+              f.write(summary)
+          PY
+      - name: Upload aggregate artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: aggregate-scores
+          path: |
+            aggregate.json
+            summary.md
+      - name: Upsert PR comment
+        if: ${{ github.event_name == 'pull_request' }}
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const fs = require("fs");
+            const marker = "<!-- pr-task-eval -->";
+            const body = fs.readFileSync("summary.md", "utf8");
+            const { owner, repo } = context.repo;
+            const issue_number = context.payload.pull_request.number;
+            const comments = await github.paginate(github.rest.issues.listComments, {
+              owner,
+              repo,
+              issue_number,
+              per_page: 100
+            });
+            const existing = comments.find(c => c.user?.type === "Bot" && c.body?.includes(marker));
+            if (existing) {
+              await github.rest.issues.updateComment({
+                owner,
+                repo,
+                comment_id: existing.id,
+                body
+              });
+            } else {
+              await github.rest.issues.createComment({
+                owner,
+                repo,
+                issue_number,
+                body
+              });
+            }

src/white_agent/agent.py

@@ -1,5 +1,4 @@
 import os
-import uuid
 
 import uvicorn
 from a2a.server.agent_execution import AgentExecutor, RequestContext
@@ -38,9 +37,8 @@ def __init__(self, model: str | None = None):
         self._orchestrator = Orchestrator(model)
 
     async def execute(self, context: RequestContext, event_queue: EventQueue) -> None:
-        ctx_id = context.context_id or uuid.uuid4().hex
-        response = await self._orchestrator.handle(ctx_id, context.get_user_input())
-        await event_queue.enqueue_event(new_agent_text_message(response, context_id=ctx_id))
+        response = await self._orchestrator.handle(context.get_user_input())
+        await event_queue.enqueue_event(new_agent_text_message(response))
 
     async def cancel(self, context: RequestContext, event_queue: EventQueue) -> None:
         raise NotImplementedError

src/white_agent/orchestrator.py

@@ -13,7 +13,7 @@
 
 class Orchestrator:
     def __init__(self, model: str | None = None):
-        base_model = model or os.getenv("LITELLM_MODEL") or "openai/gpt-4o"
+        base_model = model or os.getenv("LITELLM_MODEL") or "litellm_proxy/claude-opus-4-6"
         pe_model = os.getenv("PE_MODEL", "litellm_proxy/claude-sonnet-4-5")
 
         self.planner = Planner(AgentConfig(model=base_model, total_turns=5, thinking_budget=10000))
@@ -22,7 +22,7 @@ def __init__(self, model: str | None = None):
         # Doesnt use total_turns
         self.pe = PromptEngineer(AgentConfig(model=pe_model, total_turns=0))
 
-        self._states: dict[str, TaskState] = {}
+        self._state = TaskState()
 
     @staticmethod
     def _turns_left(state: TaskState) -> int:
@@ -62,8 +62,8 @@ async def _run_verifier(self, state: TaskState, message: str) -> Action:
         return action
 
     @tracer.observe(span_name="Orchestrator.Handle")
-    async def handle(self, ctx_id: str, user_input: str) -> str:
-        state = self._states.setdefault(ctx_id, TaskState())
+    async def handle(self, user_input: str) -> str:
+        state = self._state
         state.turn += 1
         action: Action