diff --git a/.github/workflows/pr-task-eval.yml b/.github/workflows/pr-task-eval.yml new file mode 100644 index 0000000..1c911d7 --- /dev/null +++ b/.github/workflows/pr-task-eval.yml @@ -0,0 +1,236 @@ +name: All PR Task Evaluation + +on: + pull_request: + branches: [main] + workflow_dispatch: + inputs: + task_ids: + description: Comma-separated task ids (empty = all) + required: false + type: string + litellm_model: + description: Optional LITELLM_MODEL override + required: false + type: string + +permissions: + contents: read + pull-requests: write + +concurrency: + group: pr-task-eval-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + discover-tasks: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.discover.outputs.matrix }} + task_count: ${{ steps.discover.outputs.task_count }} + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Discover tasks + id: discover + env: + INPUT_TASK_IDS: ${{ github.event.inputs.task_ids }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + + tasks = sorted(p.parent.name for p in Path("data/task").glob("*/task_config.json")) + raw = (os.getenv("INPUT_TASK_IDS") or "").strip() + if raw: + requested = [t.strip() for t in raw.split(",") if t.strip()] + missing = sorted(set(requested) - set(tasks)) + if missing: + raise SystemExit(f"Unknown task_ids: {', '.join(missing)}") + tasks = requested + + matrix = json.dumps({"task_id": tasks}) + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as f: + f.write(f"matrix={matrix}\n") + f.write(f"task_count={len(tasks)}\n") + PY + + run-task: + runs-on: ubuntu-latest + needs: discover-tasks + if: ${{ needs.discover-tasks.outputs.task_count != '0' }} + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.discover-tasks.outputs.matrix) }} + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Set up uv + uses: astral-sh/setup-uv@v6 + - name: Sync dependencies + run: uv sync + - name: Evaluate task + id: evaluate + continue-on-error: true + env: + LITELLM_PROXY_API_KEY: ${{ secrets.LITELLM_PROXY_API_KEY }} + LITELLM_PROXY_API_BASE: ${{ vars.LITELLM_PROXY_API_BASE || secrets.LITELLM_PROXY_API_BASE }} + LITELLM_MODEL: ${{ github.event.inputs.litellm_model || vars.LITELLM_MODEL || secrets.LITELLM_MODEL }} + JUDGMENT_ORG_ID: ${{ secrets.JUDGMENT_ORG_ID }} + JUDGMENT_API_KEY: ${{ secrets.JUDGMENT_API_KEY }} + JUDGMENT_PROJECT_NAME: ${{ secrets.JUDGMENT_PROJECT_NAME }} + run: | + [ -n "$LITELLM_PROXY_API_KEY" ] || { echo "Missing LITELLM_PROXY_API_KEY"; exit 1; } + [ -n "$LITELLM_PROXY_API_BASE" ] || { echo "Missing LITELLM_PROXY_API_BASE"; exit 1; } + [ -n "$JUDGMENT_ORG_ID" ] || { echo "Missing JUDGMENT_ORG_ID"; exit 1; } + [ -n "$JUDGMENT_API_KEY" ] || { echo "Missing JUDGMENT_API_KEY"; exit 1; } + [ -n "$JUDGMENT_PROJECT_NAME" ] || { echo "Missing JUDGMENT_PROJECT_NAME"; exit 1; } + uv run python main.py run-one "${{ matrix.task_id }}" + - name: Build score artifact + if: ${{ always() }} + env: + TASK_ID: ${{ matrix.task_id }} + EVALUATE_OUTCOME: ${{ steps.evaluate.outcome }} + run: | + mkdir -p artifacts + python - <<'PY' + import json + import os + from pathlib import Path + + task_id = os.environ["TASK_ID"] + evaluate_outcome = os.environ["EVALUATE_OUTCOME"] + files = sorted(Path("results").glob(f"{task_id}_*.json"), key=lambda p: p.stat().st_mtime) + latest = files[-1] if files else None + payload = { + "task_id": task_id, + "status": "ok" if evaluate_outcome == "success" and latest else "error", + "score": 0, + "success": False, + "result_file": str(latest) if latest else None, + } + if latest: + data = json.loads(latest.read_text(encoding="utf-8")) + payload["score"] = float(data.get("score", 0)) + payload["success"] = bool(data.get("success", False)) + out = Path("artifacts") / f"score-{task_id}.json" + out.write_text(json.dumps(payload, indent=2), encoding="utf-8") + PY + - name: Upload score artifact + if: ${{ always() }} + uses: actions/upload-artifact@v4 + with: + name: score-${{ matrix.task_id }} + path: artifacts/score-${{ matrix.task_id }}.json + - name: Fail job if evaluation failed + if: ${{ steps.evaluate.outcome == 'failure' }} + run: exit 1 + + aggregate: + runs-on: ubuntu-latest + needs: [discover-tasks, run-task] + if: ${{ always() && needs.discover-tasks.outputs.task_count != '0' }} + steps: + - name: Download score artifacts + continue-on-error: true + uses: actions/download-artifact@v4 + with: + pattern: score-* + merge-multiple: true + path: artifacts + - name: Aggregate scores + run: | + python - <<'PY' + import json + import os + from pathlib import Path + + rows = [] + for p in sorted(Path("artifacts").glob("score-*.json")): + rows.append(json.loads(p.read_text(encoding="utf-8"))) + + total_tasks = len(rows) + total_score = sum(float(r.get("score", 0)) for r in rows) + average_score = (total_score / total_tasks) if total_tasks else 0.0 + success_count = sum(1 for r in rows if r.get("success")) + error_count = sum(1 for r in rows if r.get("status") != "ok") + + aggregate = { + "total_tasks": total_tasks, + "total_score": total_score, + "average_score": average_score, + "success_count": success_count, + "error_count": error_count, + "results": rows, + } + Path("aggregate.json").write_text(json.dumps(aggregate, indent=2), encoding="utf-8") + + lines = [ + "", + "## PR Task Evaluation", + "", + f"- Total tasks: {total_tasks}", + f"- Total score: {total_score:.2f}", + f"- Average score: {average_score:.2f}", + f"- Successful tasks: {success_count}", + f"- Error tasks: {error_count}", + "", + "| Task | Score | Success | Status |", + "|---|---:|:---:|:---:|", + ] + for r in rows: + success = "yes" if r.get("success") else "no" + status = r.get("status", "error") + lines.append(f"| {r['task_id']} | {float(r.get('score', 0)):.2f} | {success} | {status} |") + + summary = "\n".join(lines) + "\n" + Path("summary.md").write_text(summary, encoding="utf-8") + with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as f: + f.write(summary) + PY + - name: Upload aggregate artifact + uses: actions/upload-artifact@v4 + with: + name: aggregate-scores + path: | + aggregate.json + summary.md + - name: Upsert PR comment + if: ${{ github.event_name == 'pull_request' }} + uses: actions/github-script@v7 + with: + script: | + const fs = require("fs"); + const marker = ""; + const body = fs.readFileSync("summary.md", "utf8"); + const { owner, repo } = context.repo; + const issue_number = context.payload.pull_request.number; + const comments = await github.paginate(github.rest.issues.listComments, { + owner, + repo, + issue_number, + per_page: 100 + }); + const existing = comments.find(c => c.user?.type === "Bot" && c.body?.includes(marker)); + if (existing) { + await github.rest.issues.updateComment({ + owner, + repo, + comment_id: existing.id, + body + }); + } else { + await github.rest.issues.createComment({ + owner, + repo, + issue_number, + body + }); + } diff --git a/src/white_agent/agent.py b/src/white_agent/agent.py index 0dc0f7a..c9003e0 100644 --- a/src/white_agent/agent.py +++ b/src/white_agent/agent.py @@ -1,5 +1,4 @@ import os -import uuid import uvicorn from a2a.server.agent_execution import AgentExecutor, RequestContext @@ -38,9 +37,8 @@ def __init__(self, model: str | None = None): self._orchestrator = Orchestrator(model) async def execute(self, context: RequestContext, event_queue: EventQueue) -> None: - ctx_id = context.context_id or uuid.uuid4().hex - response = await self._orchestrator.handle(ctx_id, context.get_user_input()) - await event_queue.enqueue_event(new_agent_text_message(response, context_id=ctx_id)) + response = await self._orchestrator.handle(context.get_user_input()) + await event_queue.enqueue_event(new_agent_text_message(response)) async def cancel(self, context: RequestContext, event_queue: EventQueue) -> None: raise NotImplementedError diff --git a/src/white_agent/orchestrator.py b/src/white_agent/orchestrator.py index 48a1a52..d6cc66d 100644 --- a/src/white_agent/orchestrator.py +++ b/src/white_agent/orchestrator.py @@ -13,7 +13,7 @@ class Orchestrator: def __init__(self, model: str | None = None): - base_model = model or os.getenv("LITELLM_MODEL") or "openai/gpt-4o" + base_model = model or os.getenv("LITELLM_MODEL") or "litellm_proxy/claude-opus-4-6" pe_model = os.getenv("PE_MODEL", "litellm_proxy/claude-sonnet-4-5") self.planner = Planner(AgentConfig(model=base_model, total_turns=5, thinking_budget=10000)) @@ -22,7 +22,7 @@ def __init__(self, model: str | None = None): # Doesnt use total_turns self.pe = PromptEngineer(AgentConfig(model=pe_model, total_turns=0)) - self._states: dict[str, TaskState] = {} + self._state = TaskState() @staticmethod def _turns_left(state: TaskState) -> int: @@ -62,8 +62,8 @@ async def _run_verifier(self, state: TaskState, message: str) -> Action: return action @tracer.observe(span_name="Orchestrator.Handle") - async def handle(self, ctx_id: str, user_input: str) -> str: - state = self._states.setdefault(ctx_id, TaskState()) + async def handle(self, user_input: str) -> str: + state = self._state state.turn += 1 action: Action