From 98a58e5e69be94b2d474c355eb6437d39143a9ba Mon Sep 17 00:00:00 2001 From: Abhishek Govindarasu Date: Fri, 20 Mar 2026 18:28:21 -0700 Subject: [PATCH 1/4] reverts and fixes --- .github/workflows/pr-task-eval.yml | 501 +++++++++++++++++++++++++++++ 1 file changed, 501 insertions(+) create mode 100644 .github/workflows/pr-task-eval.yml diff --git a/.github/workflows/pr-task-eval.yml b/.github/workflows/pr-task-eval.yml new file mode 100644 index 0000000..df9df12 --- /dev/null +++ b/.github/workflows/pr-task-eval.yml @@ -0,0 +1,501 @@ +name: PR Task Evaluation + +on: + pull_request: + branches: [main] + workflow_dispatch: + inputs: + task_ids: + description: Comma-separated task ids (empty = all) + required: false + type: string + max_parallel: + description: Matrix max parallel jobs + required: false + default: "6" + type: string + litellm_model: + description: Optional LITELLM_MODEL override + required: false + type: string + +permissions: + contents: read + pull-requests: write + +concurrency: + group: pr-task-eval-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + discover-tasks: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.discover.outputs.matrix }} + task_count: ${{ steps.discover.outputs.task_count }} + max_parallel: ${{ steps.discover.outputs.max_parallel }} + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Discover tasks + id: discover + env: + INPUT_TASK_IDS: ${{ github.event.inputs.task_ids }} + INPUT_MAX_PARALLEL: ${{ github.event.inputs.max_parallel }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + + tasks = sorted(p.parent.name for p in Path("data/task").glob("*/task_config.json")) + raw = (os.getenv("INPUT_TASK_IDS") or "").strip() + if raw: + requested = [t.strip() for t in raw.split(",") if t.strip()] + missing = sorted(set(requested) - set(tasks)) + if missing: + raise SystemExit(f"Unknown task_ids: {', '.join(missing)}") + tasks = requested + + max_parallel_raw = (os.getenv("INPUT_MAX_PARALLEL") or "6").strip() + max_parallel = int(max_parallel_raw) if max_parallel_raw else 6 + if max_parallel < 1: + raise SystemExit("max_parallel must be >= 1") + + matrix = json.dumps({"task_id": tasks}) + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as f: + f.write(f"matrix={matrix}\n") + f.write(f"task_count={len(tasks)}\n") + f.write(f"max_parallel={max_parallel}\n") + PY + + run-task: + runs-on: ubuntu-latest + needs: discover-tasks + if: ${{ needs.discover-tasks.outputs.task_count != '0' }} + strategy: + fail-fast: false + max-parallel: ${{ fromJSON(needs.discover-tasks.outputs.max_parallel) }} + matrix: ${{ fromJSON(needs.discover-tasks.outputs.matrix) }} + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Set up uv + uses: astral-sh/setup-uv@v6 + - name: Sync dependencies + run: uv sync + - name: Evaluate task + id: evaluate + continue-on-error: true + env: + LITELLM_PROXY_API_KEY: ${{ secrets.LITELLM_PROXY_API_KEY }} + LITELLM_PROXY_API_BASE: ${{ vars.LITELLM_PROXY_API_BASE || secrets.LITELLM_PROXY_API_BASE }} + LITELLM_MODEL: ${{ github.event.inputs.litellm_model || vars.LITELLM_MODEL || secrets.LITELLM_MODEL }} + JUDGMENT_ORG_ID: ${{ secrets.JUDGMENT_ORG_ID }} + JUDGMENT_API_KEY: ${{ secrets.JUDGMENT_API_KEY }} + JUDGMENT_PROJECT_NAME: ${{ secrets.JUDGMENT_PROJECT_NAME }} + run: | + [ -n "$LITELLM_PROXY_API_KEY" ] || { echo "Missing LITELLM_PROXY_API_KEY"; exit 1; } + [ -n "$LITELLM_PROXY_API_BASE" ] || { echo "Missing LITELLM_PROXY_API_BASE"; exit 1; } + [ -n "$JUDGMENT_ORG_ID" ] || { echo "Missing JUDGMENT_ORG_ID"; exit 1; } + [ -n "$JUDGMENT_API_KEY" ] || { echo "Missing JUDGMENT_API_KEY"; exit 1; } + [ -n "$JUDGMENT_PROJECT_NAME" ] || { echo "Missing JUDGMENT_PROJECT_NAME"; exit 1; } + uv run python main.py run-one "${{ matrix.task_id }}" + - name: Build score artifact + if: ${{ always() }} + env: + TASK_ID: ${{ matrix.task_id }} + EVALUATE_OUTCOME: ${{ steps.evaluate.outcome }} + run: | + mkdir -p artifacts + python - <<'PY' + import json + import os + from pathlib import Path + + task_id = os.environ["TASK_ID"] + evaluate_outcome = os.environ["EVALUATE_OUTCOME"] + files = sorted(Path("results").glob(f"{task_id}_*.json"), key=lambda p: p.stat().st_mtime) + latest = files[-1] if files else None + payload = { + "task_id": task_id, + "status": "ok" if evaluate_outcome == "success" and latest else "error", + "score": 0, + "success": False, + "result_file": str(latest) if latest else None, + } + if latest: + data = json.loads(latest.read_text(encoding="utf-8")) + payload["score"] = float(data.get("score", 0)) + payload["success"] = bool(data.get("success", False)) + out = Path("artifacts") / f"score-{task_id}.json" + out.write_text(json.dumps(payload, indent=2), encoding="utf-8") + PY + - name: Upload score artifact + if: ${{ always() }} + uses: actions/upload-artifact@v4 + with: + name: score-${{ matrix.task_id }} + path: artifacts/score-${{ matrix.task_id }}.json + - name: Fail job if evaluation failed + if: ${{ steps.evaluate.outcome == 'failure' }} + run: exit 1 + + aggregate: + runs-on: ubuntu-latest + needs: [discover-tasks, run-task] + if: ${{ always() && needs.discover-tasks.outputs.task_count != '0' }} + steps: + - name: Download score artifacts + continue-on-error: true + uses: actions/download-artifact@v4 + with: + pattern: score-* + merge-multiple: true + path: artifacts + - name: Aggregate scores + run: | + python - <<'PY' + import json + import os + from pathlib import Path + + rows = [] + for p in sorted(Path("artifacts").glob("score-*.json")): + rows.append(json.loads(p.read_text(encoding="utf-8"))) + + total_tasks = len(rows) + total_score = sum(float(r.get("score", 0)) for r in rows) + average_score = (total_score / total_tasks) if total_tasks else 0.0 + success_count = sum(1 for r in rows if r.get("success")) + error_count = sum(1 for r in rows if r.get("status") != "ok") + + aggregate = { + "total_tasks": total_tasks, + "total_score": total_score, + "average_score": average_score, + "success_count": success_count, + "error_count": error_count, + "results": rows, + } + Path("aggregate.json").write_text(json.dumps(aggregate, indent=2), encoding="utf-8") + + lines = [ + "", + "## PR Task Evaluation", + "", + f"- Total tasks: {total_tasks}", + f"- Total score: {total_score:.2f}", + f"- Average score: {average_score:.2f}", + f"- Successful tasks: {success_count}", + f"- Error tasks: {error_count}", + "", + "| Task | Score | Success | Status |", + "|---|---:|:---:|:---:|", + ] + for r in rows: + success = "yes" if r.get("success") else "no" + status = r.get("status", "error") + lines.append(f"| {r['task_id']} | {float(r.get('score', 0)):.2f} | {success} | {status} |") + + summary = "\n".join(lines) + "\n" + Path("summary.md").write_text(summary, encoding="utf-8") + with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as f: + f.write(summary) + PY + - name: Upload aggregate artifact + uses: actions/upload-artifact@v4 + with: + name: aggregate-scores + path: | + aggregate.json + summary.md + - name: Upsert PR comment + if: ${{ github.event_name == 'pull_request' }} + uses: actions/github-script@v7 + with: + script: | + const fs = require("fs"); + const marker = ""; + const body = fs.readFileSync("summary.md", "utf8"); + const { owner, repo } = context.repo; + const issue_number = context.payload.pull_request.number; + const comments = await github.paginate(github.rest.issues.listComments, { + owner, + repo, + issue_number, + per_page: 100 + }); + const existing = comments.find(c => c.user?.type === "Bot" && c.body?.includes(marker)); + if (existing) { + await github.rest.issues.updateComment({ + owner, + repo, + comment_id: existing.id, + body + }); + } else { + await github.rest.issues.createComment({ + owner, + repo, + issue_number, + body + }); + } +name: PR Task Evaluation + +on: + pull_request: + branches: [main] + workflow_dispatch: + inputs: + task_ids: + description: Comma-separated task ids (empty = all) + required: false + type: string + max_parallel: + description: Matrix max parallel jobs + required: false + default: "6" + type: string + litellm_model: + description: Optional LITELLM_MODEL override + required: false + type: string + +permissions: + contents: read + pull-requests: write + +concurrency: + group: pr-task-eval-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + discover-tasks: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.discover.outputs.matrix }} + task_count: ${{ steps.discover.outputs.task_count }} + max_parallel: ${{ steps.discover.outputs.max_parallel }} + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Discover tasks + id: discover + env: + EVENT_NAME: ${{ github.event_name }} + INPUT_TASK_IDS: ${{ github.event.inputs.task_ids }} + INPUT_MAX_PARALLEL: ${{ github.event.inputs.max_parallel }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + + tasks = sorted(p.parent.name for p in Path("data/task").glob("*/task_config.json")) + raw = (os.getenv("INPUT_TASK_IDS") or "").strip() + if raw: + requested = [t.strip() for t in raw.split(",") if t.strip()] + missing = sorted(set(requested) - set(tasks)) + if missing: + raise SystemExit(f"Unknown task_ids: {', '.join(missing)}") + tasks = requested + + max_parallel_raw = (os.getenv("INPUT_MAX_PARALLEL") or "6").strip() + max_parallel = int(max_parallel_raw) if max_parallel_raw else 6 + if max_parallel < 1: + raise SystemExit("max_parallel must be >= 1") + + matrix = json.dumps({"task_id": tasks}) + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as f: + f.write(f"matrix={matrix}\n") + f.write(f"task_count={len(tasks)}\n") + f.write(f"max_parallel={max_parallel}\n") + PY + + run-task: + runs-on: ubuntu-latest + needs: discover-tasks + if: ${{ needs.discover-tasks.outputs.task_count != '0' }} + strategy: + fail-fast: false + max-parallel: ${{ fromJSON(needs.discover-tasks.outputs.max_parallel) }} + matrix: ${{ fromJSON(needs.discover-tasks.outputs.matrix) }} + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Set up uv + uses: astral-sh/setup-uv@v6 + - name: Sync dependencies + run: uv sync + - name: Evaluate task + id: evaluate + continue-on-error: true + env: + LITELLM_PROXY_API_KEY: ${{ secrets.LITELLM_PROXY_API_KEY }} + LITELLM_PROXY_API_BASE: ${{ vars.LITELLM_PROXY_API_BASE || secrets.LITELLM_PROXY_API_BASE }} + LITELLM_MODEL: ${{ github.event.inputs.litellm_model || vars.LITELLM_MODEL || secrets.LITELLM_MODEL }} + JUDGMENT_ORG_ID: ${{ secrets.JUDGMENT_ORG_ID }} + JUDGMENT_API_KEY: ${{ secrets.JUDGMENT_API_KEY }} + JUDGMENT_PROJECT_NAME: ${{ secrets.JUDGMENT_PROJECT_NAME }} + run: | + [ -n "$LITELLM_PROXY_API_KEY" ] || { echo "Missing LITELLM_PROXY_API_KEY"; exit 1; } + [ -n "$LITELLM_PROXY_API_BASE" ] || { echo "Missing LITELLM_PROXY_API_BASE"; exit 1; } + [ -n "$JUDGMENT_ORG_ID" ] || { echo "Missing JUDGMENT_ORG_ID"; exit 1; } + [ -n "$JUDGMENT_API_KEY" ] || { echo "Missing JUDGMENT_API_KEY"; exit 1; } + [ -n "$JUDGMENT_PROJECT_NAME" ] || { echo "Missing JUDGMENT_PROJECT_NAME"; exit 1; } + uv run python main.py run-one "${{ matrix.task_id }}" + - name: Build score artifact + if: ${{ always() }} + env: + TASK_ID: ${{ matrix.task_id }} + EVALUATE_OUTCOME: ${{ steps.evaluate.outcome }} + run: | + mkdir -p artifacts + python - <<'PY' + import json + import os + from pathlib import Path + + task_id = os.environ["TASK_ID"] + evaluate_outcome = os.environ["EVALUATE_OUTCOME"] + files = sorted(Path("results").glob(f"{task_id}_*.json"), key=lambda p: p.stat().st_mtime) + latest = files[-1] if files else None + payload = { + "task_id": task_id, + "status": "ok" if evaluate_outcome == "success" and latest else "error", + "score": 0, + "success": False, + "result_file": str(latest) if latest else None, + } + if latest: + data = json.loads(latest.read_text(encoding="utf-8")) + payload["score"] = float(data.get("score", 0)) + payload["success"] = bool(data.get("success", False)) + out = Path("artifacts") / f"score-{task_id}.json" + out.write_text(json.dumps(payload, indent=2), encoding="utf-8") + PY + - name: Upload score artifact + if: ${{ always() }} + uses: actions/upload-artifact@v4 + with: + name: score-${{ matrix.task_id }} + path: artifacts/score-${{ matrix.task_id }}.json + - name: Fail job if evaluation failed + if: ${{ steps.evaluate.outcome == 'failure' }} + run: exit 1 + + aggregate: + runs-on: ubuntu-latest + needs: [discover-tasks, run-task] + if: ${{ always() && needs.discover-tasks.outputs.task_count != '0' }} + steps: + - name: Download score artifacts + continue-on-error: true + uses: actions/download-artifact@v4 + with: + pattern: score-* + merge-multiple: true + path: artifacts + - name: Aggregate scores + run: | + python - <<'PY' + import json + import os + from pathlib import Path + + rows = [] + for p in sorted(Path("artifacts").glob("score-*.json")): + rows.append(json.loads(p.read_text(encoding="utf-8"))) + + total_tasks = len(rows) + total_score = sum(float(r.get("score", 0)) for r in rows) + average_score = (total_score / total_tasks) if total_tasks else 0.0 + success_count = sum(1 for r in rows if r.get("success")) + error_count = sum(1 for r in rows if r.get("status") != "ok") + + aggregate = { + "total_tasks": total_tasks, + "total_score": total_score, + "average_score": average_score, + "success_count": success_count, + "error_count": error_count, + "results": rows, + } + Path("aggregate.json").write_text(json.dumps(aggregate, indent=2), encoding="utf-8") + + lines = [ + "", + "## PR Task Evaluation", + "", + f"- Total tasks: {total_tasks}", + f"- Total score: {total_score:.2f}", + f"- Average score: {average_score:.2f}", + f"- Successful tasks: {success_count}", + f"- Error tasks: {error_count}", + "", + "| Task | Score | Success | Status |", + "|---|---:|:---:|:---:|", + ] + for r in rows: + success = "yes" if r.get("success") else "no" + status = r.get("status", "error") + lines.append(f"| {r['task_id']} | {float(r.get('score', 0)):.2f} | {success} | {status} |") + + summary = "\n".join(lines) + "\n" + Path("summary.md").write_text(summary, encoding="utf-8") + with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as f: + f.write(summary) + PY + - name: Upload aggregate artifact + uses: actions/upload-artifact@v4 + with: + name: aggregate-scores + path: | + aggregate.json + summary.md + - name: Upsert PR comment + if: ${{ github.event_name == 'pull_request' }} + uses: actions/github-script@v7 + with: + script: | + const fs = require("fs"); + const marker = ""; + const body = fs.readFileSync("summary.md", "utf8"); + const { owner, repo } = context.repo; + const issue_number = context.payload.pull_request.number; + const comments = await github.paginate(github.rest.issues.listComments, { + owner, + repo, + issue_number, + per_page: 100 + }); + const existing = comments.find(c => c.user?.type === "Bot" && c.body?.includes(marker)); + if (existing) { + await github.rest.issues.updateComment({ + owner, + repo, + comment_id: existing.id, + body + }); + } else { + await github.rest.issues.createComment({ + owner, + repo, + issue_number, + body + }); + } From aed8eaeb31c02e514778ff98510f65fde117bae1 Mon Sep 17 00:00:00 2001 From: Abhishek Govindarasu Date: Fri, 20 Mar 2026 18:29:04 -0700 Subject: [PATCH 2/4] reverts and fixes --- .github/workflows/pr-task-eval.yml | 251 ----------------------------- 1 file changed, 251 deletions(-) diff --git a/.github/workflows/pr-task-eval.yml b/.github/workflows/pr-task-eval.yml index df9df12..8b18fc3 100644 --- a/.github/workflows/pr-task-eval.yml +++ b/.github/workflows/pr-task-eval.yml @@ -248,254 +248,3 @@ jobs: body }); } -name: PR Task Evaluation - -on: - pull_request: - branches: [main] - workflow_dispatch: - inputs: - task_ids: - description: Comma-separated task ids (empty = all) - required: false - type: string - max_parallel: - description: Matrix max parallel jobs - required: false - default: "6" - type: string - litellm_model: - description: Optional LITELLM_MODEL override - required: false - type: string - -permissions: - contents: read - pull-requests: write - -concurrency: - group: pr-task-eval-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - discover-tasks: - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.discover.outputs.matrix }} - task_count: ${{ steps.discover.outputs.task_count }} - max_parallel: ${{ steps.discover.outputs.max_parallel }} - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - name: Discover tasks - id: discover - env: - EVENT_NAME: ${{ github.event_name }} - INPUT_TASK_IDS: ${{ github.event.inputs.task_ids }} - INPUT_MAX_PARALLEL: ${{ github.event.inputs.max_parallel }} - run: | - python - <<'PY' - import json - import os - from pathlib import Path - - tasks = sorted(p.parent.name for p in Path("data/task").glob("*/task_config.json")) - raw = (os.getenv("INPUT_TASK_IDS") or "").strip() - if raw: - requested = [t.strip() for t in raw.split(",") if t.strip()] - missing = sorted(set(requested) - set(tasks)) - if missing: - raise SystemExit(f"Unknown task_ids: {', '.join(missing)}") - tasks = requested - - max_parallel_raw = (os.getenv("INPUT_MAX_PARALLEL") or "6").strip() - max_parallel = int(max_parallel_raw) if max_parallel_raw else 6 - if max_parallel < 1: - raise SystemExit("max_parallel must be >= 1") - - matrix = json.dumps({"task_id": tasks}) - with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as f: - f.write(f"matrix={matrix}\n") - f.write(f"task_count={len(tasks)}\n") - f.write(f"max_parallel={max_parallel}\n") - PY - - run-task: - runs-on: ubuntu-latest - needs: discover-tasks - if: ${{ needs.discover-tasks.outputs.task_count != '0' }} - strategy: - fail-fast: false - max-parallel: ${{ fromJSON(needs.discover-tasks.outputs.max_parallel) }} - matrix: ${{ fromJSON(needs.discover-tasks.outputs.matrix) }} - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - name: Set up uv - uses: astral-sh/setup-uv@v6 - - name: Sync dependencies - run: uv sync - - name: Evaluate task - id: evaluate - continue-on-error: true - env: - LITELLM_PROXY_API_KEY: ${{ secrets.LITELLM_PROXY_API_KEY }} - LITELLM_PROXY_API_BASE: ${{ vars.LITELLM_PROXY_API_BASE || secrets.LITELLM_PROXY_API_BASE }} - LITELLM_MODEL: ${{ github.event.inputs.litellm_model || vars.LITELLM_MODEL || secrets.LITELLM_MODEL }} - JUDGMENT_ORG_ID: ${{ secrets.JUDGMENT_ORG_ID }} - JUDGMENT_API_KEY: ${{ secrets.JUDGMENT_API_KEY }} - JUDGMENT_PROJECT_NAME: ${{ secrets.JUDGMENT_PROJECT_NAME }} - run: | - [ -n "$LITELLM_PROXY_API_KEY" ] || { echo "Missing LITELLM_PROXY_API_KEY"; exit 1; } - [ -n "$LITELLM_PROXY_API_BASE" ] || { echo "Missing LITELLM_PROXY_API_BASE"; exit 1; } - [ -n "$JUDGMENT_ORG_ID" ] || { echo "Missing JUDGMENT_ORG_ID"; exit 1; } - [ -n "$JUDGMENT_API_KEY" ] || { echo "Missing JUDGMENT_API_KEY"; exit 1; } - [ -n "$JUDGMENT_PROJECT_NAME" ] || { echo "Missing JUDGMENT_PROJECT_NAME"; exit 1; } - uv run python main.py run-one "${{ matrix.task_id }}" - - name: Build score artifact - if: ${{ always() }} - env: - TASK_ID: ${{ matrix.task_id }} - EVALUATE_OUTCOME: ${{ steps.evaluate.outcome }} - run: | - mkdir -p artifacts - python - <<'PY' - import json - import os - from pathlib import Path - - task_id = os.environ["TASK_ID"] - evaluate_outcome = os.environ["EVALUATE_OUTCOME"] - files = sorted(Path("results").glob(f"{task_id}_*.json"), key=lambda p: p.stat().st_mtime) - latest = files[-1] if files else None - payload = { - "task_id": task_id, - "status": "ok" if evaluate_outcome == "success" and latest else "error", - "score": 0, - "success": False, - "result_file": str(latest) if latest else None, - } - if latest: - data = json.loads(latest.read_text(encoding="utf-8")) - payload["score"] = float(data.get("score", 0)) - payload["success"] = bool(data.get("success", False)) - out = Path("artifacts") / f"score-{task_id}.json" - out.write_text(json.dumps(payload, indent=2), encoding="utf-8") - PY - - name: Upload score artifact - if: ${{ always() }} - uses: actions/upload-artifact@v4 - with: - name: score-${{ matrix.task_id }} - path: artifacts/score-${{ matrix.task_id }}.json - - name: Fail job if evaluation failed - if: ${{ steps.evaluate.outcome == 'failure' }} - run: exit 1 - - aggregate: - runs-on: ubuntu-latest - needs: [discover-tasks, run-task] - if: ${{ always() && needs.discover-tasks.outputs.task_count != '0' }} - steps: - - name: Download score artifacts - continue-on-error: true - uses: actions/download-artifact@v4 - with: - pattern: score-* - merge-multiple: true - path: artifacts - - name: Aggregate scores - run: | - python - <<'PY' - import json - import os - from pathlib import Path - - rows = [] - for p in sorted(Path("artifacts").glob("score-*.json")): - rows.append(json.loads(p.read_text(encoding="utf-8"))) - - total_tasks = len(rows) - total_score = sum(float(r.get("score", 0)) for r in rows) - average_score = (total_score / total_tasks) if total_tasks else 0.0 - success_count = sum(1 for r in rows if r.get("success")) - error_count = sum(1 for r in rows if r.get("status") != "ok") - - aggregate = { - "total_tasks": total_tasks, - "total_score": total_score, - "average_score": average_score, - "success_count": success_count, - "error_count": error_count, - "results": rows, - } - Path("aggregate.json").write_text(json.dumps(aggregate, indent=2), encoding="utf-8") - - lines = [ - "", - "## PR Task Evaluation", - "", - f"- Total tasks: {total_tasks}", - f"- Total score: {total_score:.2f}", - f"- Average score: {average_score:.2f}", - f"- Successful tasks: {success_count}", - f"- Error tasks: {error_count}", - "", - "| Task | Score | Success | Status |", - "|---|---:|:---:|:---:|", - ] - for r in rows: - success = "yes" if r.get("success") else "no" - status = r.get("status", "error") - lines.append(f"| {r['task_id']} | {float(r.get('score', 0)):.2f} | {success} | {status} |") - - summary = "\n".join(lines) + "\n" - Path("summary.md").write_text(summary, encoding="utf-8") - with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as f: - f.write(summary) - PY - - name: Upload aggregate artifact - uses: actions/upload-artifact@v4 - with: - name: aggregate-scores - path: | - aggregate.json - summary.md - - name: Upsert PR comment - if: ${{ github.event_name == 'pull_request' }} - uses: actions/github-script@v7 - with: - script: | - const fs = require("fs"); - const marker = ""; - const body = fs.readFileSync("summary.md", "utf8"); - const { owner, repo } = context.repo; - const issue_number = context.payload.pull_request.number; - const comments = await github.paginate(github.rest.issues.listComments, { - owner, - repo, - issue_number, - per_page: 100 - }); - const existing = comments.find(c => c.user?.type === "Bot" && c.body?.includes(marker)); - if (existing) { - await github.rest.issues.updateComment({ - owner, - repo, - comment_id: existing.id, - body - }); - } else { - await github.rest.issues.createComment({ - owner, - repo, - issue_number, - body - }); - } From 3563858e094f560ce2cb211ed63feec7569ea66e Mon Sep 17 00:00:00 2001 From: Abhishek Govindarasu Date: Fri, 20 Mar 2026 18:42:25 -0700 Subject: [PATCH 3/4] chore: update workflow --- .github/workflows/pr-task-eval.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-task-eval.yml b/.github/workflows/pr-task-eval.yml index 8b18fc3..5b5a2c6 100644 --- a/.github/workflows/pr-task-eval.yml +++ b/.github/workflows/pr-task-eval.yml @@ -1,4 +1,4 @@ -name: PR Task Evaluation +name: All PR Task Evaluation on: pull_request: From 363e7bd604da43a0f8929dad8195007cc96c7263 Mon Sep 17 00:00:00 2001 From: Abhishek Govindarasu Date: Fri, 20 Mar 2026 18:48:32 -0700 Subject: [PATCH 4/4] bump: ci --- .github/workflows/pr-task-eval.yml | 14 -------------- src/white_agent/agent.py | 6 ++---- src/white_agent/orchestrator.py | 8 ++++---- 3 files changed, 6 insertions(+), 22 deletions(-) diff --git a/.github/workflows/pr-task-eval.yml b/.github/workflows/pr-task-eval.yml index 5b5a2c6..1c911d7 100644 --- a/.github/workflows/pr-task-eval.yml +++ b/.github/workflows/pr-task-eval.yml @@ -9,11 +9,6 @@ on: description: Comma-separated task ids (empty = all) required: false type: string - max_parallel: - description: Matrix max parallel jobs - required: false - default: "6" - type: string litellm_model: description: Optional LITELLM_MODEL override required: false @@ -33,7 +28,6 @@ jobs: outputs: matrix: ${{ steps.discover.outputs.matrix }} task_count: ${{ steps.discover.outputs.task_count }} - max_parallel: ${{ steps.discover.outputs.max_parallel }} steps: - uses: actions/checkout@v4 - name: Set up Python @@ -44,7 +38,6 @@ jobs: id: discover env: INPUT_TASK_IDS: ${{ github.event.inputs.task_ids }} - INPUT_MAX_PARALLEL: ${{ github.event.inputs.max_parallel }} run: | python - <<'PY' import json @@ -60,16 +53,10 @@ jobs: raise SystemExit(f"Unknown task_ids: {', '.join(missing)}") tasks = requested - max_parallel_raw = (os.getenv("INPUT_MAX_PARALLEL") or "6").strip() - max_parallel = int(max_parallel_raw) if max_parallel_raw else 6 - if max_parallel < 1: - raise SystemExit("max_parallel must be >= 1") - matrix = json.dumps({"task_id": tasks}) with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as f: f.write(f"matrix={matrix}\n") f.write(f"task_count={len(tasks)}\n") - f.write(f"max_parallel={max_parallel}\n") PY run-task: @@ -78,7 +65,6 @@ jobs: if: ${{ needs.discover-tasks.outputs.task_count != '0' }} strategy: fail-fast: false - max-parallel: ${{ fromJSON(needs.discover-tasks.outputs.max_parallel) }} matrix: ${{ fromJSON(needs.discover-tasks.outputs.matrix) }} steps: - uses: actions/checkout@v4 diff --git a/src/white_agent/agent.py b/src/white_agent/agent.py index 0dc0f7a..c9003e0 100644 --- a/src/white_agent/agent.py +++ b/src/white_agent/agent.py @@ -1,5 +1,4 @@ import os -import uuid import uvicorn from a2a.server.agent_execution import AgentExecutor, RequestContext @@ -38,9 +37,8 @@ def __init__(self, model: str | None = None): self._orchestrator = Orchestrator(model) async def execute(self, context: RequestContext, event_queue: EventQueue) -> None: - ctx_id = context.context_id or uuid.uuid4().hex - response = await self._orchestrator.handle(ctx_id, context.get_user_input()) - await event_queue.enqueue_event(new_agent_text_message(response, context_id=ctx_id)) + response = await self._orchestrator.handle(context.get_user_input()) + await event_queue.enqueue_event(new_agent_text_message(response)) async def cancel(self, context: RequestContext, event_queue: EventQueue) -> None: raise NotImplementedError diff --git a/src/white_agent/orchestrator.py b/src/white_agent/orchestrator.py index 48a1a52..d6cc66d 100644 --- a/src/white_agent/orchestrator.py +++ b/src/white_agent/orchestrator.py @@ -13,7 +13,7 @@ class Orchestrator: def __init__(self, model: str | None = None): - base_model = model or os.getenv("LITELLM_MODEL") or "openai/gpt-4o" + base_model = model or os.getenv("LITELLM_MODEL") or "litellm_proxy/claude-opus-4-6" pe_model = os.getenv("PE_MODEL", "litellm_proxy/claude-sonnet-4-5") self.planner = Planner(AgentConfig(model=base_model, total_turns=5, thinking_budget=10000)) @@ -22,7 +22,7 @@ def __init__(self, model: str | None = None): # Doesnt use total_turns self.pe = PromptEngineer(AgentConfig(model=pe_model, total_turns=0)) - self._states: dict[str, TaskState] = {} + self._state = TaskState() @staticmethod def _turns_left(state: TaskState) -> int: @@ -62,8 +62,8 @@ async def _run_verifier(self, state: TaskState, message: str) -> Action: return action @tracer.observe(span_name="Orchestrator.Handle") - async def handle(self, ctx_id: str, user_input: str) -> str: - state = self._states.setdefault(ctx_id, TaskState()) + async def handle(self, user_input: str) -> str: + state = self._state state.turn += 1 action: Action