From 37310ea6a05c580215e6dc04449b99d523b9bc48 Mon Sep 17 00:00:00 2001 From: openhands Date: Mon, 8 Jun 2026 21:19:24 +0000 Subject: [PATCH 01/10] feat: agent-powered E2E test subset selection for PRs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add an LLM-based CI workflow that analyzes PR file changes and selects the most relevant mock-LLM E2E test specs to run, then triggers the E2E workflow with only that subset. Changes: - .github/workflows/mock-llm-e2e.yml: Add workflow_dispatch inputs (test_specs, test_grep, pr_number) so the workflow can be triggered with a filtered subset. Fully backward-compatible — pull_request events still run the full suite. Use EFFECTIVE_PR_NUMBER for comment/artifact steps to support both trigger types. - scripts/select-e2e-tests.py: Python script using OpenHands SDK LLM to intelligently map changed files to relevant specs, with a deterministic heuristic fallback when LLM is unavailable. - .github/workflows/agent-e2e-selector.yml: New workflow triggered by the 'smart-e2e' label or manual dispatch. Analyzes PR files, runs the selector, and dispatches mock-llm-e2e with the chosen subset. Posts a summary comment to the PR. Co-authored-by: openhands --- .github/workflows/agent-e2e-selector.yml | 174 ++++++++++++++ .github/workflows/mock-llm-e2e.yml | 69 +++++- scripts/select-e2e-tests.py | 283 +++++++++++++++++++++++ 3 files changed, 516 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/agent-e2e-selector.yml create mode 100644 scripts/select-e2e-tests.py diff --git a/.github/workflows/agent-e2e-selector.yml b/.github/workflows/agent-e2e-selector.yml new file mode 100644 index 000000000..d8f354130 --- /dev/null +++ b/.github/workflows/agent-e2e-selector.yml @@ -0,0 +1,174 @@ +# Agent-powered E2E test selector. +# +# Uses the OpenHands SDK LLM to analyze changed files in a PR and +# determine which mock-LLM E2E test specs are most relevant, then +# triggers the Mock-LLM E2E Tests workflow with only that subset. +# +# Trigger modes: +# - Label "smart-e2e" on a PR → runs automatically +# - Manual workflow_dispatch → provide a PR number +# +# This workflow is intentionally separate from the full-suite +# mock-llm-e2e.yml (which still fires on every PR push). Teams can +# use this as a faster feedback loop during development while keeping +# the full suite as a merge gate. + +name: "Agent E2E Selector" + +on: + pull_request: + types: [labeled] + workflow_dispatch: + inputs: + pr_number: + description: "PR number to analyze and run E2E subset for" + required: true + type: number + +permissions: + contents: read + pull-requests: write + actions: write # needed to trigger downstream workflow_dispatch + +concurrency: + group: agent-e2e-selector-${{ github.event.pull_request.number || inputs.pr_number }} + cancel-in-progress: true + +jobs: + select-and-dispatch: + # On label events, only run when the "smart-e2e" label is applied. + # On workflow_dispatch, always run. + if: >- + github.event_name == 'workflow_dispatch' || + (github.event_name == 'pull_request' && + github.event.label.name == 'smart-e2e' && + github.event.pull_request.head.repo.full_name == github.repository) + runs-on: ubuntu-24.04 + timeout-minutes: 5 + + env: + EFFECTIVE_PR: ${{ github.event.pull_request.number || inputs.pr_number }} + + steps: + - name: Check out repository + uses: actions/checkout@v6 + + - name: Read defaults from config/defaults.json + id: defaults + run: | + echo "agent_server_version=$(node -p "require('./config/defaults.json').versions.agentServer")" >> "$GITHUB_OUTPUT" + + - name: Install uv + run: | + curl -LsSf https://astral.sh/uv/install.sh | sh + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + + - name: Install openhands-sdk + env: + AGENT_SERVER_VERSION: ${{ steps.defaults.outputs.agent_server_version }} + run: | + uv venv .selector-venv + uv pip install -p .selector-venv "openhands-sdk==$AGENT_SERVER_VERSION" + + - name: Get changed files from PR + id: changed_files + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + FILES=$(gh api \ + "/repos/${{ github.repository }}/pulls/${{ env.EFFECTIVE_PR }}/files" \ + --paginate \ + --jq '.[].filename') + echo "$FILES" > changed_files.txt + echo "count=$(echo "$FILES" | wc -l)" >> "$GITHUB_OUTPUT" + echo "Changed files ($(echo "$FILES" | wc -l)):" + echo "$FILES" | head -50 + + - name: Run test selector + id: selector + env: + LLM_API_KEY: ${{ secrets.LLM_API_KEY }} + LLM_BASE_URL: ${{ vars.LIVE_E2E_LLM_BASE_URL || 'https://llm-proxy.app.all-hands.dev' }} + LLM_MODEL: "litellm_proxy/openai/gpt-4.1-mini" + run: | + RESULT=$(.selector-venv/bin/python3 scripts/select-e2e-tests.py < changed_files.txt) + echo "$RESULT" | python3 -m json.tool + echo "$RESULT" > selector_result.json + + # Parse outputs for downstream use. + SPECS=$(echo "$RESULT" | python3 -c "import sys,json; d=json.load(sys.stdin); print(','.join(d.get('specs',[])))") + REASON=$(echo "$RESULT" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('reason',''))") + MODE=$(echo "$RESULT" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('mode','full'))") + + echo "specs=$SPECS" >> "$GITHUB_OUTPUT" + echo "reason=$REASON" >> "$GITHUB_OUTPUT" + echo "mode=$MODE" >> "$GITHUB_OUTPUT" + + - name: Resolve PR head ref + id: pr_ref + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + REF=$(gh api "/repos/${{ github.repository }}/pulls/${{ env.EFFECTIVE_PR }}" --jq '.head.ref') + echo "ref=$REF" >> "$GITHUB_OUTPUT" + echo "PR head ref: $REF" + + - name: Trigger Mock-LLM E2E Tests + env: + GITHUB_TOKEN: ${{ secrets.OPENHANDS_BOT_GITHUB_PAT_PUBLIC || github.token }} + run: | + SPECS="${{ steps.selector.outputs.specs }}" + MODE="${{ steps.selector.outputs.mode }}" + REF="${{ steps.pr_ref.outputs.ref }}" + PR="${{ env.EFFECTIVE_PR }}" + + echo "Mode: $MODE" + echo "Specs: ${SPECS:-}" + echo "Dispatching on ref: $REF" + + gh workflow run "Mock-LLM E2E Tests" \ + --ref "$REF" \ + -f test_specs="$SPECS" \ + -f pr_number="$PR" + + echo "::notice::Dispatched Mock-LLM E2E Tests (mode=$MODE, specs=${SPECS:-all})" + + - name: Post selection summary to PR + continue-on-error: true + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + MODE="${{ steps.selector.outputs.mode }}" + SPECS="${{ steps.selector.outputs.specs }}" + REASON="${{ steps.selector.outputs.reason }}" + FILE_COUNT="${{ steps.changed_files.outputs.count }}" + + if [ "$MODE" = "full" ]; then + SPEC_DISPLAY="**Full suite** (all specs)" + else + SPEC_DISPLAY=$(echo "$SPECS" | tr ',' '\n' | sed 's/^/- `/' | sed 's/$/`/') + fi + + BODY=$(cat <<'COMMENT_EOF' + + ### 🤖 Agent E2E Test Selector + + | | | + |---|---| + | **Mode** | `MODE_PLACEHOLDER` | + | **Files analyzed** | FILE_COUNT_PLACEHOLDER | + | **Reason** | REASON_PLACEHOLDER | + + **Selected specs:** + SPECS_PLACEHOLDER + + _Triggered run: [Mock-LLM E2E Tests](/${{ github.repository }}/actions/workflows/mock-llm-e2e.yml)_ + COMMENT_EOF + ) + + BODY="${BODY//MODE_PLACEHOLDER/$MODE}" + BODY="${BODY//FILE_COUNT_PLACEHOLDER/$FILE_COUNT}" + BODY="${BODY//REASON_PLACEHOLDER/$REASON}" + BODY="${BODY//SPECS_PLACEHOLDER/$SPEC_DISPLAY}" + + echo "$BODY" | gh pr comment "${{ env.EFFECTIVE_PR }}" --body-file - diff --git a/.github/workflows/mock-llm-e2e.yml b/.github/workflows/mock-llm-e2e.yml index 11c2dd244..52f9ea8c7 100644 --- a/.github/workflows/mock-llm-e2e.yml +++ b/.github/workflows/mock-llm-e2e.yml @@ -4,9 +4,30 @@ on: pull_request: types: [opened, synchronize, reopened] workflow_dispatch: + inputs: + test_specs: + description: >- + Comma-separated spec filenames to run (e.g. + "mock-llm-conversation.spec.ts,mock-llm-automation.spec.ts"). + Leave empty to run the full suite. + required: false + default: "" + test_grep: + description: >- + Playwright --grep pattern to filter tests by title + (e.g. "conversation|automation"). Leave empty for no filter. + required: false + default: "" + pr_number: + description: >- + PR number for comment posting when triggered externally + (e.g. by the agent-e2e-selector workflow). Leave empty for + non-PR dispatches. + required: false + default: "" concurrency: - group: mock-llm-e2e-${{ github.event.pull_request.number || github.ref }} + group: mock-llm-e2e-${{ github.event.pull_request.number || inputs.pr_number || github.ref }} cancel-in-progress: true permissions: @@ -21,6 +42,8 @@ jobs: env: MOCK_LLM_REPORT_PATH: mock-llm-report.md MOCK_LLM_WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + # Resolve the effective PR number from either trigger type. + EFFECTIVE_PR_NUMBER: ${{ github.event.pull_request.number || inputs.pr_number || '' }} steps: - name: Check out repository @@ -106,6 +129,31 @@ jobs: - name: Build frontend (for agent-canvas binary) run: npm run build:app + - name: Build Playwright command + id: pw_command + run: | + CMD="npx playwright test --config=playwright.mock-llm.config.ts" + + # Append spec file filters from workflow_dispatch input. + TEST_SPECS="${{ inputs.test_specs }}" + if [ -n "$TEST_SPECS" ]; then + # Convert comma-separated filenames to space-separated paths. + for spec in $(echo "$TEST_SPECS" | tr ',' ' '); do + CMD="$CMD tests/e2e/mock-llm/$spec" + done + echo "::notice::Running subset: $TEST_SPECS" + fi + + # Append --grep filter from workflow_dispatch input. + TEST_GREP="${{ inputs.test_grep }}" + if [ -n "$TEST_GREP" ]; then + CMD="$CMD --grep \"$TEST_GREP\"" + echo "::notice::Grep filter: $TEST_GREP" + fi + + echo "command=$CMD" >> "$GITHUB_OUTPUT" + echo "Playwright command: $CMD" + - name: Run mock-LLM E2E tests id: run_tests env: @@ -119,7 +167,7 @@ jobs: # Run Playwright in background so our shell survives if we have # to kill it (the webServer teardown can hang indefinitely). - npm run test:e2e:mock-llm & + ${{ steps.pw_command.outputs.command }} & PW_PID=$! # Wait for tests to complete. Playwright's globalTimeout is 600s @@ -187,7 +235,7 @@ jobs: test-results-mock-llm/ - name: Detect newly added spec files - if: always() && github.event.pull_request.number + if: always() && env.EFFECTIVE_PR_NUMBER id: new_specs env: GITHUB_TOKEN: ${{ github.token }} @@ -195,7 +243,7 @@ jobs: # Find mock-LLM spec files added (not just modified) in this PR # Uses the GitHub API instead of git diff to avoid shallow-clone issues NEW_FILES=$(gh api \ - "/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/files" \ + "/repos/${{ github.repository }}/pulls/${{ env.EFFECTIVE_PR_NUMBER }}/files" \ --paginate \ --jq '[.[] | select(.status == "added") | .filename | select(test("tests/e2e/mock-llm/.*\\.spec\\.ts$"))] @@ -215,11 +263,11 @@ jobs: cat "$MOCK_LLM_REPORT_PATH" >> "$GITHUB_STEP_SUMMARY" - name: Save PR number for comment workflow - if: always() && github.event.pull_request.number - run: echo "${{ github.event.pull_request.number }}" > pr_number.txt + if: always() && env.EFFECTIVE_PR_NUMBER + run: echo "${{ env.EFFECTIVE_PR_NUMBER }}" > pr_number.txt - name: Upload PR comment payload - if: always() && github.event.pull_request.number + if: always() && env.EFFECTIVE_PR_NUMBER uses: actions/upload-artifact@v7 with: name: mock-llm-pr-comment-payload @@ -231,13 +279,14 @@ jobs: - name: Post PR comment (same-repo PRs only) if: >- always() && - github.event.pull_request.number && - github.event.pull_request.head.repo.full_name == github.repository + env.EFFECTIVE_PR_NUMBER && + (github.event.pull_request.head.repo.full_name == github.repository || + github.event_name == 'workflow_dispatch') continue-on-error: true env: GITHUB_TOKEN: ${{ github.token }} run: | - gh pr comment "${{ github.event.pull_request.number }}" \ + gh pr comment "${{ env.EFFECTIVE_PR_NUMBER }}" \ --body-file "$MOCK_LLM_REPORT_PATH" - name: Fail job when tests fail diff --git a/scripts/select-e2e-tests.py b/scripts/select-e2e-tests.py new file mode 100644 index 000000000..eff93a075 --- /dev/null +++ b/scripts/select-e2e-tests.py @@ -0,0 +1,283 @@ +#!/usr/bin/env python3 +"""Select which mock-LLM E2E test specs to run based on changed files. + +Uses the OpenHands SDK's LLM class to intelligently map PR file changes +to the most relevant test specs. Falls back to running the full suite +when the LLM is unavailable or when the change set is too broad. + +Usage: + # Pipe changed files (one per line): + git diff --name-only origin/main | python scripts/select-e2e-tests.py + + # Or pass as arguments: + python scripts/select-e2e-tests.py src/routes/automations.tsx src/api/automation-service/... + +Environment variables: + LLM_API_KEY – required (unless --fallback-only) + LLM_BASE_URL – optional, defaults to https://llm-proxy.app.all-hands.dev + LLM_MODEL – optional, defaults to litellm_proxy/openai/gpt-4.1-mini + +Output (stdout): JSON object with keys: + specs – list of spec filenames to run (empty ⇒ full suite) + reason – human-readable explanation + mode – "llm" | "heuristic" | "full" +""" + +from __future__ import annotations + +import json +import os +import sys + +# --------------------------------------------------------------------------- +# Spec catalog – each entry maps a spec filename to a brief description +# of what source areas it exercises. The LLM receives this catalog so it +# can reason about coverage. +# --------------------------------------------------------------------------- +SPEC_CATALOG: dict[str, str] = { + "mock-llm-acp-agent.spec.ts": ( + "ACP (Agent Client Protocol) agent configuration via Settings UI, " + "ACP conversation lifecycle, agent_kind=acp payload." + ), + "mock-llm-auth-modes.spec.ts": ( + "Session API key injection, key rotation recovery, public-mode " + "auth gate (ApiKeyEntryScreen), localStorage key sync." + ), + "mock-llm-automation.spec.ts": ( + "Full automation lifecycle: create cron automation via terminal " + "curl, dispatch a run, verify automation list/detail pages, " + "automation backend integration." + ), + "mock-llm-conversation.spec.ts": ( + "Core conversation flow: LLM profile creation, settings API, " + "terminal tool call, bash execution, agent reply, sidebar resume." + ), + "mock-llm-cross-connect.spec.ts": ( + "Frontend-only ↔ backend-only cross-connect, multi-backend " + "switching, manage-backends modal, backend registry." + ), + "mock-llm-image-upload.spec.ts": ( + "Image attachment via file input, base64 encoding in LLM " + "completion payload, image_urls in user message event." + ), + "mock-llm-model-switch.spec.ts": ( + "/model slash command mid-conversation, LLM profile switching, " + "switchLLM API, chat header profile display." + ), + "mock-llm-onboarding-happy-path.spec.ts": ( + "Full onboarding wizard: agent selection, backend check, LLM " + "setup, hello message. OnboardingModal flow." + ), + "mock-llm-onboarding-regressions.spec.ts": ( + "Onboarding edge cases: modal dismiss behavior, default model " + "selection, backdrop/Escape handling." + ), + "mock-llm-partial-stack.spec.ts": ( + "Partial stack modes: --frontend-only (503 for backend), " + "--backend-only (503 for frontend), port conflict detection. " + "bin/agent-canvas.mjs, static-server, ingress." + ), + "mock-llm-preset-automation.spec.ts": ( + "Preset automation cards, slash commands from home page, " + "skill activation via slash command." + ), + "mock-llm-profile-management.spec.ts": ( + "Active profile deletion + reconciliation, same-model profile " + "identity, litellm_proxy base_url preservation." + ), + "mock-llm-skills.spec.ts": ( + "Project skills (.agents/skills/), user skills (~/.openhands/skills/), " + "skill deletion, keyword-triggered activation." + ), + "mock-llm-ui-regressions.spec.ts": ( + "CSS isolation scoping, critic results rendering, event " + "pagination on scroll-up, workspace selection persistence." + ), +} + +ALL_SPECS = sorted(SPEC_CATALOG.keys()) + +# --------------------------------------------------------------------------- +# Heuristic path → spec mapping (fast fallback when LLM is unavailable) +# --------------------------------------------------------------------------- +HEURISTIC_MAP: list[tuple[list[str], list[str]]] = [ + # (path prefixes, relevant specs) + ( + ["src/components/features/onboarding/", "src/hooks/use-onboarding"], + ["mock-llm-onboarding-happy-path.spec.ts", "mock-llm-onboarding-regressions.spec.ts"], + ), + ( + ["src/api/automation-service/", "src/routes/automations", "src/components/features/automations/"], + ["mock-llm-automation.spec.ts", "mock-llm-preset-automation.spec.ts"], + ), + ( + ["src/components/features/backends/", "src/api/backend-registry/"], + ["mock-llm-auth-modes.spec.ts", "mock-llm-cross-connect.spec.ts"], + ), + ( + ["src/components/features/settings/", "src/routes/llm-settings", "src/routes/agent-settings"], + [ + "mock-llm-profile-management.spec.ts", + "mock-llm-conversation.spec.ts", + "mock-llm-model-switch.spec.ts", + "mock-llm-acp-agent.spec.ts", + ], + ), + ( + ["src/components/conversation-events/", "src/components/features/chat/"], + ["mock-llm-conversation.spec.ts", "mock-llm-image-upload.spec.ts", "mock-llm-ui-regressions.spec.ts"], + ), + ( + ["src/components/features/conversation-panel/"], + ["mock-llm-conversation.spec.ts", "mock-llm-model-switch.spec.ts"], + ), + ( + ["bin/", "scripts/static-server", "scripts/ingress", "docker/entrypoint"], + ["mock-llm-partial-stack.spec.ts", "mock-llm-auth-modes.spec.ts"], + ), + ( + ["src/hooks/query/use-conversation", "src/hooks/use-load-older-events"], + ["mock-llm-conversation.spec.ts", "mock-llm-ui-regressions.spec.ts"], + ), + ( + [".agents/skills/", "src/api/skills-service"], + ["mock-llm-skills.spec.ts"], + ), + ( + ["src/styles/", "src/components/shared/", "src/tailwind.css"], + ["mock-llm-ui-regressions.spec.ts"], + ), + ( + ["src/components/features/workspace/", "src/hooks/use-workspaces"], + ["mock-llm-ui-regressions.spec.ts", "mock-llm-onboarding-happy-path.spec.ts"], + ), + ( + ["tests/e2e/mock-llm/"], + [], # Modified test files → run those specific specs (handled separately) + ), +] + +# If changed files touch only these paths, no E2E tests are needed. +SKIP_PATHS = [ + "README", "AGENTS.md", "DEVELOPMENT.md", "LICENSE", "CHANGELOG", + ".github/workflows/", "docs/", "specs/", ".openhands/", + "__tests__/", "src/mocks/", ".env.sample", + "tests/e2e/snapshots/", "tests/e2e/live/", +] + + +def heuristic_select(changed_files: list[str]) -> tuple[list[str], str]: + """Return (specs, reason) using the static heuristic map.""" + selected: set[str] = set() + matched_areas: list[str] = [] + + # Direct test-file changes: run the changed spec itself. + for f in changed_files: + if f.startswith("tests/e2e/mock-llm/") and f.endswith(".spec.ts"): + basename = f.rsplit("/", 1)[-1] + if basename in SPEC_CATALOG: + selected.add(basename) + + # Check whether all files are skip-only. + non_skip = [f for f in changed_files if not any(f.startswith(s) or f == s for s in SKIP_PATHS)] + if not non_skip and not selected: + return [], "All changed files are docs/CI/tests — no mock-LLM E2E needed." + + for prefixes, specs in HEURISTIC_MAP: + for f in non_skip: + if any(f.startswith(p) for p in prefixes): + selected.update(specs) + matched_areas.extend(prefixes) + break + + if not selected and non_skip: + # Source files changed but no heuristic matched → run full suite. + return [], f"Heuristic could not narrow: {len(non_skip)} source files changed." + + reason = f"Heuristic matched {len(selected)} spec(s) from {len(set(matched_areas))} area(s)." + return sorted(selected), reason + + +def llm_select(changed_files: list[str]) -> tuple[list[str], str]: + """Use the OpenHands SDK LLM to pick the relevant specs.""" + try: + from openhands.sdk import LLM + except ImportError: + return [], "openhands-sdk not installed; falling back to heuristic." + + api_key = os.environ.get("LLM_API_KEY", "") + if not api_key: + return [], "LLM_API_KEY not set; falling back to heuristic." + + base_url = os.environ.get("LLM_BASE_URL", "https://llm-proxy.app.all-hands.dev") + model = os.environ.get("LLM_MODEL", "litellm_proxy/openai/gpt-4.1-mini") + + catalog_text = "\n".join( + f" - {name}: {desc}" for name, desc in sorted(SPEC_CATALOG.items()) + ) + files_text = "\n".join(f" - {f}" for f in changed_files[:200]) + + prompt = f"""\ +You are a CI test-selection assistant for the agent-canvas frontend project. + +Given the list of files modified in a pull request, decide which E2E test +specs are RELEVANT and should be run. Return ONLY a JSON object with two +keys: "specs" (list of spec filenames) and "reason" (one-sentence explanation). + +If the changes are broad (e.g. package.json, vite.config.ts, tsconfig, +root layout, core API layer) or you are unsure, return an empty "specs" +list to trigger the full suite. + +Available test specs and what they cover: +{catalog_text} + +Changed files in this PR: +{files_text} + +Respond with ONLY the JSON object, no markdown fences.""" + + try: + llm = LLM(model=model, api_key=api_key, base_url=base_url) + response = llm.completion( + messages=[{"role": "user", "content": prompt}], + temperature=0.0, + ) + content = response.choices[0].message.content.strip() + # Strip markdown fences if the model adds them anyway. + if content.startswith("```"): + content = content.split("\n", 1)[1].rsplit("```", 1)[0].strip() + result = json.loads(content) + specs = [s for s in result.get("specs", []) if s in SPEC_CATALOG] + reason = result.get("reason", "LLM selection") + return specs, reason + except Exception as e: + return [], f"LLM call failed ({e}); falling back to heuristic." + + +def main() -> None: + # Read changed files from args or stdin. + if len(sys.argv) > 1: + changed_files = sys.argv[1:] + else: + changed_files = [line.strip() for line in sys.stdin if line.strip()] + + if not changed_files: + print(json.dumps({"specs": [], "reason": "No changed files provided.", "mode": "full"})) + return + + # Try LLM first, fall back to heuristic. + specs, reason = llm_select(changed_files) + mode = "llm" + + if reason.startswith(("LLM_API_KEY not set", "openhands-sdk not installed", "LLM call failed")): + specs, reason = heuristic_select(changed_files) + mode = "heuristic" + + if not specs: + mode = "full" + + print(json.dumps({"specs": specs, "reason": reason, "mode": mode}, indent=2)) + + +if __name__ == "__main__": + main() From 6368db891e550309fa61d479c833365637e84d30 Mon Sep 17 00:00:00 2001 From: openhands Date: Mon, 8 Jun 2026 21:28:18 +0000 Subject: [PATCH 02/10] fix: use workflow_call instead of dispatch, switch model to openhands/gpt-5.1 - mock-llm-e2e.yml: Add workflow_call trigger with matching inputs so the workflow can be called as a reusable workflow (no PAT needed). - agent-e2e-selector.yml: Replace gh workflow run dispatch (which requires a PAT with actions:write scope) with a workflow_call job. The select job outputs feed directly into the run-e2e job. - select-e2e-tests.py: Change default model from litellm_proxy/openai/ gpt-4.1-mini to openhands/gpt-5.1. Co-authored-by: openhands --- .github/workflows/agent-e2e-selector.yml | 57 ++++++++++-------------- .github/workflows/mock-llm-e2e.yml | 20 +++++++++ scripts/select-e2e-tests.py | 4 +- 3 files changed, 45 insertions(+), 36 deletions(-) diff --git a/.github/workflows/agent-e2e-selector.yml b/.github/workflows/agent-e2e-selector.yml index d8f354130..9161f2ba5 100644 --- a/.github/workflows/agent-e2e-selector.yml +++ b/.github/workflows/agent-e2e-selector.yml @@ -2,7 +2,8 @@ # # Uses the OpenHands SDK LLM to analyze changed files in a PR and # determine which mock-LLM E2E test specs are most relevant, then -# triggers the Mock-LLM E2E Tests workflow with only that subset. +# calls the Mock-LLM E2E Tests workflow (via workflow_call) with only +# that subset. # # Trigger modes: # - Label "smart-e2e" on a PR → runs automatically @@ -28,14 +29,14 @@ on: permissions: contents: read pull-requests: write - actions: write # needed to trigger downstream workflow_dispatch concurrency: group: agent-e2e-selector-${{ github.event.pull_request.number || inputs.pr_number }} cancel-in-progress: true jobs: - select-and-dispatch: + # ── Phase 1: analyze changed files and pick the test subset ── + select: # On label events, only run when the "smart-e2e" label is applied. # On workflow_dispatch, always run. if: >- @@ -46,6 +47,12 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 5 + outputs: + specs: ${{ steps.selector.outputs.specs }} + reason: ${{ steps.selector.outputs.reason }} + mode: ${{ steps.selector.outputs.mode }} + file_count: ${{ steps.changed_files.outputs.count }} + env: EFFECTIVE_PR: ${{ github.event.pull_request.number || inputs.pr_number }} @@ -89,7 +96,7 @@ jobs: env: LLM_API_KEY: ${{ secrets.LLM_API_KEY }} LLM_BASE_URL: ${{ vars.LIVE_E2E_LLM_BASE_URL || 'https://llm-proxy.app.all-hands.dev' }} - LLM_MODEL: "litellm_proxy/openai/gpt-4.1-mini" + LLM_MODEL: "openhands/gpt-5.1" run: | RESULT=$(.selector-venv/bin/python3 scripts/select-e2e-tests.py < changed_files.txt) echo "$RESULT" | python3 -m json.tool @@ -104,35 +111,6 @@ jobs: echo "reason=$REASON" >> "$GITHUB_OUTPUT" echo "mode=$MODE" >> "$GITHUB_OUTPUT" - - name: Resolve PR head ref - id: pr_ref - env: - GITHUB_TOKEN: ${{ github.token }} - run: | - REF=$(gh api "/repos/${{ github.repository }}/pulls/${{ env.EFFECTIVE_PR }}" --jq '.head.ref') - echo "ref=$REF" >> "$GITHUB_OUTPUT" - echo "PR head ref: $REF" - - - name: Trigger Mock-LLM E2E Tests - env: - GITHUB_TOKEN: ${{ secrets.OPENHANDS_BOT_GITHUB_PAT_PUBLIC || github.token }} - run: | - SPECS="${{ steps.selector.outputs.specs }}" - MODE="${{ steps.selector.outputs.mode }}" - REF="${{ steps.pr_ref.outputs.ref }}" - PR="${{ env.EFFECTIVE_PR }}" - - echo "Mode: $MODE" - echo "Specs: ${SPECS:-}" - echo "Dispatching on ref: $REF" - - gh workflow run "Mock-LLM E2E Tests" \ - --ref "$REF" \ - -f test_specs="$SPECS" \ - -f pr_number="$PR" - - echo "::notice::Dispatched Mock-LLM E2E Tests (mode=$MODE, specs=${SPECS:-all})" - - name: Post selection summary to PR continue-on-error: true env: @@ -162,7 +140,7 @@ jobs: **Selected specs:** SPECS_PLACEHOLDER - _Triggered run: [Mock-LLM E2E Tests](/${{ github.repository }}/actions/workflows/mock-llm-e2e.yml)_ + _Running via: [Mock-LLM E2E Tests](/${{ github.repository }}/actions/runs/${{ github.run_id }})_ COMMENT_EOF ) @@ -172,3 +150,14 @@ jobs: BODY="${BODY//SPECS_PLACEHOLDER/$SPEC_DISPLAY}" echo "$BODY" | gh pr comment "${{ env.EFFECTIVE_PR }}" --body-file - + + # ── Phase 2: run the E2E suite via workflow_call ── + run-e2e: + needs: select + uses: ./.github/workflows/mock-llm-e2e.yml + with: + test_specs: ${{ needs.select.outputs.specs }} + pr_number: ${{ github.event.pull_request.number && format('{0}', github.event.pull_request.number) || format('{0}', inputs.pr_number) }} + permissions: + contents: read + pull-requests: write diff --git a/.github/workflows/mock-llm-e2e.yml b/.github/workflows/mock-llm-e2e.yml index 52f9ea8c7..53a1923b3 100644 --- a/.github/workflows/mock-llm-e2e.yml +++ b/.github/workflows/mock-llm-e2e.yml @@ -25,6 +25,26 @@ on: non-PR dispatches. required: false default: "" + # Reusable workflow entry point — called by agent-e2e-selector.yml. + # Inputs mirror workflow_dispatch so the job body can use + # `inputs.*` regardless of which trigger fired. + workflow_call: + inputs: + test_specs: + description: "Comma-separated spec filenames to run." + required: false + type: string + default: "" + test_grep: + description: "Playwright --grep pattern." + required: false + type: string + default: "" + pr_number: + description: "PR number for comment posting." + required: false + type: string + default: "" concurrency: group: mock-llm-e2e-${{ github.event.pull_request.number || inputs.pr_number || github.ref }} diff --git a/scripts/select-e2e-tests.py b/scripts/select-e2e-tests.py index eff93a075..2090074fb 100644 --- a/scripts/select-e2e-tests.py +++ b/scripts/select-e2e-tests.py @@ -15,7 +15,7 @@ Environment variables: LLM_API_KEY – required (unless --fallback-only) LLM_BASE_URL – optional, defaults to https://llm-proxy.app.all-hands.dev - LLM_MODEL – optional, defaults to litellm_proxy/openai/gpt-4.1-mini + LLM_MODEL – optional, defaults to openhands/gpt-5.1 Output (stdout): JSON object with keys: specs – list of spec filenames to run (empty ⇒ full suite) @@ -210,7 +210,7 @@ def llm_select(changed_files: list[str]) -> tuple[list[str], str]: return [], "LLM_API_KEY not set; falling back to heuristic." base_url = os.environ.get("LLM_BASE_URL", "https://llm-proxy.app.all-hands.dev") - model = os.environ.get("LLM_MODEL", "litellm_proxy/openai/gpt-4.1-mini") + model = os.environ.get("LLM_MODEL", "openhands/gpt-5.1") catalog_text = "\n".join( f" - {name}: {desc}" for name, desc in sorted(SPEC_CATALOG.items()) From 7a8ab8afe5c68b8996da283fcff620a8595ebe08 Mon Sep 17 00:00:00 2001 From: openhands Date: Mon, 8 Jun 2026 21:34:21 +0000 Subject: [PATCH 03/10] refactor: remove heuristic fallback, make agent selector the PR entry point MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - select-e2e-tests.py: Remove heuristic_select() and all static path-prefix mapping. Always use the LLM — if LLM_API_KEY is missing the script fails loudly instead of silently degrading. - mock-llm-e2e.yml: Remove pull_request trigger. E2E tests no longer run the full suite on every PR commit. They are now invoked only via workflow_call (from agent-e2e-selector) or workflow_dispatch. - agent-e2e-selector.yml: Trigger on pull_request [opened, synchronize, reopened] — this is now the primary entry point for PR-driven E2E. The LLM picks the relevant subset and the E2E workflow runs only those specs. Co-authored-by: openhands --- .github/workflows/agent-e2e-selector.yml | 21 +-- .github/workflows/mock-llm-e2e.yml | 2 - scripts/select-e2e-tests.py | 165 ++++------------------- 3 files changed, 31 insertions(+), 157 deletions(-) diff --git a/.github/workflows/agent-e2e-selector.yml b/.github/workflows/agent-e2e-selector.yml index 9161f2ba5..ecdc5133e 100644 --- a/.github/workflows/agent-e2e-selector.yml +++ b/.github/workflows/agent-e2e-selector.yml @@ -5,20 +5,16 @@ # calls the Mock-LLM E2E Tests workflow (via workflow_call) with only # that subset. # -# Trigger modes: -# - Label "smart-e2e" on a PR → runs automatically -# - Manual workflow_dispatch → provide a PR number -# -# This workflow is intentionally separate from the full-suite -# mock-llm-e2e.yml (which still fires on every PR push). Teams can -# use this as a faster feedback loop during development while keeping -# the full suite as a merge gate. +# This is the primary entry point for PR-driven E2E testing. +# mock-llm-e2e.yml no longer triggers on pull_request directly; +# it only runs when called from here (workflow_call) or via manual +# dispatch (workflow_dispatch). name: "Agent E2E Selector" on: pull_request: - types: [labeled] + types: [opened, synchronize, reopened] workflow_dispatch: inputs: pr_number: @@ -37,13 +33,10 @@ concurrency: jobs: # ── Phase 1: analyze changed files and pick the test subset ── select: - # On label events, only run when the "smart-e2e" label is applied. - # On workflow_dispatch, always run. + # Skip fork PRs — secrets are not available. if: >- github.event_name == 'workflow_dispatch' || - (github.event_name == 'pull_request' && - github.event.label.name == 'smart-e2e' && - github.event.pull_request.head.repo.full_name == github.repository) + github.event.pull_request.head.repo.full_name == github.repository runs-on: ubuntu-24.04 timeout-minutes: 5 diff --git a/.github/workflows/mock-llm-e2e.yml b/.github/workflows/mock-llm-e2e.yml index 53a1923b3..777895334 100644 --- a/.github/workflows/mock-llm-e2e.yml +++ b/.github/workflows/mock-llm-e2e.yml @@ -1,8 +1,6 @@ name: Mock-LLM E2E Tests on: - pull_request: - types: [opened, synchronize, reopened] workflow_dispatch: inputs: test_specs: diff --git a/scripts/select-e2e-tests.py b/scripts/select-e2e-tests.py index 2090074fb..05a59272e 100644 --- a/scripts/select-e2e-tests.py +++ b/scripts/select-e2e-tests.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 """Select which mock-LLM E2E test specs to run based on changed files. -Uses the OpenHands SDK's LLM class to intelligently map PR file changes -to the most relevant test specs. Falls back to running the full suite -when the LLM is unavailable or when the change set is too broad. +Uses the OpenHands SDK's LLM class to map PR file changes to the most +relevant test specs. Returns the full suite when the LLM decides the +changes are too broad to narrow. Usage: # Pipe changed files (one per line): @@ -13,14 +13,14 @@ python scripts/select-e2e-tests.py src/routes/automations.tsx src/api/automation-service/... Environment variables: - LLM_API_KEY – required (unless --fallback-only) + LLM_API_KEY – required LLM_BASE_URL – optional, defaults to https://llm-proxy.app.all-hands.dev LLM_MODEL – optional, defaults to openhands/gpt-5.1 Output (stdout): JSON object with keys: specs – list of spec filenames to run (empty ⇒ full suite) reason – human-readable explanation - mode – "llm" | "heuristic" | "full" + mode – "llm" | "full" """ from __future__ import annotations @@ -29,6 +29,8 @@ import os import sys +from openhands.sdk import LLM + # --------------------------------------------------------------------------- # Spec catalog – each entry maps a spec filename to a brief description # of what source areas it exercises. The LLM receives this catalog so it @@ -95,119 +97,12 @@ ), } -ALL_SPECS = sorted(SPEC_CATALOG.keys()) - -# --------------------------------------------------------------------------- -# Heuristic path → spec mapping (fast fallback when LLM is unavailable) -# --------------------------------------------------------------------------- -HEURISTIC_MAP: list[tuple[list[str], list[str]]] = [ - # (path prefixes, relevant specs) - ( - ["src/components/features/onboarding/", "src/hooks/use-onboarding"], - ["mock-llm-onboarding-happy-path.spec.ts", "mock-llm-onboarding-regressions.spec.ts"], - ), - ( - ["src/api/automation-service/", "src/routes/automations", "src/components/features/automations/"], - ["mock-llm-automation.spec.ts", "mock-llm-preset-automation.spec.ts"], - ), - ( - ["src/components/features/backends/", "src/api/backend-registry/"], - ["mock-llm-auth-modes.spec.ts", "mock-llm-cross-connect.spec.ts"], - ), - ( - ["src/components/features/settings/", "src/routes/llm-settings", "src/routes/agent-settings"], - [ - "mock-llm-profile-management.spec.ts", - "mock-llm-conversation.spec.ts", - "mock-llm-model-switch.spec.ts", - "mock-llm-acp-agent.spec.ts", - ], - ), - ( - ["src/components/conversation-events/", "src/components/features/chat/"], - ["mock-llm-conversation.spec.ts", "mock-llm-image-upload.spec.ts", "mock-llm-ui-regressions.spec.ts"], - ), - ( - ["src/components/features/conversation-panel/"], - ["mock-llm-conversation.spec.ts", "mock-llm-model-switch.spec.ts"], - ), - ( - ["bin/", "scripts/static-server", "scripts/ingress", "docker/entrypoint"], - ["mock-llm-partial-stack.spec.ts", "mock-llm-auth-modes.spec.ts"], - ), - ( - ["src/hooks/query/use-conversation", "src/hooks/use-load-older-events"], - ["mock-llm-conversation.spec.ts", "mock-llm-ui-regressions.spec.ts"], - ), - ( - [".agents/skills/", "src/api/skills-service"], - ["mock-llm-skills.spec.ts"], - ), - ( - ["src/styles/", "src/components/shared/", "src/tailwind.css"], - ["mock-llm-ui-regressions.spec.ts"], - ), - ( - ["src/components/features/workspace/", "src/hooks/use-workspaces"], - ["mock-llm-ui-regressions.spec.ts", "mock-llm-onboarding-happy-path.spec.ts"], - ), - ( - ["tests/e2e/mock-llm/"], - [], # Modified test files → run those specific specs (handled separately) - ), -] - -# If changed files touch only these paths, no E2E tests are needed. -SKIP_PATHS = [ - "README", "AGENTS.md", "DEVELOPMENT.md", "LICENSE", "CHANGELOG", - ".github/workflows/", "docs/", "specs/", ".openhands/", - "__tests__/", "src/mocks/", ".env.sample", - "tests/e2e/snapshots/", "tests/e2e/live/", -] - - -def heuristic_select(changed_files: list[str]) -> tuple[list[str], str]: - """Return (specs, reason) using the static heuristic map.""" - selected: set[str] = set() - matched_areas: list[str] = [] - - # Direct test-file changes: run the changed spec itself. - for f in changed_files: - if f.startswith("tests/e2e/mock-llm/") and f.endswith(".spec.ts"): - basename = f.rsplit("/", 1)[-1] - if basename in SPEC_CATALOG: - selected.add(basename) - - # Check whether all files are skip-only. - non_skip = [f for f in changed_files if not any(f.startswith(s) or f == s for s in SKIP_PATHS)] - if not non_skip and not selected: - return [], "All changed files are docs/CI/tests — no mock-LLM E2E needed." - - for prefixes, specs in HEURISTIC_MAP: - for f in non_skip: - if any(f.startswith(p) for p in prefixes): - selected.update(specs) - matched_areas.extend(prefixes) - break - - if not selected and non_skip: - # Source files changed but no heuristic matched → run full suite. - return [], f"Heuristic could not narrow: {len(non_skip)} source files changed." - reason = f"Heuristic matched {len(selected)} spec(s) from {len(set(matched_areas))} area(s)." - return sorted(selected), reason - - -def llm_select(changed_files: list[str]) -> tuple[list[str], str]: +def select(changed_files: list[str]) -> tuple[list[str], str]: """Use the OpenHands SDK LLM to pick the relevant specs.""" - try: - from openhands.sdk import LLM - except ImportError: - return [], "openhands-sdk not installed; falling back to heuristic." - api_key = os.environ.get("LLM_API_KEY", "") if not api_key: - return [], "LLM_API_KEY not set; falling back to heuristic." + raise RuntimeError("LLM_API_KEY is required but not set.") base_url = os.environ.get("LLM_BASE_URL", "https://llm-proxy.app.all-hands.dev") model = os.environ.get("LLM_MODEL", "openhands/gpt-5.1") @@ -236,22 +131,19 @@ def llm_select(changed_files: list[str]) -> tuple[list[str], str]: Respond with ONLY the JSON object, no markdown fences.""" - try: - llm = LLM(model=model, api_key=api_key, base_url=base_url) - response = llm.completion( - messages=[{"role": "user", "content": prompt}], - temperature=0.0, - ) - content = response.choices[0].message.content.strip() - # Strip markdown fences if the model adds them anyway. - if content.startswith("```"): - content = content.split("\n", 1)[1].rsplit("```", 1)[0].strip() - result = json.loads(content) - specs = [s for s in result.get("specs", []) if s in SPEC_CATALOG] - reason = result.get("reason", "LLM selection") - return specs, reason - except Exception as e: - return [], f"LLM call failed ({e}); falling back to heuristic." + llm = LLM(model=model, api_key=api_key, base_url=base_url) + response = llm.completion( + messages=[{"role": "user", "content": prompt}], + temperature=0.0, + ) + content = response.choices[0].message.content.strip() + # Strip markdown fences if the model adds them anyway. + if content.startswith("```"): + content = content.split("\n", 1)[1].rsplit("```", 1)[0].strip() + result = json.loads(content) + specs = [s for s in result.get("specs", []) if s in SPEC_CATALOG] + reason = result.get("reason", "LLM selection") + return specs, reason def main() -> None: @@ -265,17 +157,8 @@ def main() -> None: print(json.dumps({"specs": [], "reason": "No changed files provided.", "mode": "full"})) return - # Try LLM first, fall back to heuristic. - specs, reason = llm_select(changed_files) - mode = "llm" - - if reason.startswith(("LLM_API_KEY not set", "openhands-sdk not installed", "LLM call failed")): - specs, reason = heuristic_select(changed_files) - mode = "heuristic" - - if not specs: - mode = "full" - + specs, reason = select(changed_files) + mode = "llm" if specs else "full" print(json.dumps({"specs": specs, "reason": reason, "mode": mode}, indent=2)) From fffa857238248014c771270b31af44270f0745ec Mon Sep 17 00:00:00 2001 From: openhands Date: Mon, 8 Jun 2026 21:39:20 +0000 Subject: [PATCH 04/10] refactor: use SDK Agent + Conversation, skip E2E when no specs selected - select-e2e-tests.py: Rewritten to use Agent + Conversation instead of raw LLM.completion(). Adds a CIVisualizer (streams every event to stderr for CI log visibility) and a capture_event callback (collects agent messages for parsing). The agent outputs a structured block that gets parsed for the spec list. Empty specs now means 'skip E2E' not 'run full suite'. - agent-e2e-selector.yml: run-e2e job now has an if-guard that skips mock-llm-e2e entirely when the agent returns no specs. Co-authored-by: openhands --- .github/workflows/agent-e2e-selector.yml | 2 + scripts/select-e2e-tests.py | 160 ++++++++++++++++------- 2 files changed, 118 insertions(+), 44 deletions(-) diff --git a/.github/workflows/agent-e2e-selector.yml b/.github/workflows/agent-e2e-selector.yml index ecdc5133e..01db9556d 100644 --- a/.github/workflows/agent-e2e-selector.yml +++ b/.github/workflows/agent-e2e-selector.yml @@ -145,8 +145,10 @@ jobs: echo "$BODY" | gh pr comment "${{ env.EFFECTIVE_PR }}" --body-file - # ── Phase 2: run the E2E suite via workflow_call ── + # Skipped entirely when the agent returns no specs (docs/CI-only changes). run-e2e: needs: select + if: needs.select.outputs.specs != '' uses: ./.github/workflows/mock-llm-e2e.yml with: test_specs: ${{ needs.select.outputs.specs }} diff --git a/scripts/select-e2e-tests.py b/scripts/select-e2e-tests.py index 05a59272e..f77200b49 100644 --- a/scripts/select-e2e-tests.py +++ b/scripts/select-e2e-tests.py @@ -1,16 +1,17 @@ #!/usr/bin/env python3 """Select which mock-LLM E2E test specs to run based on changed files. -Uses the OpenHands SDK's LLM class to map PR file changes to the most -relevant test specs. Returns the full suite when the LLM decides the -changes are too broad to narrow. +Uses an OpenHands SDK Agent + Conversation to reason about which E2E +test specs are relevant to a set of changed files. The agent's +thinking is streamed to stderr so CI logs show the full reasoning; +the final JSON result is written to stdout. Usage: # Pipe changed files (one per line): git diff --name-only origin/main | python scripts/select-e2e-tests.py # Or pass as arguments: - python scripts/select-e2e-tests.py src/routes/automations.tsx src/api/automation-service/... + python scripts/select-e2e-tests.py src/routes/automations.tsx ... Environment variables: LLM_API_KEY – required @@ -18,23 +19,30 @@ LLM_MODEL – optional, defaults to openhands/gpt-5.1 Output (stdout): JSON object with keys: - specs – list of spec filenames to run (empty ⇒ full suite) + specs – list of spec filenames to run (empty ⇒ no E2E needed) reason – human-readable explanation - mode – "llm" | "full" + mode – "llm" """ from __future__ import annotations import json +import logging import os +import re import sys +import tempfile -from openhands.sdk import LLM +from pydantic import SecretStr + +from openhands.sdk import LLM, Agent, Conversation, Event +from openhands.sdk.conversation.visualizer import ConversationVisualizerBase + +# Suppress noisy SDK / litellm logs — our visualizer handles output. +logging.getLogger().setLevel(logging.WARNING) # --------------------------------------------------------------------------- -# Spec catalog – each entry maps a spec filename to a brief description -# of what source areas it exercises. The LLM receives this catalog so it -# can reason about coverage. +# Spec catalog – the agent receives this so it can reason about coverage. # --------------------------------------------------------------------------- SPEC_CATALOG: dict[str, str] = { "mock-llm-acp-agent.spec.ts": ( @@ -97,31 +105,47 @@ ), } +OUTPUT_TAG = "TEST_SELECTION" -def select(changed_files: list[str]) -> tuple[list[str], str]: - """Use the OpenHands SDK LLM to pick the relevant specs.""" - api_key = os.environ.get("LLM_API_KEY", "") - if not api_key: - raise RuntimeError("LLM_API_KEY is required but not set.") - base_url = os.environ.get("LLM_BASE_URL", "https://llm-proxy.app.all-hands.dev") - model = os.environ.get("LLM_MODEL", "openhands/gpt-5.1") +# --------------------------------------------------------------------------- +# Visualizer — streams every agent event to stderr for CI visibility +# --------------------------------------------------------------------------- +class CIVisualizer(ConversationVisualizerBase): + """Prints agent events to stderr so CI logs show the full reasoning.""" + + def on_event(self, event: Event) -> None: + name = type(event).__name__ + dump = event.model_dump_json()[:500] + print(f"[agent] {name}: {dump}", file=sys.stderr) + +# --------------------------------------------------------------------------- +# Conversation callback — collects the raw content for parsing +# --------------------------------------------------------------------------- +collected_messages: list[str] = [] + + +def capture_event(event: Event) -> None: + """Callback passed to Conversation to capture agent messages.""" + # MessageAction events have a 'message' field with the agent's text. + msg = getattr(event, "message", None) or getattr(event, "text", None) + if msg and isinstance(msg, str): + collected_messages.append(msg) + + +# --------------------------------------------------------------------------- +# Build the user prompt +# --------------------------------------------------------------------------- +def build_prompt(changed_files: list[str]) -> str: catalog_text = "\n".join( f" - {name}: {desc}" for name, desc in sorted(SPEC_CATALOG.items()) ) files_text = "\n".join(f" - {f}" for f in changed_files[:200]) - prompt = f"""\ -You are a CI test-selection assistant for the agent-canvas frontend project. - -Given the list of files modified in a pull request, decide which E2E test -specs are RELEVANT and should be run. Return ONLY a JSON object with two -keys: "specs" (list of spec filenames) and "reason" (one-sentence explanation). - -If the changes are broad (e.g. package.json, vite.config.ts, tsconfig, -root layout, core API layer) or you are unsure, return an empty "specs" -list to trigger the full suite. + return f"""\ +Analyze the following list of files modified in a pull request and decide +which E2E test specs should be run. Available test specs and what they cover: {catalog_text} @@ -129,37 +153,85 @@ def select(changed_files: list[str]) -> tuple[list[str], str]: Changed files in this PR: {files_text} -Respond with ONLY the JSON object, no markdown fences.""" +Rules: +- Pick ONLY the specs whose covered areas are affected by the changed files. +- If no spec is relevant (e.g. only docs, CI configs, or unit tests changed), + return an empty "specs" list — we will skip E2E entirely. +- If the changes are very broad (package.json, vite.config.ts, tsconfig, + root layout, core shared utilities) and could affect anything, return + ALL spec filenames. - llm = LLM(model=model, api_key=api_key, base_url=base_url) - response = llm.completion( - messages=[{"role": "user", "content": prompt}], - temperature=0.0, - ) - content = response.choices[0].message.content.strip() - # Strip markdown fences if the model adds them anyway. - if content.startswith("```"): - content = content.split("\n", 1)[1].rsplit("```", 1)[0].strip() - result = json.loads(content) +You MUST end your response with exactly this block (no markdown fences): + +<{OUTPUT_TAG}> +{{"specs": ["spec1.spec.ts", ...], "reason": "one sentence explanation"}} +""" + + +# --------------------------------------------------------------------------- +# Parse the structured output from the agent's response +# --------------------------------------------------------------------------- +def parse_selection(text: str) -> dict: + pattern = rf"<{OUTPUT_TAG}>\s*(.*?)\s*" + match = re.search(pattern, text, re.DOTALL) + if not match: + raise ValueError(f"Agent response missing <{OUTPUT_TAG}> block") + raw = match.group(1).strip() + result = json.loads(raw) specs = [s for s in result.get("specs", []) if s in SPEC_CATALOG] reason = result.get("reason", "LLM selection") - return specs, reason + return {"specs": specs, "reason": reason} +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- def main() -> None: - # Read changed files from args or stdin. if len(sys.argv) > 1: changed_files = sys.argv[1:] else: changed_files = [line.strip() for line in sys.stdin if line.strip()] if not changed_files: - print(json.dumps({"specs": [], "reason": "No changed files provided.", "mode": "full"})) + print(json.dumps({"specs": [], "reason": "No changed files provided.", "mode": "llm"})) return - specs, reason = select(changed_files) - mode = "llm" if specs else "full" - print(json.dumps({"specs": specs, "reason": reason, "mode": mode}, indent=2)) + api_key = os.environ.get("LLM_API_KEY", "") + if not api_key: + raise RuntimeError("LLM_API_KEY is required but not set.") + + base_url = os.environ.get("LLM_BASE_URL", "https://llm-proxy.app.all-hands.dev") + model = os.environ.get("LLM_MODEL", "openhands/gpt-5.1") + + llm = LLM( + model=model, + api_key=SecretStr(api_key), + base_url=base_url, + usage_id="e2e-selector", + ) + agent = Agent(llm=llm, tools=[]) + + with tempfile.TemporaryDirectory() as workspace: + conversation = Conversation( + agent=agent, + workspace=workspace, + visualizer=CIVisualizer(), + callbacks=[capture_event], + ) + + prompt = build_prompt(changed_files) + conversation.send_message(prompt) + conversation.run() + + # Find the structured output in any captured message. + full_text = "\n".join(collected_messages) + result = parse_selection(full_text) + result["mode"] = "llm" + + print(json.dumps(result, indent=2)) + + cost = llm.metrics.accumulated_cost + print(f"LLM cost: ${cost:.4f}", file=sys.stderr) if __name__ == "__main__": From 2b1add156242b6e10fec308dd7aec25cbb1e38ed Mon Sep 17 00:00:00 2001 From: openhands Date: Mon, 8 Jun 2026 21:41:39 +0000 Subject: [PATCH 05/10] fix: extract text from event JSON dumps to find TEST_SELECTION block The capture_event callback collects model_dump_json() from every event. The agent's response text lives at event.llm_message.content[].text, not at event.message or event.text. Added extract_text_from_dumps() to properly walk the JSON structure and pull out all text fragments before searching for the tag. Also improved the ValueError message to include the extracted text for easier debugging if parsing still fails. Co-authored-by: openhands --- scripts/select-e2e-tests.py | 42 +++++++++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/scripts/select-e2e-tests.py b/scripts/select-e2e-tests.py index f77200b49..bfaa04d19 100644 --- a/scripts/select-e2e-tests.py +++ b/scripts/select-e2e-tests.py @@ -123,15 +123,12 @@ def on_event(self, event: Event) -> None: # --------------------------------------------------------------------------- # Conversation callback — collects the raw content for parsing # --------------------------------------------------------------------------- -collected_messages: list[str] = [] +collected_dumps: list[str] = [] def capture_event(event: Event) -> None: - """Callback passed to Conversation to capture agent messages.""" - # MessageAction events have a 'message' field with the agent's text. - msg = getattr(event, "message", None) or getattr(event, "text", None) - if msg and isinstance(msg, str): - collected_messages.append(msg) + """Callback passed to Conversation to capture every event's JSON.""" + collected_dumps.append(event.model_dump_json()) # --------------------------------------------------------------------------- @@ -168,6 +165,30 @@ def build_prompt(changed_files: list[str]) -> str: """ +# --------------------------------------------------------------------------- +# Extract text content from collected event dumps +# --------------------------------------------------------------------------- +def extract_text_from_dumps(dumps: list[str]) -> str: + """Walk every collected event JSON and pull out all text content.""" + fragments: list[str] = [] + for raw in dumps: + try: + obj = json.loads(raw) + except json.JSONDecodeError: + continue + # MessageEvent → llm_message.content[].text + llm_msg = obj.get("llm_message") or {} + for block in llm_msg.get("content") or []: + if isinstance(block, dict) and block.get("text"): + fragments.append(block["text"]) + # FinishAction or plain message fields + for key in ("message", "text", "thought"): + val = obj.get(key) + if val and isinstance(val, str): + fragments.append(val) + return "\n".join(fragments) + + # --------------------------------------------------------------------------- # Parse the structured output from the agent's response # --------------------------------------------------------------------------- @@ -175,7 +196,10 @@ def parse_selection(text: str) -> dict: pattern = rf"<{OUTPUT_TAG}>\s*(.*?)\s*" match = re.search(pattern, text, re.DOTALL) if not match: - raise ValueError(f"Agent response missing <{OUTPUT_TAG}> block") + raise ValueError( + f"Agent response missing <{OUTPUT_TAG}> block.\n" + f"Full extracted text ({len(text)} chars):\n{text[:1000]}" + ) raw = match.group(1).strip() result = json.loads(raw) specs = [s for s in result.get("specs", []) if s in SPEC_CATALOG] @@ -223,8 +247,8 @@ def main() -> None: conversation.send_message(prompt) conversation.run() - # Find the structured output in any captured message. - full_text = "\n".join(collected_messages) + # Extract text from all captured events and find the structured output. + full_text = extract_text_from_dumps(collected_dumps) result = parse_selection(full_text) result["mode"] = "llm" From 814950931d76a2e9bdd4dc079ee0ba3f130a07df Mon Sep 17 00:00:00 2001 From: openhands Date: Mon, 8 Jun 2026 21:45:33 +0000 Subject: [PATCH 06/10] refactor: give agent repo access with tools, fix parsing bugs - select-e2e-tests.py: Agent now has terminal + file_editor tools and runs against the repo checkout (WORKSPACE env var, defaults to cwd). It can read source files and test specs to make informed decisions. Fixed two parsing bugs: (1) only collect agent-source events (skip user prompt to avoid matching the template), (2) use re.findall and take the last match as an extra safety net. Increased visualizer dump to 800 chars for better CI log visibility. - agent-e2e-selector.yml: Install openhands-tools alongside openhands-sdk (needed for TerminalTool/FileEditorTool). Suppress SDK banner. Bumped timeout to 10 min for agent tool-call iterations. Co-authored-by: openhands --- .github/workflows/agent-e2e-selector.yml | 9 +- scripts/select-e2e-tests.py | 109 ++++++++++++++--------- 2 files changed, 71 insertions(+), 47 deletions(-) diff --git a/.github/workflows/agent-e2e-selector.yml b/.github/workflows/agent-e2e-selector.yml index 01db9556d..1fce3943a 100644 --- a/.github/workflows/agent-e2e-selector.yml +++ b/.github/workflows/agent-e2e-selector.yml @@ -38,7 +38,7 @@ jobs: github.event_name == 'workflow_dispatch' || github.event.pull_request.head.repo.full_name == github.repository runs-on: ubuntu-24.04 - timeout-minutes: 5 + timeout-minutes: 10 outputs: specs: ${{ steps.selector.outputs.specs }} @@ -63,12 +63,14 @@ jobs: curl -LsSf https://astral.sh/uv/install.sh | sh echo "$HOME/.local/bin" >> "$GITHUB_PATH" - - name: Install openhands-sdk + - name: Install openhands-sdk + tools env: AGENT_SERVER_VERSION: ${{ steps.defaults.outputs.agent_server_version }} run: | uv venv .selector-venv - uv pip install -p .selector-venv "openhands-sdk==$AGENT_SERVER_VERSION" + uv pip install -p .selector-venv \ + "openhands-sdk==$AGENT_SERVER_VERSION" \ + "openhands-tools==$AGENT_SERVER_VERSION" - name: Get changed files from PR id: changed_files @@ -90,6 +92,7 @@ jobs: LLM_API_KEY: ${{ secrets.LLM_API_KEY }} LLM_BASE_URL: ${{ vars.LIVE_E2E_LLM_BASE_URL || 'https://llm-proxy.app.all-hands.dev' }} LLM_MODEL: "openhands/gpt-5.1" + OPENHANDS_SUPPRESS_BANNER: "1" run: | RESULT=$(.selector-venv/bin/python3 scripts/select-e2e-tests.py < changed_files.txt) echo "$RESULT" | python3 -m json.tool diff --git a/scripts/select-e2e-tests.py b/scripts/select-e2e-tests.py index bfaa04d19..95c853b62 100644 --- a/scripts/select-e2e-tests.py +++ b/scripts/select-e2e-tests.py @@ -1,22 +1,27 @@ #!/usr/bin/env python3 """Select which mock-LLM E2E test specs to run based on changed files. -Uses an OpenHands SDK Agent + Conversation to reason about which E2E -test specs are relevant to a set of changed files. The agent's -thinking is streamed to stderr so CI logs show the full reasoning; -the final JSON result is written to stdout. +Spins up an OpenHands SDK Agent with terminal + file_editor tools, +pointed at the checked-out repository, so it can read source files +and test specs to make an informed decision. The agent's events are +streamed to stderr for CI visibility; the final JSON result goes to +stdout. Usage: - # Pipe changed files (one per line): + # From the repo root, pipe changed file paths: git diff --name-only origin/main | python scripts/select-e2e-tests.py # Or pass as arguments: python scripts/select-e2e-tests.py src/routes/automations.tsx ... + # Specify a workspace (defaults to cwd): + WORKSPACE=/path/to/repo python scripts/select-e2e-tests.py < files.txt + Environment variables: LLM_API_KEY – required LLM_BASE_URL – optional, defaults to https://llm-proxy.app.all-hands.dev LLM_MODEL – optional, defaults to openhands/gpt-5.1 + WORKSPACE – optional, repo root (defaults to cwd) Output (stdout): JSON object with keys: specs – list of spec filenames to run (empty ⇒ no E2E needed) @@ -31,12 +36,13 @@ import os import re import sys -import tempfile from pydantic import SecretStr -from openhands.sdk import LLM, Agent, Conversation, Event +from openhands.sdk import LLM, Agent, Conversation, Event, Tool from openhands.sdk.conversation.visualizer import ConversationVisualizerBase +from openhands.tools.file_editor import FileEditorTool +from openhands.tools.terminal import TerminalTool # Suppress noisy SDK / litellm logs — our visualizer handles output. logging.getLogger().setLevel(logging.WARNING) @@ -116,19 +122,21 @@ class CIVisualizer(ConversationVisualizerBase): def on_event(self, event: Event) -> None: name = type(event).__name__ - dump = event.model_dump_json()[:500] - print(f"[agent] {name}: {dump}", file=sys.stderr) + dump = event.model_dump_json()[:800] + print(f"[agent] {name}: {dump}", file=sys.stderr, flush=True) # --------------------------------------------------------------------------- -# Conversation callback — collects the raw content for parsing +# Conversation callback — collects agent-source event dumps for parsing # --------------------------------------------------------------------------- -collected_dumps: list[str] = [] +collected_agent_dumps: list[str] = [] def capture_event(event: Event) -> None: - """Callback passed to Conversation to capture every event's JSON.""" - collected_dumps.append(event.model_dump_json()) + """Capture only agent-originated events for post-run parsing.""" + source = getattr(event, "source", None) + if source == "agent": + collected_agent_dumps.append(event.model_dump_json()) # --------------------------------------------------------------------------- @@ -141,35 +149,41 @@ def build_prompt(changed_files: list[str]) -> str: files_text = "\n".join(f" - {f}" for f in changed_files[:200]) return f"""\ -Analyze the following list of files modified in a pull request and decide -which E2E test specs should be run. +You have access to the full repository checkout in your working directory. +Your task: decide which E2E test specs should be run for this pull request. + +You can use the terminal and file_editor tools to read any source files or +test specs if you need to understand what the changed code does. Do NOT +modify any files. -Available test specs and what they cover: +Available test specs (in tests/e2e/mock-llm/) and what they cover: {catalog_text} -Changed files in this PR: +Files modified in this PR: {files_text} Rules: - Pick ONLY the specs whose covered areas are affected by the changed files. + If you are unsure whether a file is relevant, read it first. - If no spec is relevant (e.g. only docs, CI configs, or unit tests changed), return an empty "specs" list — we will skip E2E entirely. - If the changes are very broad (package.json, vite.config.ts, tsconfig, root layout, core shared utilities) and could affect anything, return ALL spec filenames. -You MUST end your response with exactly this block (no markdown fences): +When you are done, call the `finish` tool with a message that contains +exactly this block: <{OUTPUT_TAG}> -{{"specs": ["spec1.spec.ts", ...], "reason": "one sentence explanation"}} +{{"specs": ["spec-filename.spec.ts"], "reason": "one sentence explanation"}} """ # --------------------------------------------------------------------------- -# Extract text content from collected event dumps +# Extract text from agent-source event dumps # --------------------------------------------------------------------------- -def extract_text_from_dumps(dumps: list[str]) -> str: - """Walk every collected event JSON and pull out all text content.""" +def extract_agent_text(dumps: list[str]) -> str: + """Pull text content from agent-originated event JSON dumps.""" fragments: list[str] = [] for raw in dumps: try: @@ -177,11 +191,10 @@ def extract_text_from_dumps(dumps: list[str]) -> str: except json.JSONDecodeError: continue # MessageEvent → llm_message.content[].text - llm_msg = obj.get("llm_message") or {} - for block in llm_msg.get("content") or []: + for block in (obj.get("llm_message") or {}).get("content") or []: if isinstance(block, dict) and block.get("text"): fragments.append(block["text"]) - # FinishAction or plain message fields + # FinishAction, MessageAction, or other events with direct fields for key in ("message", "text", "thought"): val = obj.get(key) if val and isinstance(val, str): @@ -194,13 +207,14 @@ def extract_text_from_dumps(dumps: list[str]) -> str: # --------------------------------------------------------------------------- def parse_selection(text: str) -> dict: pattern = rf"<{OUTPUT_TAG}>\s*(.*?)\s*" - match = re.search(pattern, text, re.DOTALL) - if not match: + matches = re.findall(pattern, text, re.DOTALL) + if not matches: raise ValueError( f"Agent response missing <{OUTPUT_TAG}> block.\n" - f"Full extracted text ({len(text)} chars):\n{text[:1000]}" + f"Full extracted text ({len(text)} chars):\n{text[:2000]}" ) - raw = match.group(1).strip() + # Take the LAST match — earlier ones may be the prompt template. + raw = matches[-1].strip() result = json.loads(raw) specs = [s for s in result.get("specs", []) if s in SPEC_CATALOG] reason = result.get("reason", "LLM selection") @@ -226,6 +240,7 @@ def main() -> None: base_url = os.environ.get("LLM_BASE_URL", "https://llm-proxy.app.all-hands.dev") model = os.environ.get("LLM_MODEL", "openhands/gpt-5.1") + workspace = os.environ.get("WORKSPACE", os.getcwd()) llm = LLM( model=model, @@ -233,23 +248,29 @@ def main() -> None: base_url=base_url, usage_id="e2e-selector", ) - agent = Agent(llm=llm, tools=[]) - - with tempfile.TemporaryDirectory() as workspace: - conversation = Conversation( - agent=agent, - workspace=workspace, - visualizer=CIVisualizer(), - callbacks=[capture_event], - ) + agent = Agent( + llm=llm, + tools=[ + Tool(name=TerminalTool.name), + Tool(name=FileEditorTool.name), + ], + ) + + conversation = Conversation( + agent=agent, + workspace=workspace, + visualizer=CIVisualizer(), + callbacks=[capture_event], + max_iterations=30, + ) - prompt = build_prompt(changed_files) - conversation.send_message(prompt) - conversation.run() + prompt = build_prompt(changed_files) + conversation.send_message(prompt) + conversation.run() - # Extract text from all captured events and find the structured output. - full_text = extract_text_from_dumps(collected_dumps) - result = parse_selection(full_text) + # Extract text from agent-only events and parse the structured output. + agent_text = extract_agent_text(collected_agent_dumps) + result = parse_selection(agent_text) result["mode"] = "llm" print(json.dumps(result, indent=2)) From 64702ac3eecdb45814e4bf970c3796d55705a02b Mon Sep 17 00:00:00 2001 From: openhands Date: Mon, 8 Jun 2026 21:47:04 +0000 Subject: [PATCH 07/10] fix: use correct SDK param name max_iteration_per_run Co-authored-by: openhands --- scripts/select-e2e-tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/select-e2e-tests.py b/scripts/select-e2e-tests.py index 95c853b62..2c6e54f6e 100644 --- a/scripts/select-e2e-tests.py +++ b/scripts/select-e2e-tests.py @@ -261,7 +261,7 @@ def main() -> None: workspace=workspace, visualizer=CIVisualizer(), callbacks=[capture_event], - max_iterations=30, + max_iteration_per_run=30, ) prompt = build_prompt(changed_files) From dee20cfd414f61263ae6d702285ad30be8596401 Mon Sep 17 00:00:00 2001 From: openhands Date: Mon, 8 Jun 2026 21:49:00 +0000 Subject: [PATCH 08/10] fix: capture all events and walk full JSON tree for text extraction The TEST_SELECTION block lands in the FinishObservation event (source='environment'), not in agent-source events. Removed the source filter from the callback and replaced the flat key-lookup extractor with a recursive _collect_text() that walks the entire JSON tree collecting every 'text' and 'message' string value. parse_selection() already uses re.findall + last-match to avoid matching the prompt template. Co-authored-by: openhands --- scripts/select-e2e-tests.py | 45 ++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/scripts/select-e2e-tests.py b/scripts/select-e2e-tests.py index 2c6e54f6e..082169976 100644 --- a/scripts/select-e2e-tests.py +++ b/scripts/select-e2e-tests.py @@ -127,16 +127,14 @@ def on_event(self, event: Event) -> None: # --------------------------------------------------------------------------- -# Conversation callback — collects agent-source event dumps for parsing +# Conversation callback — collects all event dumps for parsing # --------------------------------------------------------------------------- -collected_agent_dumps: list[str] = [] +collected_dumps: list[str] = [] def capture_event(event: Event) -> None: - """Capture only agent-originated events for post-run parsing.""" - source = getattr(event, "source", None) - if source == "agent": - collected_agent_dumps.append(event.model_dump_json()) + """Capture every event for post-run parsing.""" + collected_dumps.append(event.model_dump_json()) # --------------------------------------------------------------------------- @@ -180,25 +178,30 @@ def build_prompt(changed_files: list[str]) -> str: # --------------------------------------------------------------------------- -# Extract text from agent-source event dumps +# Extract text from event dumps # --------------------------------------------------------------------------- -def extract_agent_text(dumps: list[str]) -> str: - """Pull text content from agent-originated event JSON dumps.""" +def _collect_text(obj: object, out: list[str]) -> None: + """Recursively collect every string value named 'text' or 'message'.""" + if isinstance(obj, dict): + for key, val in obj.items(): + if key in ("text", "message") and isinstance(val, str) and val.strip(): + out.append(val) + else: + _collect_text(val, out) + elif isinstance(obj, list): + for item in obj: + _collect_text(item, out) + + +def extract_all_text(dumps: list[str]) -> str: + """Walk every collected event JSON and pull out all text fragments.""" fragments: list[str] = [] for raw in dumps: try: obj = json.loads(raw) except json.JSONDecodeError: continue - # MessageEvent → llm_message.content[].text - for block in (obj.get("llm_message") or {}).get("content") or []: - if isinstance(block, dict) and block.get("text"): - fragments.append(block["text"]) - # FinishAction, MessageAction, or other events with direct fields - for key in ("message", "text", "thought"): - val = obj.get(key) - if val and isinstance(val, str): - fragments.append(val) + _collect_text(obj, fragments) return "\n".join(fragments) @@ -268,9 +271,9 @@ def main() -> None: conversation.send_message(prompt) conversation.run() - # Extract text from agent-only events and parse the structured output. - agent_text = extract_agent_text(collected_agent_dumps) - result = parse_selection(agent_text) + # Extract text from all events and parse the structured output. + all_text = extract_all_text(collected_dumps) + result = parse_selection(all_text) result["mode"] = "llm" print(json.dumps(result, indent=2)) From d6581db674bf2a3b2963e973ca2a6ae29d578d9e Mon Sep 17 00:00:00 2001 From: openhands Date: Mon, 8 Jun 2026 21:51:57 +0000 Subject: [PATCH 09/10] simplify: agent writes selection to a file instead of message parsing Instead of wrangling agent event dumps / regex / recursive JSON walking, the agent now writes its result to a temp JSON file. After the conversation finishes, we just read that file. Removed: OUTPUT_TAG, capture_event callback, _collect_text, extract_all_text, parse_selection, re import. The CIVisualizer still streams all events to stderr for CI log visibility. Co-authored-by: openhands --- scripts/select-e2e-tests.py | 117 +++++++++++------------------------- 1 file changed, 36 insertions(+), 81 deletions(-) diff --git a/scripts/select-e2e-tests.py b/scripts/select-e2e-tests.py index 082169976..b7a3b829f 100644 --- a/scripts/select-e2e-tests.py +++ b/scripts/select-e2e-tests.py @@ -2,10 +2,10 @@ """Select which mock-LLM E2E test specs to run based on changed files. Spins up an OpenHands SDK Agent with terminal + file_editor tools, -pointed at the checked-out repository, so it can read source files -and test specs to make an informed decision. The agent's events are -streamed to stderr for CI visibility; the final JSON result goes to -stdout. +pointed at the checked-out repository. The agent reads source files +and test specs as needed, then writes its selection to a JSON file. +Agent events are streamed to stderr for CI visibility; the final +JSON result is printed to stdout. Usage: # From the repo root, pipe changed file paths: @@ -14,9 +14,6 @@ # Or pass as arguments: python scripts/select-e2e-tests.py src/routes/automations.tsx ... - # Specify a workspace (defaults to cwd): - WORKSPACE=/path/to/repo python scripts/select-e2e-tests.py < files.txt - Environment variables: LLM_API_KEY – required LLM_BASE_URL – optional, defaults to https://llm-proxy.app.all-hands.dev @@ -34,8 +31,8 @@ import json import logging import os -import re import sys +import tempfile from pydantic import SecretStr @@ -111,8 +108,6 @@ ), } -OUTPUT_TAG = "TEST_SELECTION" - # --------------------------------------------------------------------------- # Visualizer — streams every agent event to stderr for CI visibility @@ -126,21 +121,10 @@ def on_event(self, event: Event) -> None: print(f"[agent] {name}: {dump}", file=sys.stderr, flush=True) -# --------------------------------------------------------------------------- -# Conversation callback — collects all event dumps for parsing -# --------------------------------------------------------------------------- -collected_dumps: list[str] = [] - - -def capture_event(event: Event) -> None: - """Capture every event for post-run parsing.""" - collected_dumps.append(event.model_dump_json()) - - # --------------------------------------------------------------------------- # Build the user prompt # --------------------------------------------------------------------------- -def build_prompt(changed_files: list[str]) -> str: +def build_prompt(changed_files: list[str], output_path: str) -> str: catalog_text = "\n".join( f" - {name}: {desc}" for name, desc in sorted(SPEC_CATALOG.items()) ) @@ -152,7 +136,7 @@ def build_prompt(changed_files: list[str]) -> str: You can use the terminal and file_editor tools to read any source files or test specs if you need to understand what the changed code does. Do NOT -modify any files. +modify any repository files. Available test specs (in tests/e2e/mock-llm/) and what they cover: {catalog_text} @@ -169,59 +153,13 @@ def build_prompt(changed_files: list[str]) -> str: root layout, core shared utilities) and could affect anything, return ALL spec filenames. -When you are done, call the `finish` tool with a message that contains -exactly this block: - -<{OUTPUT_TAG}> -{{"specs": ["spec-filename.spec.ts"], "reason": "one sentence explanation"}} -""" +When you have decided, write your result as a JSON file to: + {output_path} +The JSON must have exactly these keys: + {{"specs": ["spec-filename.spec.ts", ...], "reason": "one sentence explanation"}} -# --------------------------------------------------------------------------- -# Extract text from event dumps -# --------------------------------------------------------------------------- -def _collect_text(obj: object, out: list[str]) -> None: - """Recursively collect every string value named 'text' or 'message'.""" - if isinstance(obj, dict): - for key, val in obj.items(): - if key in ("text", "message") and isinstance(val, str) and val.strip(): - out.append(val) - else: - _collect_text(val, out) - elif isinstance(obj, list): - for item in obj: - _collect_text(item, out) - - -def extract_all_text(dumps: list[str]) -> str: - """Walk every collected event JSON and pull out all text fragments.""" - fragments: list[str] = [] - for raw in dumps: - try: - obj = json.loads(raw) - except json.JSONDecodeError: - continue - _collect_text(obj, fragments) - return "\n".join(fragments) - - -# --------------------------------------------------------------------------- -# Parse the structured output from the agent's response -# --------------------------------------------------------------------------- -def parse_selection(text: str) -> dict: - pattern = rf"<{OUTPUT_TAG}>\s*(.*?)\s*" - matches = re.findall(pattern, text, re.DOTALL) - if not matches: - raise ValueError( - f"Agent response missing <{OUTPUT_TAG}> block.\n" - f"Full extracted text ({len(text)} chars):\n{text[:2000]}" - ) - # Take the LAST match — earlier ones may be the prompt template. - raw = matches[-1].strip() - result = json.loads(raw) - specs = [s for s in result.get("specs", []) if s in SPEC_CATALOG] - reason = result.get("reason", "LLM selection") - return {"specs": specs, "reason": reason} +Then call the `finish` tool.""" # --------------------------------------------------------------------------- @@ -245,6 +183,13 @@ def main() -> None: model = os.environ.get("LLM_MODEL", "openhands/gpt-5.1") workspace = os.environ.get("WORKSPACE", os.getcwd()) + # The agent writes its selection here; we read it after the run. + output_file = tempfile.NamedTemporaryFile( + prefix="e2e-selection-", suffix=".json", delete=False + ) + output_path = output_file.name + output_file.close() + llm = LLM( model=model, api_key=SecretStr(api_key), @@ -263,20 +208,30 @@ def main() -> None: agent=agent, workspace=workspace, visualizer=CIVisualizer(), - callbacks=[capture_event], max_iteration_per_run=30, ) - prompt = build_prompt(changed_files) + prompt = build_prompt(changed_files, output_path) conversation.send_message(prompt) conversation.run() - # Extract text from all events and parse the structured output. - all_text = extract_all_text(collected_dumps) - result = parse_selection(all_text) - result["mode"] = "llm" + # Read the agent's output file. + try: + with open(output_path) as f: + result = json.load(f) + except (FileNotFoundError, json.JSONDecodeError) as e: + raise RuntimeError( + f"Agent did not write valid JSON to {output_path}: {e}" + ) from e + finally: + if os.path.exists(output_path): + os.unlink(output_path) + + # Validate specs against the catalog. + specs = [s for s in result.get("specs", []) if s in SPEC_CATALOG] + reason = result.get("reason", "LLM selection") - print(json.dumps(result, indent=2)) + print(json.dumps({"specs": specs, "reason": reason, "mode": "llm"}, indent=2)) cost = llm.metrics.accumulated_cost print(f"LLM cost: ${cost:.4f}", file=sys.stderr) From 8d1976bd6bee4810375f52737fc310951b9b03bf Mon Sep 17 00:00:00 2001 From: openhands Date: Mon, 8 Jun 2026 21:56:13 +0000 Subject: [PATCH 10/10] refactor: auto-discover specs from repo instead of hardcoded catalog Replaced the 14-entry SPEC_CATALOG dict with discover_specs() which globs tests/e2e/mock-llm/*.spec.ts at runtime. The agent gets the list of available spec filenames and is told to read their source to understand what each one tests. New specs are picked up automatically without any script changes. Co-authored-by: openhands --- scripts/select-e2e-tests.py | 130 +++++++++++++----------------------- 1 file changed, 46 insertions(+), 84 deletions(-) diff --git a/scripts/select-e2e-tests.py b/scripts/select-e2e-tests.py index b7a3b829f..ee86e97ab 100644 --- a/scripts/select-e2e-tests.py +++ b/scripts/select-e2e-tests.py @@ -2,10 +2,12 @@ """Select which mock-LLM E2E test specs to run based on changed files. Spins up an OpenHands SDK Agent with terminal + file_editor tools, -pointed at the checked-out repository. The agent reads source files -and test specs as needed, then writes its selection to a JSON file. -Agent events are streamed to stderr for CI visibility; the final -JSON result is printed to stdout. +pointed at the checked-out repository. The agent discovers available +test specs by reading the repo, inspects changed source files as +needed, then writes its selection to a JSON file. + +No hardcoded spec catalog — the agent finds and reads specs at runtime, +so newly added tests are picked up automatically. Usage: # From the repo root, pipe changed file paths: @@ -28,6 +30,7 @@ from __future__ import annotations +import glob import json import logging import os @@ -44,69 +47,7 @@ # Suppress noisy SDK / litellm logs — our visualizer handles output. logging.getLogger().setLevel(logging.WARNING) -# --------------------------------------------------------------------------- -# Spec catalog – the agent receives this so it can reason about coverage. -# --------------------------------------------------------------------------- -SPEC_CATALOG: dict[str, str] = { - "mock-llm-acp-agent.spec.ts": ( - "ACP (Agent Client Protocol) agent configuration via Settings UI, " - "ACP conversation lifecycle, agent_kind=acp payload." - ), - "mock-llm-auth-modes.spec.ts": ( - "Session API key injection, key rotation recovery, public-mode " - "auth gate (ApiKeyEntryScreen), localStorage key sync." - ), - "mock-llm-automation.spec.ts": ( - "Full automation lifecycle: create cron automation via terminal " - "curl, dispatch a run, verify automation list/detail pages, " - "automation backend integration." - ), - "mock-llm-conversation.spec.ts": ( - "Core conversation flow: LLM profile creation, settings API, " - "terminal tool call, bash execution, agent reply, sidebar resume." - ), - "mock-llm-cross-connect.spec.ts": ( - "Frontend-only ↔ backend-only cross-connect, multi-backend " - "switching, manage-backends modal, backend registry." - ), - "mock-llm-image-upload.spec.ts": ( - "Image attachment via file input, base64 encoding in LLM " - "completion payload, image_urls in user message event." - ), - "mock-llm-model-switch.spec.ts": ( - "/model slash command mid-conversation, LLM profile switching, " - "switchLLM API, chat header profile display." - ), - "mock-llm-onboarding-happy-path.spec.ts": ( - "Full onboarding wizard: agent selection, backend check, LLM " - "setup, hello message. OnboardingModal flow." - ), - "mock-llm-onboarding-regressions.spec.ts": ( - "Onboarding edge cases: modal dismiss behavior, default model " - "selection, backdrop/Escape handling." - ), - "mock-llm-partial-stack.spec.ts": ( - "Partial stack modes: --frontend-only (503 for backend), " - "--backend-only (503 for frontend), port conflict detection. " - "bin/agent-canvas.mjs, static-server, ingress." - ), - "mock-llm-preset-automation.spec.ts": ( - "Preset automation cards, slash commands from home page, " - "skill activation via slash command." - ), - "mock-llm-profile-management.spec.ts": ( - "Active profile deletion + reconciliation, same-model profile " - "identity, litellm_proxy base_url preservation." - ), - "mock-llm-skills.spec.ts": ( - "Project skills (.agents/skills/), user skills (~/.openhands/skills/), " - "skill deletion, keyword-triggered activation." - ), - "mock-llm-ui-regressions.spec.ts": ( - "CSS isolation scoping, critic results rendering, event " - "pagination on scroll-up, workspace selection persistence." - ), -} +SPEC_DIR = "tests/e2e/mock-llm" # --------------------------------------------------------------------------- @@ -121,32 +62,43 @@ def on_event(self, event: Event) -> None: print(f"[agent] {name}: {dump}", file=sys.stderr, flush=True) +# --------------------------------------------------------------------------- +# Discover available spec files from the repo checkout +# --------------------------------------------------------------------------- +def discover_specs(workspace: str) -> list[str]: + pattern = os.path.join(workspace, SPEC_DIR, "*.spec.ts") + return sorted(os.path.basename(p) for p in glob.glob(pattern)) + + # --------------------------------------------------------------------------- # Build the user prompt # --------------------------------------------------------------------------- -def build_prompt(changed_files: list[str], output_path: str) -> str: - catalog_text = "\n".join( - f" - {name}: {desc}" for name, desc in sorted(SPEC_CATALOG.items()) - ) +def build_prompt( + changed_files: list[str], + available_specs: list[str], + output_path: str, +) -> str: + specs_text = "\n".join(f" - {s}" for s in available_specs) files_text = "\n".join(f" - {f}" for f in changed_files[:200]) return f"""\ You have access to the full repository checkout in your working directory. Your task: decide which E2E test specs should be run for this pull request. -You can use the terminal and file_editor tools to read any source files or -test specs if you need to understand what the changed code does. Do NOT -modify any repository files. +You SHOULD use the terminal and file_editor tools to read changed source +files and test specs to understand what they do. Do NOT modify any files. + +Available E2E test specs (in {SPEC_DIR}/): +{specs_text} -Available test specs (in tests/e2e/mock-llm/) and what they cover: -{catalog_text} +To understand what each spec tests, read its source — for example: + file_editor view {SPEC_DIR}/.spec.ts Files modified in this PR: {files_text} Rules: - Pick ONLY the specs whose covered areas are affected by the changed files. - If you are unsure whether a file is relevant, read it first. - If no spec is relevant (e.g. only docs, CI configs, or unit tests changed), return an empty "specs" list — we will skip E2E entirely. - If the changes are very broad (package.json, vite.config.ts, tsconfig, @@ -183,12 +135,22 @@ def main() -> None: model = os.environ.get("LLM_MODEL", "openhands/gpt-5.1") workspace = os.environ.get("WORKSPACE", os.getcwd()) + available_specs = discover_specs(workspace) + if not available_specs: + print(json.dumps({ + "specs": [], + "reason": f"No *.spec.ts files found in {SPEC_DIR}/.", + "mode": "llm", + })) + return + + print(f"Discovered {len(available_specs)} spec(s) in {SPEC_DIR}/", file=sys.stderr) + # The agent writes its selection here; we read it after the run. - output_file = tempfile.NamedTemporaryFile( - prefix="e2e-selection-", suffix=".json", delete=False + output_fd, output_path = tempfile.mkstemp( + prefix="e2e-selection-", suffix=".json" ) - output_path = output_file.name - output_file.close() + os.close(output_fd) llm = LLM( model=model, @@ -211,7 +173,7 @@ def main() -> None: max_iteration_per_run=30, ) - prompt = build_prompt(changed_files, output_path) + prompt = build_prompt(changed_files, available_specs, output_path) conversation.send_message(prompt) conversation.run() @@ -227,8 +189,8 @@ def main() -> None: if os.path.exists(output_path): os.unlink(output_path) - # Validate specs against the catalog. - specs = [s for s in result.get("specs", []) if s in SPEC_CATALOG] + # Validate specs — only keep filenames that actually exist. + specs = [s for s in result.get("specs", []) if s in available_specs] reason = result.get("reason", "LLM selection") print(json.dumps({"specs": specs, "reason": reason, "mode": "llm"}, indent=2))