diff --git a/.github/workflows/agent-e2e-selector.yml b/.github/workflows/agent-e2e-selector.yml new file mode 100644 index 000000000..1fce3943a --- /dev/null +++ b/.github/workflows/agent-e2e-selector.yml @@ -0,0 +1,161 @@ +# Agent-powered E2E test selector. +# +# Uses the OpenHands SDK LLM to analyze changed files in a PR and +# determine which mock-LLM E2E test specs are most relevant, then +# calls the Mock-LLM E2E Tests workflow (via workflow_call) with only +# that subset. +# +# This is the primary entry point for PR-driven E2E testing. +# mock-llm-e2e.yml no longer triggers on pull_request directly; +# it only runs when called from here (workflow_call) or via manual +# dispatch (workflow_dispatch). + +name: "Agent E2E Selector" + +on: + pull_request: + types: [opened, synchronize, reopened] + workflow_dispatch: + inputs: + pr_number: + description: "PR number to analyze and run E2E subset for" + required: true + type: number + +permissions: + contents: read + pull-requests: write + +concurrency: + group: agent-e2e-selector-${{ github.event.pull_request.number || inputs.pr_number }} + cancel-in-progress: true + +jobs: + # ── Phase 1: analyze changed files and pick the test subset ── + select: + # Skip fork PRs — secrets are not available. + if: >- + github.event_name == 'workflow_dispatch' || + github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-24.04 + timeout-minutes: 10 + + outputs: + specs: ${{ steps.selector.outputs.specs }} + reason: ${{ steps.selector.outputs.reason }} + mode: ${{ steps.selector.outputs.mode }} + file_count: ${{ steps.changed_files.outputs.count }} + + env: + EFFECTIVE_PR: ${{ github.event.pull_request.number || inputs.pr_number }} + + steps: + - name: Check out repository + uses: actions/checkout@v6 + + - name: Read defaults from config/defaults.json + id: defaults + run: | + echo "agent_server_version=$(node -p "require('./config/defaults.json').versions.agentServer")" >> "$GITHUB_OUTPUT" + + - name: Install uv + run: | + curl -LsSf https://astral.sh/uv/install.sh | sh + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + + - name: Install openhands-sdk + tools + env: + AGENT_SERVER_VERSION: ${{ steps.defaults.outputs.agent_server_version }} + run: | + uv venv .selector-venv + uv pip install -p .selector-venv \ + "openhands-sdk==$AGENT_SERVER_VERSION" \ + "openhands-tools==$AGENT_SERVER_VERSION" + + - name: Get changed files from PR + id: changed_files + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + FILES=$(gh api \ + "/repos/${{ github.repository }}/pulls/${{ env.EFFECTIVE_PR }}/files" \ + --paginate \ + --jq '.[].filename') + echo "$FILES" > changed_files.txt + echo "count=$(echo "$FILES" | wc -l)" >> "$GITHUB_OUTPUT" + echo "Changed files ($(echo "$FILES" | wc -l)):" + echo "$FILES" | head -50 + + - name: Run test selector + id: selector + env: + LLM_API_KEY: ${{ secrets.LLM_API_KEY }} + LLM_BASE_URL: ${{ vars.LIVE_E2E_LLM_BASE_URL || 'https://llm-proxy.app.all-hands.dev' }} + LLM_MODEL: "openhands/gpt-5.1" + OPENHANDS_SUPPRESS_BANNER: "1" + run: | + RESULT=$(.selector-venv/bin/python3 scripts/select-e2e-tests.py < changed_files.txt) + echo "$RESULT" | python3 -m json.tool + echo "$RESULT" > selector_result.json + + # Parse outputs for downstream use. + SPECS=$(echo "$RESULT" | python3 -c "import sys,json; d=json.load(sys.stdin); print(','.join(d.get('specs',[])))") + REASON=$(echo "$RESULT" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('reason',''))") + MODE=$(echo "$RESULT" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('mode','full'))") + + echo "specs=$SPECS" >> "$GITHUB_OUTPUT" + echo "reason=$REASON" >> "$GITHUB_OUTPUT" + echo "mode=$MODE" >> "$GITHUB_OUTPUT" + + - name: Post selection summary to PR + continue-on-error: true + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + MODE="${{ steps.selector.outputs.mode }}" + SPECS="${{ steps.selector.outputs.specs }}" + REASON="${{ steps.selector.outputs.reason }}" + FILE_COUNT="${{ steps.changed_files.outputs.count }}" + + if [ "$MODE" = "full" ]; then + SPEC_DISPLAY="**Full suite** (all specs)" + else + SPEC_DISPLAY=$(echo "$SPECS" | tr ',' '\n' | sed 's/^/- `/' | sed 's/$/`/') + fi + + BODY=$(cat <<'COMMENT_EOF' + + ### 🤖 Agent E2E Test Selector + + | | | + |---|---| + | **Mode** | `MODE_PLACEHOLDER` | + | **Files analyzed** | FILE_COUNT_PLACEHOLDER | + | **Reason** | REASON_PLACEHOLDER | + + **Selected specs:** + SPECS_PLACEHOLDER + + _Running via: [Mock-LLM E2E Tests](/${{ github.repository }}/actions/runs/${{ github.run_id }})_ + COMMENT_EOF + ) + + BODY="${BODY//MODE_PLACEHOLDER/$MODE}" + BODY="${BODY//FILE_COUNT_PLACEHOLDER/$FILE_COUNT}" + BODY="${BODY//REASON_PLACEHOLDER/$REASON}" + BODY="${BODY//SPECS_PLACEHOLDER/$SPEC_DISPLAY}" + + echo "$BODY" | gh pr comment "${{ env.EFFECTIVE_PR }}" --body-file - + + # ── Phase 2: run the E2E suite via workflow_call ── + # Skipped entirely when the agent returns no specs (docs/CI-only changes). + run-e2e: + needs: select + if: needs.select.outputs.specs != '' + uses: ./.github/workflows/mock-llm-e2e.yml + with: + test_specs: ${{ needs.select.outputs.specs }} + pr_number: ${{ github.event.pull_request.number && format('{0}', github.event.pull_request.number) || format('{0}', inputs.pr_number) }} + permissions: + contents: read + pull-requests: write diff --git a/.github/workflows/mock-llm-e2e.yml b/.github/workflows/mock-llm-e2e.yml index 11c2dd244..777895334 100644 --- a/.github/workflows/mock-llm-e2e.yml +++ b/.github/workflows/mock-llm-e2e.yml @@ -1,12 +1,51 @@ name: Mock-LLM E2E Tests on: - pull_request: - types: [opened, synchronize, reopened] workflow_dispatch: + inputs: + test_specs: + description: >- + Comma-separated spec filenames to run (e.g. + "mock-llm-conversation.spec.ts,mock-llm-automation.spec.ts"). + Leave empty to run the full suite. + required: false + default: "" + test_grep: + description: >- + Playwright --grep pattern to filter tests by title + (e.g. "conversation|automation"). Leave empty for no filter. + required: false + default: "" + pr_number: + description: >- + PR number for comment posting when triggered externally + (e.g. by the agent-e2e-selector workflow). Leave empty for + non-PR dispatches. + required: false + default: "" + # Reusable workflow entry point — called by agent-e2e-selector.yml. + # Inputs mirror workflow_dispatch so the job body can use + # `inputs.*` regardless of which trigger fired. + workflow_call: + inputs: + test_specs: + description: "Comma-separated spec filenames to run." + required: false + type: string + default: "" + test_grep: + description: "Playwright --grep pattern." + required: false + type: string + default: "" + pr_number: + description: "PR number for comment posting." + required: false + type: string + default: "" concurrency: - group: mock-llm-e2e-${{ github.event.pull_request.number || github.ref }} + group: mock-llm-e2e-${{ github.event.pull_request.number || inputs.pr_number || github.ref }} cancel-in-progress: true permissions: @@ -21,6 +60,8 @@ jobs: env: MOCK_LLM_REPORT_PATH: mock-llm-report.md MOCK_LLM_WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + # Resolve the effective PR number from either trigger type. + EFFECTIVE_PR_NUMBER: ${{ github.event.pull_request.number || inputs.pr_number || '' }} steps: - name: Check out repository @@ -106,6 +147,31 @@ jobs: - name: Build frontend (for agent-canvas binary) run: npm run build:app + - name: Build Playwright command + id: pw_command + run: | + CMD="npx playwright test --config=playwright.mock-llm.config.ts" + + # Append spec file filters from workflow_dispatch input. + TEST_SPECS="${{ inputs.test_specs }}" + if [ -n "$TEST_SPECS" ]; then + # Convert comma-separated filenames to space-separated paths. + for spec in $(echo "$TEST_SPECS" | tr ',' ' '); do + CMD="$CMD tests/e2e/mock-llm/$spec" + done + echo "::notice::Running subset: $TEST_SPECS" + fi + + # Append --grep filter from workflow_dispatch input. + TEST_GREP="${{ inputs.test_grep }}" + if [ -n "$TEST_GREP" ]; then + CMD="$CMD --grep \"$TEST_GREP\"" + echo "::notice::Grep filter: $TEST_GREP" + fi + + echo "command=$CMD" >> "$GITHUB_OUTPUT" + echo "Playwright command: $CMD" + - name: Run mock-LLM E2E tests id: run_tests env: @@ -119,7 +185,7 @@ jobs: # Run Playwright in background so our shell survives if we have # to kill it (the webServer teardown can hang indefinitely). - npm run test:e2e:mock-llm & + ${{ steps.pw_command.outputs.command }} & PW_PID=$! # Wait for tests to complete. Playwright's globalTimeout is 600s @@ -187,7 +253,7 @@ jobs: test-results-mock-llm/ - name: Detect newly added spec files - if: always() && github.event.pull_request.number + if: always() && env.EFFECTIVE_PR_NUMBER id: new_specs env: GITHUB_TOKEN: ${{ github.token }} @@ -195,7 +261,7 @@ jobs: # Find mock-LLM spec files added (not just modified) in this PR # Uses the GitHub API instead of git diff to avoid shallow-clone issues NEW_FILES=$(gh api \ - "/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/files" \ + "/repos/${{ github.repository }}/pulls/${{ env.EFFECTIVE_PR_NUMBER }}/files" \ --paginate \ --jq '[.[] | select(.status == "added") | .filename | select(test("tests/e2e/mock-llm/.*\\.spec\\.ts$"))] @@ -215,11 +281,11 @@ jobs: cat "$MOCK_LLM_REPORT_PATH" >> "$GITHUB_STEP_SUMMARY" - name: Save PR number for comment workflow - if: always() && github.event.pull_request.number - run: echo "${{ github.event.pull_request.number }}" > pr_number.txt + if: always() && env.EFFECTIVE_PR_NUMBER + run: echo "${{ env.EFFECTIVE_PR_NUMBER }}" > pr_number.txt - name: Upload PR comment payload - if: always() && github.event.pull_request.number + if: always() && env.EFFECTIVE_PR_NUMBER uses: actions/upload-artifact@v7 with: name: mock-llm-pr-comment-payload @@ -231,13 +297,14 @@ jobs: - name: Post PR comment (same-repo PRs only) if: >- always() && - github.event.pull_request.number && - github.event.pull_request.head.repo.full_name == github.repository + env.EFFECTIVE_PR_NUMBER && + (github.event.pull_request.head.repo.full_name == github.repository || + github.event_name == 'workflow_dispatch') continue-on-error: true env: GITHUB_TOKEN: ${{ github.token }} run: | - gh pr comment "${{ github.event.pull_request.number }}" \ + gh pr comment "${{ env.EFFECTIVE_PR_NUMBER }}" \ --body-file "$MOCK_LLM_REPORT_PATH" - name: Fail job when tests fail diff --git a/scripts/select-e2e-tests.py b/scripts/select-e2e-tests.py new file mode 100644 index 000000000..ee86e97ab --- /dev/null +++ b/scripts/select-e2e-tests.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +"""Select which mock-LLM E2E test specs to run based on changed files. + +Spins up an OpenHands SDK Agent with terminal + file_editor tools, +pointed at the checked-out repository. The agent discovers available +test specs by reading the repo, inspects changed source files as +needed, then writes its selection to a JSON file. + +No hardcoded spec catalog — the agent finds and reads specs at runtime, +so newly added tests are picked up automatically. + +Usage: + # From the repo root, pipe changed file paths: + git diff --name-only origin/main | python scripts/select-e2e-tests.py + + # Or pass as arguments: + python scripts/select-e2e-tests.py src/routes/automations.tsx ... + +Environment variables: + LLM_API_KEY – required + LLM_BASE_URL – optional, defaults to https://llm-proxy.app.all-hands.dev + LLM_MODEL – optional, defaults to openhands/gpt-5.1 + WORKSPACE – optional, repo root (defaults to cwd) + +Output (stdout): JSON object with keys: + specs – list of spec filenames to run (empty ⇒ no E2E needed) + reason – human-readable explanation + mode – "llm" +""" + +from __future__ import annotations + +import glob +import json +import logging +import os +import sys +import tempfile + +from pydantic import SecretStr + +from openhands.sdk import LLM, Agent, Conversation, Event, Tool +from openhands.sdk.conversation.visualizer import ConversationVisualizerBase +from openhands.tools.file_editor import FileEditorTool +from openhands.tools.terminal import TerminalTool + +# Suppress noisy SDK / litellm logs — our visualizer handles output. +logging.getLogger().setLevel(logging.WARNING) + +SPEC_DIR = "tests/e2e/mock-llm" + + +# --------------------------------------------------------------------------- +# Visualizer — streams every agent event to stderr for CI visibility +# --------------------------------------------------------------------------- +class CIVisualizer(ConversationVisualizerBase): + """Prints agent events to stderr so CI logs show the full reasoning.""" + + def on_event(self, event: Event) -> None: + name = type(event).__name__ + dump = event.model_dump_json()[:800] + print(f"[agent] {name}: {dump}", file=sys.stderr, flush=True) + + +# --------------------------------------------------------------------------- +# Discover available spec files from the repo checkout +# --------------------------------------------------------------------------- +def discover_specs(workspace: str) -> list[str]: + pattern = os.path.join(workspace, SPEC_DIR, "*.spec.ts") + return sorted(os.path.basename(p) for p in glob.glob(pattern)) + + +# --------------------------------------------------------------------------- +# Build the user prompt +# --------------------------------------------------------------------------- +def build_prompt( + changed_files: list[str], + available_specs: list[str], + output_path: str, +) -> str: + specs_text = "\n".join(f" - {s}" for s in available_specs) + files_text = "\n".join(f" - {f}" for f in changed_files[:200]) + + return f"""\ +You have access to the full repository checkout in your working directory. +Your task: decide which E2E test specs should be run for this pull request. + +You SHOULD use the terminal and file_editor tools to read changed source +files and test specs to understand what they do. Do NOT modify any files. + +Available E2E test specs (in {SPEC_DIR}/): +{specs_text} + +To understand what each spec tests, read its source — for example: + file_editor view {SPEC_DIR}/.spec.ts + +Files modified in this PR: +{files_text} + +Rules: +- Pick ONLY the specs whose covered areas are affected by the changed files. +- If no spec is relevant (e.g. only docs, CI configs, or unit tests changed), + return an empty "specs" list — we will skip E2E entirely. +- If the changes are very broad (package.json, vite.config.ts, tsconfig, + root layout, core shared utilities) and could affect anything, return + ALL spec filenames. + +When you have decided, write your result as a JSON file to: + {output_path} + +The JSON must have exactly these keys: + {{"specs": ["spec-filename.spec.ts", ...], "reason": "one sentence explanation"}} + +Then call the `finish` tool.""" + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- +def main() -> None: + if len(sys.argv) > 1: + changed_files = sys.argv[1:] + else: + changed_files = [line.strip() for line in sys.stdin if line.strip()] + + if not changed_files: + print(json.dumps({"specs": [], "reason": "No changed files provided.", "mode": "llm"})) + return + + api_key = os.environ.get("LLM_API_KEY", "") + if not api_key: + raise RuntimeError("LLM_API_KEY is required but not set.") + + base_url = os.environ.get("LLM_BASE_URL", "https://llm-proxy.app.all-hands.dev") + model = os.environ.get("LLM_MODEL", "openhands/gpt-5.1") + workspace = os.environ.get("WORKSPACE", os.getcwd()) + + available_specs = discover_specs(workspace) + if not available_specs: + print(json.dumps({ + "specs": [], + "reason": f"No *.spec.ts files found in {SPEC_DIR}/.", + "mode": "llm", + })) + return + + print(f"Discovered {len(available_specs)} spec(s) in {SPEC_DIR}/", file=sys.stderr) + + # The agent writes its selection here; we read it after the run. + output_fd, output_path = tempfile.mkstemp( + prefix="e2e-selection-", suffix=".json" + ) + os.close(output_fd) + + llm = LLM( + model=model, + api_key=SecretStr(api_key), + base_url=base_url, + usage_id="e2e-selector", + ) + agent = Agent( + llm=llm, + tools=[ + Tool(name=TerminalTool.name), + Tool(name=FileEditorTool.name), + ], + ) + + conversation = Conversation( + agent=agent, + workspace=workspace, + visualizer=CIVisualizer(), + max_iteration_per_run=30, + ) + + prompt = build_prompt(changed_files, available_specs, output_path) + conversation.send_message(prompt) + conversation.run() + + # Read the agent's output file. + try: + with open(output_path) as f: + result = json.load(f) + except (FileNotFoundError, json.JSONDecodeError) as e: + raise RuntimeError( + f"Agent did not write valid JSON to {output_path}: {e}" + ) from e + finally: + if os.path.exists(output_path): + os.unlink(output_path) + + # Validate specs — only keep filenames that actually exist. + specs = [s for s in result.get("specs", []) if s in available_specs] + reason = result.get("reason", "LLM selection") + + print(json.dumps({"specs": specs, "reason": reason, "mode": "llm"}, indent=2)) + + cost = llm.metrics.accumulated_cost + print(f"LLM cost: ${cost:.4f}", file=sys.stderr) + + +if __name__ == "__main__": + main()