feat: agent-powered E2E test subset selection for PRs

.github/workflows/agent-e2e-selector.yml

@@ -0,0 +1,161 @@
+# Agent-powered E2E test selector.
+#
+# Uses the OpenHands SDK LLM to analyze changed files in a PR and
+# determine which mock-LLM E2E test specs are most relevant, then
+# calls the Mock-LLM E2E Tests workflow (via workflow_call) with only
+# that subset.
+#
+# This is the primary entry point for PR-driven E2E testing.
+# mock-llm-e2e.yml no longer triggers on pull_request directly;
+# it only runs when called from here (workflow_call) or via manual
+# dispatch (workflow_dispatch).
+
+name: "Agent E2E Selector"
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+  workflow_dispatch:
+    inputs:
+      pr_number:
+        description: "PR number to analyze and run E2E subset for"
+        required: true
+        type: number
+
+permissions:
+  contents: read
+  pull-requests: write
+
+concurrency:
+  group: agent-e2e-selector-${{ github.event.pull_request.number || inputs.pr_number }}
+  cancel-in-progress: true
+
+jobs:
+  # ── Phase 1: analyze changed files and pick the test subset ──
+  select:
+    # Skip fork PRs — secrets are not available.
+    if: >-
+      github.event_name == 'workflow_dispatch' ||
+      github.event.pull_request.head.repo.full_name == github.repository
+    runs-on: ubuntu-24.04
+    timeout-minutes: 10
+
+    outputs:
+      specs: ${{ steps.selector.outputs.specs }}
+      reason: ${{ steps.selector.outputs.reason }}
+      mode: ${{ steps.selector.outputs.mode }}
+      file_count: ${{ steps.changed_files.outputs.count }}
+
+    env:
+      EFFECTIVE_PR: ${{ github.event.pull_request.number || inputs.pr_number }}
+
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v6
+
+      - name: Read defaults from config/defaults.json
+        id: defaults
+        run: |
+          echo "agent_server_version=$(node -p "require('./config/defaults.json').versions.agentServer")" >> "$GITHUB_OUTPUT"
+
+      - name: Install uv
+        run: |
+          curl -LsSf https://astral.sh/uv/install.sh | sh
+          echo "$HOME/.local/bin" >> "$GITHUB_PATH"
+
+      - name: Install openhands-sdk + tools
+        env:
+          AGENT_SERVER_VERSION: ${{ steps.defaults.outputs.agent_server_version }}
+        run: |
+          uv venv .selector-venv
+          uv pip install -p .selector-venv \
+            "openhands-sdk==$AGENT_SERVER_VERSION" \
+            "openhands-tools==$AGENT_SERVER_VERSION"
+
+      - name: Get changed files from PR
+        id: changed_files
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: |
+          FILES=$(gh api \
+            "/repos/${{ github.repository }}/pulls/${{ env.EFFECTIVE_PR }}/files" \
+            --paginate \
+            --jq '.[].filename')
+          echo "$FILES" > changed_files.txt
+          echo "count=$(echo "$FILES" | wc -l)" >> "$GITHUB_OUTPUT"
+          echo "Changed files ($(echo "$FILES" | wc -l)):"
+          echo "$FILES" | head -50
+
+      - name: Run test selector
+        id: selector
+        env:
+          LLM_API_KEY: ${{ secrets.LLM_API_KEY }}
+          LLM_BASE_URL: ${{ vars.LIVE_E2E_LLM_BASE_URL || 'https://llm-proxy.app.all-hands.dev' }}
+          LLM_MODEL: "openhands/gpt-5.1"
+          OPENHANDS_SUPPRESS_BANNER: "1"
+        run: |
+          RESULT=$(.selector-venv/bin/python3 scripts/select-e2e-tests.py < changed_files.txt)
+          echo "$RESULT" | python3 -m json.tool
+          echo "$RESULT" > selector_result.json
+
+          # Parse outputs for downstream use.
+          SPECS=$(echo "$RESULT" | python3 -c "import sys,json; d=json.load(sys.stdin); print(','.join(d.get('specs',[])))")
+          REASON=$(echo "$RESULT" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('reason',''))")
+          MODE=$(echo "$RESULT" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('mode','full'))")
+
+          echo "specs=$SPECS" >> "$GITHUB_OUTPUT"
+          echo "reason=$REASON" >> "$GITHUB_OUTPUT"
+          echo "mode=$MODE" >> "$GITHUB_OUTPUT"
+
+      - name: Post selection summary to PR
+        continue-on-error: true
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: |
+          MODE="${{ steps.selector.outputs.mode }}"
+          SPECS="${{ steps.selector.outputs.specs }}"
+          REASON="${{ steps.selector.outputs.reason }}"
+          FILE_COUNT="${{ steps.changed_files.outputs.count }}"
+
+          if [ "$MODE" = "full" ]; then
+            SPEC_DISPLAY="**Full suite** (all specs)"
+          else
+            SPEC_DISPLAY=$(echo "$SPECS" | tr ',' '\n' | sed 's/^/- `/' | sed 's/$/`/')
+          fi
+
+          BODY=$(cat <<'COMMENT_EOF'
+          <!-- agent-e2e-selector -->
+          ### 🤖 Agent E2E Test Selector
+
+          | | |
+          |---|---|
+          | **Mode** | `MODE_PLACEHOLDER` |
+          | **Files analyzed** | FILE_COUNT_PLACEHOLDER |
+          | **Reason** | REASON_PLACEHOLDER |
+
+          **Selected specs:**
+          SPECS_PLACEHOLDER
+
+          _Running via: [Mock-LLM E2E Tests](/${{ github.repository }}/actions/runs/${{ github.run_id }})_
+          COMMENT_EOF
+          )
+
+          BODY="${BODY//MODE_PLACEHOLDER/$MODE}"
+          BODY="${BODY//FILE_COUNT_PLACEHOLDER/$FILE_COUNT}"
+          BODY="${BODY//REASON_PLACEHOLDER/$REASON}"
+          BODY="${BODY//SPECS_PLACEHOLDER/$SPEC_DISPLAY}"
+
+          echo "$BODY" | gh pr comment "${{ env.EFFECTIVE_PR }}" --body-file -
+
+  # ── Phase 2: run the E2E suite via workflow_call ──
+  # Skipped entirely when the agent returns no specs (docs/CI-only changes).
+  run-e2e:
+    needs: select
+    if: needs.select.outputs.specs != ''
+    uses: ./.github/workflows/mock-llm-e2e.yml
+    with:
+      test_specs: ${{ needs.select.outputs.specs }}
+      pr_number: ${{ github.event.pull_request.number && format('{0}', github.event.pull_request.number) || format('{0}', inputs.pr_number) }}
+    permissions:
+      contents: read
+      pull-requests: write

.github/workflows/mock-llm-e2e.yml

@@ -1,12 +1,51 @@
 name: Mock-LLM E2E Tests
 
 on:
-  pull_request:
-    types: [opened, synchronize, reopened]
   workflow_dispatch:
+    inputs:
+      test_specs:
+        description: >-
+          Comma-separated spec filenames to run (e.g.
+          "mock-llm-conversation.spec.ts,mock-llm-automation.spec.ts").
+          Leave empty to run the full suite.
+        required: false
+        default: ""
+      test_grep:
+        description: >-
+          Playwright --grep pattern to filter tests by title
+          (e.g. "conversation|automation"). Leave empty for no filter.
+        required: false
+        default: ""
+      pr_number:
+        description: >-
+          PR number for comment posting when triggered externally
+          (e.g. by the agent-e2e-selector workflow). Leave empty for
+          non-PR dispatches.
+        required: false
+        default: ""
+  # Reusable workflow entry point — called by agent-e2e-selector.yml.
+  # Inputs mirror workflow_dispatch so the job body can use
+  # `inputs.*` regardless of which trigger fired.
+  workflow_call:
+    inputs:
+      test_specs:
+        description: "Comma-separated spec filenames to run."
+        required: false
+        type: string
+        default: ""
+      test_grep:
+        description: "Playwright --grep pattern."
+        required: false
+        type: string
+        default: ""
+      pr_number:
+        description: "PR number for comment posting."
+        required: false
+        type: string
+        default: ""
 
 concurrency:
-  group: mock-llm-e2e-${{ github.event.pull_request.number || github.ref }}
+  group: mock-llm-e2e-${{ github.event.pull_request.number || inputs.pr_number || github.ref }}
   cancel-in-progress: true
 
 permissions:
@@ -21,6 +60,8 @@ jobs:
     env:
       MOCK_LLM_REPORT_PATH: mock-llm-report.md
       MOCK_LLM_WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+      # Resolve the effective PR number from either trigger type.
+      EFFECTIVE_PR_NUMBER: ${{ github.event.pull_request.number || inputs.pr_number || '' }}
 
     steps:
       - name: Check out repository
@@ -106,6 +147,31 @@ jobs:
       - name: Build frontend (for agent-canvas binary)
         run: npm run build:app
 
+      - name: Build Playwright command
+        id: pw_command
+        run: |
+          CMD="npx playwright test --config=playwright.mock-llm.config.ts"
+
+          # Append spec file filters from workflow_dispatch input.
+          TEST_SPECS="${{ inputs.test_specs }}"
+          if [ -n "$TEST_SPECS" ]; then
+            # Convert comma-separated filenames to space-separated paths.
+            for spec in $(echo "$TEST_SPECS" | tr ',' ' '); do
+              CMD="$CMD tests/e2e/mock-llm/$spec"
+            done
+            echo "::notice::Running subset: $TEST_SPECS"
+          fi
+
+          # Append --grep filter from workflow_dispatch input.
+          TEST_GREP="${{ inputs.test_grep }}"
+          if [ -n "$TEST_GREP" ]; then
+            CMD="$CMD --grep \"$TEST_GREP\""
+            echo "::notice::Grep filter: $TEST_GREP"
+          fi
+
+          echo "command=$CMD" >> "$GITHUB_OUTPUT"
+          echo "Playwright command: $CMD"
+
       - name: Run mock-LLM E2E tests
         id: run_tests
         env:
@@ -119,7 +185,7 @@ jobs:
 
           # Run Playwright in background so our shell survives if we have
           # to kill it (the webServer teardown can hang indefinitely).
-          npm run test:e2e:mock-llm &
+          ${{ steps.pw_command.outputs.command }} &
           PW_PID=$!
 
           # Wait for tests to complete. Playwright's globalTimeout is 600s
@@ -187,15 +253,15 @@ jobs:
             test-results-mock-llm/
 
       - name: Detect newly added spec files
-        if: always() && github.event.pull_request.number
+        if: always() && env.EFFECTIVE_PR_NUMBER
         id: new_specs
         env:
           GITHUB_TOKEN: ${{ github.token }}
         run: |
           # Find mock-LLM spec files added (not just modified) in this PR
           # Uses the GitHub API instead of git diff to avoid shallow-clone issues
           NEW_FILES=$(gh api \
-            "/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/files" \
+            "/repos/${{ github.repository }}/pulls/${{ env.EFFECTIVE_PR_NUMBER }}/files" \
             --paginate \
             --jq '[.[] | select(.status == "added") | .filename
                    | select(test("tests/e2e/mock-llm/.*\\.spec\\.ts$"))]
@@ -215,11 +281,11 @@ jobs:
           cat "$MOCK_LLM_REPORT_PATH" >> "$GITHUB_STEP_SUMMARY"
 
       - name: Save PR number for comment workflow
-        if: always() && github.event.pull_request.number
-        run: echo "${{ github.event.pull_request.number }}" > pr_number.txt
+        if: always() && env.EFFECTIVE_PR_NUMBER
+        run: echo "${{ env.EFFECTIVE_PR_NUMBER }}" > pr_number.txt
 
       - name: Upload PR comment payload
-        if: always() && github.event.pull_request.number
+        if: always() && env.EFFECTIVE_PR_NUMBER
         uses: actions/upload-artifact@v7
         with:
           name: mock-llm-pr-comment-payload
@@ -231,13 +297,14 @@ jobs:
       - name: Post PR comment (same-repo PRs only)
         if: >-
           always() &&
-          github.event.pull_request.number &&
-          github.event.pull_request.head.repo.full_name == github.repository
+          env.EFFECTIVE_PR_NUMBER &&
+          (github.event.pull_request.head.repo.full_name == github.repository ||
+           github.event_name == 'workflow_dispatch')
         continue-on-error: true
         env:
           GITHUB_TOKEN: ${{ github.token }}
         run: |
-          gh pr comment "${{ github.event.pull_request.number }}" \
+          gh pr comment "${{ env.EFFECTIVE_PR_NUMBER }}" \
             --body-file "$MOCK_LLM_REPORT_PATH"
 
       - name: Fail job when tests fail