Repository navigation
perf: remove spare Chrome renderers and preserve feature overrides #10539
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| pull_request: | |
| types: | |
| - opened | |
| - reopened | |
| - synchronize | |
| - labeled | |
| - unlabeled | |
| workflow_dispatch: | |
| inputs: | |
| pull_request_number: | |
| description: Internal mirrored pull request number | |
| required: true | |
| type: string | |
| permissions: | |
| contents: read | |
| pull-requests: read | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| determine-changes: | |
| name: Determine changes | |
| runs-on: ubuntu-latest | |
| if: >- | |
| !contains(fromJSON('["labeled", "unlabeled"]'), github.event.action) || | |
| contains(fromJSON('["skip-evals", "skip-regression-evals"]'), github.event.label.name) | |
| outputs: | |
| docs-only: ${{ steps.docs-filter.outputs.docs-only }} | |
| evals: ${{ steps.filter.outputs.evals }} | |
| extension: ${{ steps.filter.outputs.extension }} | |
| is_internal_head: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} | |
| sdk-ts: ${{ steps.filter.outputs.sdk-ts }} | |
| sdk-python: ${{ steps.filter.outputs.sdk-python }} | |
| deepagents-runner: ${{ steps.filter.outputs.deepagents-runner }} | |
| sdk-go: ${{ steps.filter.outputs.sdk-go }} | |
| steps: | |
| - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 | |
| - uses: dorny/paths-filter@d1c1ffe0248fe513906c8e24db8ea791d46f8590 # v3.0.3 | |
| id: filter | |
| with: | |
| filters: | | |
| evals: | |
| - 'packages/evals/**' | |
| - 'packages/sdk-ts/**' | |
| - 'packages/extension/**' | |
| - 'packages/protocol/**' | |
| - 'package.json' | |
| - 'pnpm-lock.yaml' | |
| - 'pnpm-workspace.yaml' | |
| - 'turbo.json' | |
| - '.github/**' | |
| extension: | |
| - 'packages/extension/**' | |
| - 'packages/protocol/**' | |
| - 'pnpm-lock.yaml' | |
| - 'turbo.json' | |
| - '.github/**' | |
| sdk-ts: | |
| - 'packages/sdk-ts/**' | |
| - 'packages/extension/**' | |
| - 'packages/protocol/**' | |
| - 'packages/integrations/**' | |
| - 'pnpm-lock.yaml' | |
| - 'turbo.json' | |
| - '.github/**' | |
| sdk-python: | |
| - 'packages/sdk-python/**' | |
| - 'packages/extension/**' | |
| - 'packages/protocol/**' | |
| - 'pnpm-lock.yaml' | |
| - '.github/**' | |
| deepagents-runner: | |
| - 'packages/integrations/deepagents/runner/**' | |
| - '.github/**' | |
| sdk-go: | |
| - 'packages/sdk-go/**' | |
| - 'packages/extension/**' | |
| - 'packages/protocol/**' | |
| - '.github/**' | |
| - uses: dorny/paths-filter@d1c1ffe0248fe513906c8e24db8ea791d46f8590 # v3.0.3 | |
| id: docs-filter | |
| with: | |
| predicate-quantifier: every | |
| filters: | | |
| docs-only: | |
| - '**/*.md' | |
| - 'examples/**' | |
| - '!packages/**/*.md' | |
| load-pr-context: | |
| name: Load PR context | |
| runs-on: ubuntu-latest | |
| if: >- | |
| !contains(fromJSON('["labeled", "unlabeled"]'), github.event.action) || | |
| contains(fromJSON('["skip-evals", "skip-regression-evals"]'), github.event.label.name) | |
| outputs: | |
| is-internal-head: ${{ steps.load.outputs.is-internal-head }} | |
| skip-evals: ${{ steps.load.outputs.skip-evals }} | |
| skip-regression-evals: ${{ steps.load.outputs.skip-regression-evals }} | |
| steps: | |
| - id: load | |
| uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 | |
| env: | |
| PR_NUMBER: ${{ github.event.inputs.pull_request_number }} | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| script: | | |
| const repoFullName = `${context.repo.owner}/${context.repo.repo}`; | |
| let pr = context.payload.pull_request; | |
| if (!pr) { | |
| const prNumber = Number(process.env.PR_NUMBER); | |
| if (!prNumber) { | |
| throw new Error('workflow_dispatch requires pull_request_number input'); | |
| } | |
| const { data } = await github.rest.pulls.get({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| pull_number: prNumber, | |
| }); | |
| pr = data; | |
| } | |
| const labels = new Set( | |
| (pr.labels || []) | |
| .map((label) => typeof label === 'string' ? label : label.name) | |
| .filter(Boolean), | |
| ); | |
| core.setOutput('is-internal-head', pr.head.repo?.full_name === repoFullName ? 'true' : 'false'); | |
| core.setOutput('skip-evals', labels.has('skip-evals') ? 'true' : 'false'); | |
| core.setOutput( | |
| 'skip-regression-evals', | |
| labels.has('skip-regression-evals') ? 'true' : 'false', | |
| ); | |
| docs: | |
| name: Docs validation | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| needs: [determine-changes] | |
| steps: | |
| - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 | |
| - uses: ./.github/actions/setup-node-pnpm | |
| with: | |
| use-prebuilt-artifacts: "false" | |
| # Run directly on every PR, including docs-only and Python/Go-only changes. | |
| # The checked-out SDK and docs must describe the same API on either branch. | |
| - run: pnpm --filter ./packages/protocol build | |
| - run: git diff --exit-code -- packages/protocol/stagehand.v4.json | |
| - run: pnpm --filter ./packages/docs test:unit | |
| - run: pnpm --filter ./packages/docs check | |
| check: | |
| name: Check | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| needs: [determine-changes] | |
| steps: | |
| - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 | |
| - uses: ./.github/actions/setup-node-pnpm | |
| with: | |
| use-prebuilt-artifacts: "false" | |
| - name: Set up Python | |
| uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 | |
| with: | |
| python-version-file: packages/sdk-python/.python-version | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 | |
| with: | |
| version: "0.11.29" | |
| working-directory: packages/sdk-python | |
| enable-cache: true | |
| cache-dependency-glob: packages/sdk-python/uv.lock | |
| - name: Set up Go | |
| uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 | |
| with: | |
| go-version: "1.26.x" | |
| cache-dependency-path: packages/sdk-go/internal/generator/go.sum | |
| # Keep the legacy build command here so the generated protocol drift check | |
| # still compares freshly generated output with the committed schema. | |
| - run: pnpm build | |
| - run: git diff --exit-code -- packages/protocol/stagehand.v4.json | |
| - run: pnpm check | |
| - run: pnpm test:unit | |
| - run: pnpm --filter browse test:cli | |
| - run: uv lock --check | |
| working-directory: packages/sdk-python | |
| - run: uv run --locked python scripts/generate.py --check | |
| working-directory: packages/sdk-python | |
| - run: uv run --locked ruff format --check . | |
| working-directory: packages/sdk-python | |
| - run: uv run --locked ruff check . | |
| working-directory: packages/sdk-python | |
| - run: uv run --locked ty check | |
| working-directory: packages/sdk-python | |
| - run: go -C internal/generator mod download | |
| working-directory: packages/sdk-go | |
| - run: go -C internal/generator run . --check | |
| working-directory: packages/sdk-go | |
| - name: Check Go formatting | |
| run: test -z "$(gofmt -l $(find . -name '*.go' -type f))" | |
| working-directory: packages/sdk-go | |
| - run: go vet $(go list ./... | grep -v '/examples$') | |
| working-directory: packages/sdk-go | |
| - run: go -C internal/generator vet ./... | |
| working-directory: packages/sdk-go | |
| - name: Check Go examples | |
| run: sh scripts/check-examples.sh | |
| working-directory: packages/sdk-go | |
| cancel-after-check-failure: | |
| name: Cancel after check failure | |
| runs-on: ubuntu-latest | |
| # Scoped here rather than workflow-wide: this is the only job that needs to write to the | |
| # Actions API, and the fan-out means ~60 other jobs would otherwise inherit the privilege. | |
| permissions: | |
| contents: read | |
| actions: write | |
| needs: [check] | |
| if: failure() | |
| continue-on-error: true | |
| steps: | |
| - name: Cancel workflow run | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| curl -sSfL -X POST \ | |
| -H "Authorization: Bearer ${GITHUB_TOKEN}" \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "X-GitHub-Api-Version: 2022-11-28" \ | |
| "https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/cancel" | |
| build: | |
| name: Build | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| needs: [determine-changes] | |
| steps: | |
| - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 | |
| - uses: ./.github/actions/setup-node-pnpm | |
| with: | |
| use-prebuilt-artifacts: "false" | |
| - run: pnpm exec turbo run build | |
| - name: Save Turbo cache | |
| if: always() | |
| uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 | |
| with: | |
| path: .turbo | |
| key: ${{ runner.os }}-turbo-${{ hashFiles('pnpm-lock.yaml','pnpm-workspace.yaml','package.json','turbo.json') }}-${{ github.sha }} | |
| - name: Upload build artifacts | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: build-artifacts | |
| path: | | |
| package.json | |
| packages/protocol/stagehand.v4.json | |
| packages/extension/dist/** | |
| packages/extension/artifacts/** | |
| packages/sdk-ts/dist/** | |
| packages/integrations/core/dist/** | |
| packages/integrations/claude-agent-sdk/dist/** | |
| packages/integrations/claude-cua-sdk/dist/** | |
| packages/integrations/gemini-cua-sdk/dist/** | |
| packages/integrations/codex-sdk/dist/** | |
| packages/integrations/mastra-sdk/dist/** | |
| packages/integrations/pi-sdk/dist/** | |
| packages/integrations/eve-sdk/dist/** | |
| packages/integrations/deepagents-sdk/dist/** | |
| packages/integrations/fx-sdk/dist/** | |
| packages/integrations/cursor-sdk/dist/** | |
| packages/evals/dist/** | |
| retention-days: 1 | |
| unit-ts: | |
| name: TypeScript unit | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| needs: [build, determine-changes] | |
| if: needs.determine-changes.outputs.sdk-ts == 'true' | |
| steps: | |
| - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 | |
| - uses: ./.github/actions/setup-node-pnpm | |
| with: | |
| use-prebuilt-artifacts: "true" | |
| - uses: ./.github/actions/setup-chrome-verified | |
| # Browser-runtime specs run in browser-ts; root Vitest would repeat the unit suites. | |
| - run: pnpm exec turbo run test:unit | |
| browser-ts: | |
| name: TypeScript browser | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| needs: [build, determine-changes] | |
| if: needs.determine-changes.outputs.sdk-ts == 'true' | |
| steps: | |
| - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 | |
| - uses: ./.github/actions/setup-node-pnpm | |
| with: | |
| use-prebuilt-artifacts: "true" | |
| - uses: ./.github/actions/setup-chrome-verified | |
| id: setup-chrome | |
| - run: pnpm exec turbo run test:browser | |
| env: | |
| CHROME_PATH: ${{ steps.setup-chrome.outputs.chrome-path }} | |
| - name: Browserbase smoke test | |
| if: needs.determine-changes.outputs.is_internal_head == 'true' | |
| run: >- | |
| pnpm exec vitest run --root . | |
| packages/sdk-ts/tests/browser-runtime/stagehandBrowserbaseSmoke.test.ts | |
| env: | |
| BROWSERBASE_API_KEY: ${{ secrets.BROWSERBASE_API_KEY }} | |
| BROWSERBASE_SMOKE: "1" | |
| python-checks: | |
| name: Python checks | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| needs: [build, determine-changes] | |
| if: needs.determine-changes.outputs.sdk-python == 'true' | |
| defaults: | |
| run: | |
| working-directory: packages/sdk-python | |
| steps: | |
| - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 | |
| - uses: ./.github/actions/setup-node-pnpm | |
| with: | |
| use-prebuilt-artifacts: "true" | |
| - name: Set up Python | |
| uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 | |
| with: | |
| python-version-file: packages/sdk-python/.python-version | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 | |
| with: | |
| version: "0.11.29" | |
| working-directory: packages/sdk-python | |
| enable-cache: true | |
| cache-dependency-glob: packages/sdk-python/uv.lock | |
| - run: uv sync --locked --dev | |
| - run: uv run --locked pytest | |
| deepagents-runner-checks: | |
| name: Deep Agents runner checks | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| needs: [determine-changes] | |
| if: needs.determine-changes.outputs.deepagents-runner == 'true' | |
| defaults: | |
| run: | |
| working-directory: packages/integrations/deepagents/runner | |
| steps: | |
| - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 | |
| - name: Set up Python | |
| uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 | |
| with: | |
| python-version-file: packages/integrations/deepagents/runner/.python-version | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 | |
| with: | |
| version: "0.11.29" | |
| working-directory: packages/integrations/deepagents/runner | |
| enable-cache: true | |
| cache-dependency-glob: packages/integrations/deepagents/runner/uv.lock | |
| - run: uv lock --check | |
| - run: uv sync --locked --dev | |
| - run: uv run --locked pytest | |
| python-wheel-smoke: | |
| name: Python wheel smoke | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| needs: [build, determine-changes] | |
| if: needs.determine-changes.outputs.sdk-python == 'true' | |
| defaults: | |
| run: | |
| working-directory: packages/sdk-python | |
| steps: | |
| - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 | |
| - uses: ./.github/actions/setup-node-pnpm | |
| with: | |
| use-prebuilt-artifacts: "true" | |
| - uses: ./.github/actions/setup-chrome-verified | |
| id: setup-chrome | |
| - name: Set up Python | |
| uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 | |
| with: | |
| python-version-file: packages/sdk-python/.python-version | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 | |
| with: | |
| version: "0.11.29" | |
| working-directory: packages/sdk-python | |
| enable-cache: true | |
| cache-dependency-glob: packages/sdk-python/uv.lock | |
| - uses: extractions/setup-just@f8a3cce218d9f83db3a2ecd90e41ac3de6cdfd9b # v3.1.0 | |
| - run: just build | |
| working-directory: ${{ github.workspace }} | |
| - name: Smoke test installed Python wheel | |
| run: >- | |
| uv run --isolated --no-project | |
| --with dist/*.whl | |
| python scripts/smoke.py | |
| env: | |
| CHROME_PATH: ${{ steps.setup-chrome.outputs.chrome-path }} | |
| extension-drift: | |
| name: Extension drift (Go embed vs extension build) | |
| # Same fork gate as go-windows: installs dependencies and builds the extension, so it should | |
| # not burn hosted-runner time on an unreviewed fork PR. safe-to-test lets a maintainer opt | |
| # a specific external PR in. | |
| if: >- | |
| needs.determine-changes.outputs.sdk-go == 'true' && | |
| (needs.determine-changes.outputs.is_internal_head == 'true' || | |
| contains(github.event.pull_request.labels.*.name, 'safe-to-test')) | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| needs: [build, determine-changes] | |
| steps: | |
| - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 | |
| - uses: ./.github/actions/setup-node-pnpm | |
| with: | |
| use-prebuilt-artifacts: "true" | |
| - name: Set up Go | |
| uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 | |
| with: | |
| go-version: "1.26.x" | |
| cache: false | |
| - run: pnpm --filter ./packages/extension build | |
| # packages/sdk-go/internal/extensionassets/stagehand-extension.zip is committed and | |
| # go:embed'd, so Go tests and releases use it without ever building the extension. Without | |
| # this gate a change to the extension source ships a stale embedded copy while CI is green. | |
| - name: Check embedded Go extension matches the built extension | |
| run: go run ./internal/extensionpack --check | |
| working-directory: packages/sdk-go | |
| go-checks: | |
| name: Go checks | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| needs: [extension-drift, determine-changes] | |
| # extension-drift is fork-gated, so it is *skipped* rather than failed on external PRs. | |
| # A skipped dependency skips its dependents by default, which would silently stop running | |
| # the Go suite on fork PRs. Run unless the drift check actually failed. | |
| if: >- | |
| !cancelled() && | |
| needs.extension-drift.result != 'failure' && | |
| needs.determine-changes.outputs.sdk-go == 'true' | |
| defaults: | |
| run: | |
| working-directory: packages/sdk-go | |
| steps: | |
| - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 | |
| - uses: ./.github/actions/setup-chrome-verified | |
| id: setup-chrome | |
| - name: Set up Go | |
| uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 | |
| with: | |
| go-version: "1.26.x" | |
| cache-dependency-path: packages/sdk-go/internal/generator/go.sum | |
| - run: go test $(go list ./... | grep -v '/examples$') | |
| env: | |
| CHROME_PATH: ${{ steps.setup-chrome.outputs.chrome-path }} | |
| - run: go -C internal/generator test ./... | |
| - run: go build $(go list ./... | grep -v '/examples$') | |
| discover-integration: | |
| name: Discover integration tests | |
| runs-on: ubuntu-latest | |
| needs: [determine-changes] | |
| if: needs.determine-changes.outputs.extension == 'true' | |
| outputs: | |
| integration-tests: ${{ steps.set-matrix.outputs.integration-tests }} | |
| has-integration-tests: ${{ steps.set-matrix.outputs.has-integration-tests }} | |
| steps: | |
| - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 | |
| - uses: ./.github/actions/setup-node-pnpm | |
| with: | |
| use-prebuilt-artifacts: "false" | |
| restore-turbo-cache: "false" | |
| - name: Discover integration tests | |
| id: set-matrix | |
| run: | | |
| integration_json=$(pnpm run test:integration -- --list) | |
| echo "integration-tests=$integration_json" >> "$GITHUB_OUTPUT" | |
| if [ "$integration_json" = "[]" ]; then | |
| echo "has-integration-tests=false" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "has-integration-tests=true" >> "$GITHUB_OUTPUT" | |
| fi | |
| echo "Found integration tests: $integration_json" | |
| integration: | |
| name: integration/${{ matrix.test.name }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| needs: [build, discover-integration] | |
| if: needs.discover-integration.outputs.has-integration-tests == 'true' | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 20 | |
| matrix: | |
| test: ${{ fromJson(needs.discover-integration.outputs.integration-tests) }} | |
| steps: | |
| - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 | |
| - uses: ./.github/actions/setup-node-pnpm | |
| with: | |
| use-prebuilt-artifacts: "true" | |
| - uses: ./.github/actions/setup-chrome-verified | |
| - run: pnpm run test:integration -- "${{ matrix.test.path }}" | |
| - uses: ./.github/actions/upload-ctrf-report | |
| if: always() | |
| with: | |
| name: ctrf/integration/${{ matrix.test.safe_name }}.json | |
| - uses: ./.github/actions/upload-v8-coverage | |
| if: always() | |
| with: | |
| name: coverage/integration/${{ matrix.test.safe_name }} | |
| regression-evals: | |
| name: evals/regression | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 90 | |
| needs: [build, determine-changes, load-pr-context] | |
| if: >- | |
| ${{ | |
| always() && | |
| needs.build.result == 'success' && | |
| needs.determine-changes.result == 'success' && | |
| needs.load-pr-context.result == 'success' && | |
| needs.load-pr-context.outputs.is-internal-head == 'true' && | |
| needs.load-pr-context.outputs.skip-evals != 'true' && | |
| needs.load-pr-context.outputs.skip-regression-evals != 'true' && | |
| !(needs.determine-changes.outputs.docs-only == 'true' && | |
| needs.determine-changes.outputs.evals == 'false' && | |
| needs.determine-changes.outputs.extension == 'false' && | |
| needs.determine-changes.outputs.sdk-ts == 'false' && | |
| needs.determine-changes.outputs.sdk-python == 'false' && | |
| needs.determine-changes.outputs.sdk-go == 'false') | |
| }} | |
| env: | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }} | |
| BROWSERBASE_API_KEY: ${{ secrets.BROWSERBASE_API_KEY }} | |
| BROWSERBASE_PROJECT_ID: ${{ secrets.BROWSERBASE_PROJECT_ID }} | |
| EVAL_BROWSERBASE_PROXIES: "0" | |
| EVAL_MAX_CONCURRENCY: "25" | |
| EVAL_MODELS: ${{ vars.EVAL_MODELS }} | |
| EVAL_TRIAL_COUNT: "3" | |
| GOOGLE_GENERATIVE_AI_API_KEY: ${{ secrets.GOOGLE_GENERATIVE_AI_API_KEY }} | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| steps: | |
| - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 | |
| - uses: ./.github/actions/setup-node-pnpm | |
| with: | |
| use-prebuilt-artifacts: "true" | |
| restore-turbo-cache: "false" | |
| - name: Run regression evals | |
| id: run-evals | |
| env: | |
| NODE_V8_COVERAGE: coverage/evals/regression | |
| run: | | |
| set +e | |
| pnpm --filter @browserbasehq/stagehand-evals run test:evals -- regression --env browserbase --trials "$EVAL_TRIAL_COUNT" --concurrency "$EVAL_MAX_CONCURRENCY" | |
| eval_status=$? | |
| set -e | |
| if [ -f eval-summary.json ]; then | |
| { | |
| echo "summary-text<<EOF" | |
| jq -r ' | |
| "Experiment: \(.experimentName)", | |
| "Passed: \(.passed | length)", | |
| "Failed: \(.failed | length)", | |
| "Braintrust scores:", | |
| ((.scores // {}) | to_entries[] | " \(.key): \((.value.score * 100))%"), | |
| "Category scores:", | |
| ((.categories // {}) | to_entries[] | " \(.key): \(.value)%") | |
| ' eval-summary.json | |
| echo "EOF" | |
| } >> "$GITHUB_OUTPUT" | |
| fi | |
| exit "$eval_status" | |
| - name: Log eval score - regression | |
| if: always() | |
| env: | |
| EVAL_STDOUT_SUMMARY: ${{ steps.run-evals.outputs.summary-text }} | |
| run: | | |
| if [ ! -f eval-summary.json ]; then | |
| echo "Regression eval summary not found." | |
| exit 1 | |
| fi | |
| experiment_url=$(jq -r '.experimentUrl // empty' eval-summary.json) | |
| if [ -z "$experiment_url" ]; then | |
| experiment_name=$(jq -r '.experimentName // empty' eval-summary.json) | |
| if [ -n "$experiment_name" ]; then | |
| experiment_url="https://www.braintrust.dev/app/Browserbase/p/stagehand/experiments/${experiment_name}" | |
| fi | |
| fi | |
| primary_score=$(jq -r '(.scores["Exact match"].score // .scores["Pass"].score // empty)' eval-summary.json) | |
| if [ -z "$primary_score" ]; then | |
| echo "Braintrust primary score not found in eval summary." | |
| exit 1 | |
| fi | |
| { | |
| if [ -n "${EVAL_STDOUT_SUMMARY:-}" ]; then | |
| echo "### Regression evals" | |
| echo '```' | |
| printf '%s\n' "$EVAL_STDOUT_SUMMARY" | |
| echo '```' | |
| fi | |
| if [ -n "$experiment_url" ]; then | |
| echo "[View Braintrust experiment]($experiment_url)" | |
| fi | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| primary_score_percent=$(jq -rn --arg score "$primary_score" '$score | tonumber * 100') | |
| printf 'Regression Braintrust score: %.2f%%\n' "$primary_score_percent" | |
| if [ -n "$experiment_url" ]; then | |
| echo "View results at $experiment_url" | |
| fi | |
| if ! jq -en --arg score "$primary_score" '$score | tonumber >= 0.8' > /dev/null; then | |
| echo "Regression Braintrust score is below 80%." | |
| exit 1 | |
| fi | |
| - uses: ./.github/actions/upload-ctrf-report | |
| if: always() | |
| with: | |
| name: ctrf/evals/regression.json | |
| - uses: ./.github/actions/upload-v8-coverage | |
| if: always() | |
| with: | |
| name: coverage/evals/regression | |
| go-windows: | |
| name: Go (Windows process management) | |
| needs: [determine-changes] | |
| if: >- | |
| needs.determine-changes.outputs.is_internal_head == 'true' | |
| runs-on: windows-latest | |
| timeout-minutes: 10 | |
| defaults: | |
| run: | |
| working-directory: packages/sdk-go | |
| steps: | |
| - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 | |
| - name: Set up Go | |
| uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 | |
| with: | |
| go-version: "1.26.x" | |
| cache: false | |
| - name: Test Windows process management | |
| run: >- | |
| go test | |
| -run 'TestConfigureChromeProcessUsesDedicatedProcessGroupWindows|TestTaskkillArgsPreserveGracefulThenForcefulShutdown|TestIgnoreFinishedProcessErrorWindows' | |
| . |