diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 5525f1c5d..5ead31a1b 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -2,24 +2,40 @@ name: E2E Tests permissions: contents: read + id-token: write on: push: branches: [main] - paths: ['**/*.go', 'go.mod', 'go.sum', 'e2e/**'] + paths: + - '**/*.go' + - 'go.mod' + - 'go.sum' + - 'e2e/**' + - 'internal/scaffold/fullsend-repo/**' + - 'internal/security/hooks/**' + - 'internal/dispatch/gcf/mintsrc/**' + - 'internal/sentencetoken/english.json' + - 'Makefile' + - '.github/workflows/e2e.yml' pull_request: - paths: ['**/*.go', 'go.mod', 'go.sum', 'e2e/**'] + paths: + - '**/*.go' + - 'go.mod' + - 'go.sum' + - 'e2e/**' + - 'internal/scaffold/fullsend-repo/**' + - 'internal/security/hooks/**' + - 'internal/dispatch/gcf/mintsrc/**' + - 'internal/sentencetoken/english.json' + - 'Makefile' + - '.github/workflows/e2e.yml' workflow_dispatch: -concurrency: - group: e2e-halfsend - cancel-in-progress: false - queue: max - jobs: e2e: runs-on: ubuntu-latest - timeout-minutes: 15 + timeout-minutes: 30 steps: - uses: actions/checkout@v4 @@ -51,6 +67,13 @@ jobs: env: E2E_GITHUB_SESSION_B64: ${{ secrets.E2E_GITHUB_SESSION }} + - name: Authenticate to GCP + if: steps.secrets-check.outputs.available == 'true' + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ secrets.E2E_GCP_WIF_PROVIDER }} + service_account: ${{ secrets.E2E_GCP_SERVICE_ACCOUNT }} + - name: Run e2e tests if: steps.secrets-check.outputs.available == 'true' run: make e2e-test @@ -58,6 +81,8 @@ jobs: E2E_SCREENSHOT_DIR: ${{ runner.temp }}/e2e-screenshots E2E_GITHUB_PASSWORD: ${{ secrets.E2E_GITHUB_PASSWORD }} E2E_GITHUB_TOTP_SECRET: ${{ secrets.E2E_GITHUB_TOTP_SECRET }} + E2E_MINT_URL: ${{ secrets.E2E_MINT_URL }} + E2E_GCP_PROJECT_ID: ${{ secrets.E2E_GCP_PROJECT_ID }} - name: Upload debug screenshots if: always() && steps.secrets-check.outputs.available == 'true' diff --git a/docs/ADRs/0040-org-pool-for-parallel-e2e-tests.md b/docs/ADRs/0040-org-pool-for-parallel-e2e-tests.md new file mode 100644 index 000000000..3e8c44922 --- /dev/null +++ b/docs/ADRs/0040-org-pool-for-parallel-e2e-tests.md @@ -0,0 +1,71 @@ +--- +title: "40. Org pool for parallel e2e tests" +status: Accepted +relates_to: + - testing-agents +topics: + - e2e + - ci + - parallelism +--- + +# 40. Org pool for parallel e2e tests + +Date: 2026-05-19 + +## Status + +Accepted + +## Context + +The e2e tests exercise the full admin install/uninstall flow against a live +GitHub org using Playwright browser automation +([ADR 0010](0010-stored-session-for-e2e-browser-auth.md)). Each run creates +GitHub Apps, repos, secrets, variables, and enrollment PRs — then tears them +all down. These operations are destructive and non-reentrant: two concurrent +runs targeting the same org will collide on shared resources (`.fullsend` repo, +org secrets, app slugs) and fail unpredictably. + +With a single test org, CI runs are serialized. A push to `main` and an +in-flight PR both trigger e2e, but only one can proceed; the other waits or +fails. As the contributor count grows this becomes a bottleneck. + +## Decision + +Maintain a pool of identically-configured GitHub orgs (currently `halfsend-01` +through `halfsend-06`). Each e2e run acquires exclusive access to one org +before proceeding, using a lightweight distributed lock implemented as a +purpose-built repo (`e2e-lock`) within each org. + +**Acquisition:** The test runner scans the pool in order, attempting to create +the `e2e-lock` repo in each org. Repo creation is atomic on GitHub — if it +succeeds, the caller holds the lock. A `README.md` in the lock repo contains +the run's UUID for ownership verification. If all orgs are locked, the runner +falls back to polling with a configurable timeout (`E2E_LOCK_TIMEOUT`, +default 2 minutes). + +**Release:** On test completion (pass or fail), the runner deletes the +`e2e-lock` repo, but only after verifying the UUID matches — preventing a +run from releasing another run's lock. + +**Staleness:** If a runner crashes without releasing its lock, the lock repo's +`created_at` timestamp provides an age signal. A fresh lock (under 1 minute +old) resets the wait timer; an old lock is assumed stale and can be +force-acquired by a subsequent run. + +Adding new orgs to the pool requires only provisioning the GitHub org (with +the shared test account as owner) and appending its name to the `orgPool` +slice in the test code. No architectural changes are needed. + +## Consequences + +- Up to N e2e runs execute in parallel, where N is the pool size. +- Each org must be pre-provisioned with the test account as owner and a + `test-repo` for enrollment testing. +- A crashed run leaves a stale lock that self-heals via the age-based + staleness check. +- The single `botsend` test account and its stored browser session are shared + across all orgs; session export and PAT creation remain per-run. +- Pool expansion is an operational task (provision org, update one slice + literal), not an architectural change. diff --git a/e2e/admin/admin_test.go b/e2e/admin/admin_test.go index a85d30aef..befa096aa 100644 --- a/e2e/admin/admin_test.go +++ b/e2e/admin/admin_test.go @@ -3,12 +3,15 @@ package admin import ( + "archive/zip" + "bytes" "context" "encoding/json" "fmt" + "io" "net/http" "os" - "os/exec" + "path/filepath" "strings" "testing" "time" @@ -18,25 +21,21 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/fullsend-ai/fullsend/internal/appsetup" "github.com/fullsend-ai/fullsend/internal/config" "github.com/fullsend-ai/fullsend/internal/forge" gh "github.com/fullsend-ai/fullsend/internal/forge/github" - "github.com/fullsend-ai/fullsend/internal/inference" - "github.com/fullsend-ai/fullsend/internal/inference/vertex" - "github.com/fullsend-ai/fullsend/internal/layers" - "github.com/fullsend-ai/fullsend/internal/ui" ) // e2eEnv holds the shared state for an e2e test run. type e2eEnv struct { cfg envConfig + org string // the org acquired from the pool page playwright.Page client *gh.LiveClient token string - printer *ui.Printer runID string screenshotDir string + binary string } // setupE2ETest performs the common Playwright, login, PAT, lock, and cleanup @@ -54,6 +53,9 @@ func setupE2ETest(t *testing.T) *e2eEnv { } _ = os.MkdirAll(screenshotDir, 0o755) + // Build CLI binary early so we fail fast on compilation errors. + binary := buildCLIBinary(t) + // --- Playwright setup --- pw, err := playwright.Run() require.NoError(t, err, "starting Playwright") @@ -92,35 +94,36 @@ func setupE2ETest(t *testing.T) *e2eEnv { t.Cleanup(func() { t.Log("Deleting PAT...") if delErr := deletePAT(page, patNote, t.Logf); delErr != nil { - t.Logf("warning: could not delete PAT: %v", delErr) + t.Errorf("could not delete PAT: %v", delErr) } }) // --- GitHub client --- client := newLiveClient(token) - printer := ui.New(os.Stdout) - // Acquire lock. + // Acquire an org from the pool. runID := uuid.New().String() t.Logf("E2E run ID: %s", runID) - err = acquireLock(context.Background(), client, token, testOrg, runID, cfg.lockTimeout, t.Logf) - require.NoError(t, err, "acquiring e2e lock") + org, err := acquireOrg(context.Background(), client, token, runID, cfg.lockTimeout, t.Logf) + require.NoError(t, err, "acquiring org from pool") + t.Logf("Acquired org: %s", org) t.Cleanup(func() { - releaseLock(context.Background(), client, testOrg, runID, t) + releaseLock(context.Background(), client, org, runID, t) }) // Teardown-first cleanup. - cleanupStaleResources(context.Background(), client, page, token, screenshotDir, t) + cleanupStaleResources(context.Background(), client, token, org, t) return &e2eEnv{ cfg: cfg, + org: org, page: page, client: client, token: token, - printer: printer, runID: runID, screenshotDir: screenshotDir, + binary: binary, } } @@ -128,261 +131,35 @@ func TestAdminInstallUninstall(t *testing.T) { env := setupE2ETest(t) ctx := context.Background() - // ========================================= - // Phase 1: First install (creates resources) - // ========================================= - t.Log("=== Phase 1: First Install ===") - agentCreds, orgCfg, enabledRepos, enrolledRepoIDs := runFullInstall(t, env) - verifyInstalled(t, env, orgCfg, enabledRepos, agentCreds) - - // ========================================= - // Phase 2: Second install (idempotent no-op) - // ========================================= - t.Log("=== Phase 2: Second Install (idempotent) ===") - user, err := env.client.GetAuthenticatedUser(ctx) - require.NoError(t, err) - allRepos, err := env.client.ListOrgRepos(ctx, testOrg) - require.NoError(t, err) - hasPrivate := hasPrivateRepos(allRepos) - - // Second install should be idempotent — OIDC dispatch infra already provisioned. - // Inference provider is nil for idempotent re-install (already provisioned). - stack := buildTestLayerStack(testOrg, env.client, orgCfg, env.printer, user, hasPrivate, enabledRepos, agentCreds, enrolledRepoIDs, nil) - err = stack.InstallAll(ctx) - require.NoError(t, err, "second InstallAll should succeed") - verifyInstalled(t, env, orgCfg, enabledRepos, agentCreds) - - // ========================================= - // Phase 2.25: Merge enrollment PR - // ========================================= - // The enrollment PR must be merged before unenrollment can work (the shim - // must exist on the default branch for the removal PR to make sense). - t.Log("=== Phase 2.25: Merge Enrollment PR ===") - mergeEnrollmentPR(t, env) - - // ========================================= - // Phase 2.5: Triage dispatch smoke test - // ========================================= - if os.Getenv("E2E_HALFSEND_WIF_PROVIDER") != "" { - t.Log("=== Phase 2.5: Triage Dispatch Smoke Test ===") - vendorBinaryForE2E(t, env) - runTriageDispatchSmokeTest(t, env) - } else { - t.Log("=== Phase 2.5: Triage Dispatch Smoke Test (SKIPPED — no inference credentials) ===") + // Phase 1: Install via CLI subprocess. + t.Log("=== Phase 1: Install ===") + installArgs := []string{ + "admin", "install", env.org, + "--skip-app-setup", + "--skip-mint-check", + "--mint-url", env.cfg.mintURL, + "--app-set", e2eAppSet, + "--enroll-all", + "--vendor-fullsend-binary", } - - // ========================================= - // Phase 2.75: Unenrollment reconciliation - // ========================================= - t.Log("=== Phase 2.75: Unenrollment ===") - runUnenrollmentTest(t, env, orgCfg, agentCreds, enrolledRepoIDs) - - // ========================================= - // Phase 3: First uninstall (deletes resources) - // ========================================= - t.Log("=== Phase 3: First Uninstall ===") - runUninstall(t, env) - // Wait for repo deletion to propagate (GitHub returns 409 if checked too soon). - time.Sleep(5 * time.Second) - verifyNotInstalled(t, env) - - // ========================================= - // Phase 4: Second uninstall (idempotent no-op) - // ========================================= - t.Log("=== Phase 4: Second Uninstall (idempotent) ===") - runUninstallAllowNotFound(t, env) - time.Sleep(3 * time.Second) - verifyNotInstalled(t, env) - - t.Log("=== E2E test complete ===") -} - -// --- Install/uninstall helpers --- - -// runFullInstall executes the full install flow (app setup + layer stack install) -// and returns the agent credentials and org config for verification. -func runFullInstall(t *testing.T, env *e2eEnv) ([]layers.AgentCredentials, *config.OrgConfig, []string, []int64) { - t.Helper() - ctx := context.Background() - - // App setup via manifest flow with Playwright. - playwrightBrowser := NewPlaywrightBrowserOpener(env.page, t.Logf, env.screenshotDir) - prompter := AutoPrompter{} - setup := appsetup.NewSetup(env.client, prompter, playwrightBrowser, env.printer) - - var agentCreds []layers.AgentCredentials - for _, role := range defaultRoles { - t.Logf("Setting up app for role: %s", role) - - var appCreds *appsetup.AppCredentials - // Retry the manifest flow to handle transient callback timeouts - // (see #287). On failure, delete any partially-created app and - // wait before retrying so the next attempt starts clean. - const maxAttempts = 3 - for attempt := 1; attempt <= maxAttempts; attempt++ { - // Per-attempt timeout: generous to handle slow manifest flows - // (90s callback wait + page navigation overhead). - roleCtx, roleCancel := context.WithTimeout(ctx, 6*time.Minute) - var runErr error - appCreds, runErr = setup.Run(roleCtx, testOrg, role) - roleCancel() - - if runErr == nil { - break - } - - t.Logf("Attempt %d/%d for role %s failed: %v", attempt, maxAttempts, role, runErr) - if attempt < maxAttempts { - slug := appsetup.AppSlug(appsetup.DefaultAppSet, role) - t.Logf("Cleaning up potentially stale app %s before retry", slug) - if delErr := deleteAppViaPlaywright(env.page, slug, t.Logf, env.screenshotDir); delErr != nil { - t.Logf("Warning: cleanup of %s failed (may not exist): %v", slug, delErr) - } - t.Logf("Waiting 10s before retry to let GitHub settle...") - time.Sleep(10 * time.Second) - continue - } - require.NoError(t, runErr, "setting up app for role %s", role) - } - - agentCreds = append(agentCreds, layers.AgentCredentials{ - AgentEntry: config.AgentEntry{ - Role: role, - Name: appCreds.Name, - Slug: appCreds.Slug, - }, - PEM: appCreds.PEM, - ClientID: appCreds.ClientID, - }) - - registerAppCleanup(t, env.page, appCreds.Slug, env.screenshotDir) - } - - // Discover repos and build config. - allRepos, err := env.client.ListOrgRepos(ctx, testOrg) - require.NoError(t, err, "listing org repos") - - repoNames := repoNameList(allRepos) - hasPrivate := hasPrivateRepos(allRepos) - enabledRepos := []string{testRepo} - - agents := make([]config.AgentEntry, len(agentCreds)) - for i, ac := range agentCreds { - agents[i] = ac.AgentEntry - } - - // Build inference provider if WIF provider is available. - var inferenceProvider inference.Provider - var inferenceProviderName string - if wifProvider := os.Getenv("E2E_HALFSEND_WIF_PROVIDER"); wifProvider != "" { - gcpProjectID := os.Getenv("E2E_GCP_PROJECT_ID") - if gcpProjectID == "" { - t.Fatal("E2E_GCP_PROJECT_ID is required when E2E_HALFSEND_WIF_PROVIDER is set") - } - gcpRegion := os.Getenv("E2E_GCP_REGION") - if gcpRegion == "" { - gcpRegion = "global" - } - inferenceProvider = vertex.New(vertex.Config{ - ProjectID: gcpProjectID, - Region: gcpRegion, - WIFProvider: wifProvider, - }) - inferenceProviderName = "vertex" - t.Logf("Inference provider: vertex (project: %s)", gcpProjectID) - } else { - t.Log("E2E_HALFSEND_WIF_PROVIDER not set, skipping inference layer") - } - - orgCfg := config.NewOrgConfig(repoNames, enabledRepos, defaultRoles, agents, inferenceProviderName) - - user, err := env.client.GetAuthenticatedUser(ctx) - require.NoError(t, err, "getting authenticated user") - - // Collect repo IDs for enrolled repos (needed by DispatchTokenLayer). - var enrolledRepoIDs []int64 - for _, repoName := range enabledRepos { - repo, repoErr := env.client.GetRepo(ctx, testOrg, repoName) - require.NoError(t, repoErr, "getting repo %s for ID", repoName) - enrolledRepoIDs = append(enrolledRepoIDs, repo.ID) - } - - // Install config-repo and workflows layers first so .fullsend repo exists. - // Config-repo and workflows are idempotent, so re-running them is harmless. - configLayer := layers.NewConfigRepoLayer(testOrg, env.client, orgCfg, env.printer, hasPrivate) - err = configLayer.Install(ctx) - require.NoError(t, err, "pre-installing config-repo layer") - registerRepoCleanup(t, env.client, testOrg, forge.ConfigRepoName) - - workflowsLayer := layers.NewWorkflowsLayer(testOrg, env.client, env.printer, user) - err = workflowsLayer.Install(ctx) - require.NoError(t, err, "pre-installing workflows layer") - - // Build full layer stack and install all layers. - // Config-repo and workflows are idempotent, so re-running them is harmless. - stack := buildTestLayerStack(testOrg, env.client, orgCfg, env.printer, user, hasPrivate, enabledRepos, agentCreds, enrolledRepoIDs, inferenceProvider) - - err = stack.InstallAll(ctx) - require.NoError(t, err, "installing layers") - - return agentCreds, orgCfg, enabledRepos, enrolledRepoIDs -} - -func runUninstall(t *testing.T, env *e2eEnv) { - t.Helper() - emptyCfg := config.NewOrgConfig(nil, nil, nil, nil, "") - stack := layers.NewStack( - layers.NewConfigRepoLayer(testOrg, env.client, emptyCfg, env.printer, false), - layers.NewWorkflowsLayer(testOrg, env.client, env.printer, ""), - layers.NewSecretsLayer(testOrg, env.client, nil, env.printer), - layers.NewInferenceLayer(testOrg, env.client, nil, env.printer), - layers.NewBothModesDispatchLayer(testOrg, env.client, &e2eDispatcher{}, env.printer), - layers.NewEnrollmentLayer(testOrg, env.client, nil, nil, env.printer), - ) - errs := stack.UninstallAll(context.Background()) - assert.Empty(t, errs, "uninstall should complete without errors") -} - -// runUninstallAllowNotFound runs uninstall but accepts not-found errors -// (expected when resources are already deleted). -func runUninstallAllowNotFound(t *testing.T, env *e2eEnv) { - t.Helper() - emptyCfg := config.NewOrgConfig(nil, nil, nil, nil, "") - stack := layers.NewStack( - layers.NewConfigRepoLayer(testOrg, env.client, emptyCfg, env.printer, false), - layers.NewWorkflowsLayer(testOrg, env.client, env.printer, ""), - layers.NewSecretsLayer(testOrg, env.client, nil, env.printer), - layers.NewInferenceLayer(testOrg, env.client, nil, env.printer), - layers.NewBothModesDispatchLayer(testOrg, env.client, &e2eDispatcher{}, env.printer), - layers.NewEnrollmentLayer(testOrg, env.client, nil, nil, env.printer), - ) - errs := stack.UninstallAll(context.Background()) - for _, e := range errs { - if !forge.IsNotFound(e) { - t.Errorf("unexpected uninstall error (not a not-found): %v", e) - } + if env.cfg.gcpProjectID != "" { + installArgs = append(installArgs, "--inference-project", env.cfg.gcpProjectID) } -} - -// --- Verification helpers --- - -// verifyInstalled checks that all resources exist and analyze reports installed. -func verifyInstalled(t *testing.T, env *e2eEnv, orgCfg *config.OrgConfig, enabledRepos []string, agentCreds []layers.AgentCredentials) { - t.Helper() - ctx := context.Background() + runCLI(t, env.binary, env.token, installArgs...) - // .fullsend repo exists. - repo, err := env.client.GetRepo(ctx, testOrg, forge.ConfigRepoName) + // Verify install artifacts. + _, err := env.client.GetRepo(ctx, env.org, forge.ConfigRepoName) require.NoError(t, err, ".fullsend repo should exist") - assert.Equal(t, forge.ConfigRepoName, repo.Name) - - // config.yaml exists and parses. - cfgData, err := env.client.GetFileContent(ctx, testOrg, forge.ConfigRepoName, "config.yaml") + mintURLExists, err := env.client.OrgVariableExists(ctx, env.org, "FULLSEND_MINT_URL") + require.NoError(t, err) + require.True(t, mintURLExists, "FULLSEND_MINT_URL org variable should exist") + cfgData, err := env.client.GetFileContent(ctx, env.org, forge.ConfigRepoName, "config.yaml") require.NoError(t, err, "config.yaml should exist") parsedCfg, err := config.ParseOrgConfig(cfgData) require.NoError(t, err, "config.yaml should parse") - assert.Equal(t, "1", parsedCfg.Version) - assert.Len(t, parsedCfg.Agents, len(defaultRoles)) + require.Len(t, parsedCfg.Defaults.Roles, len(defaultRoles), "should have %d roles", len(defaultRoles)) + analyzeOutput := runCLI(t, env.binary, env.token, "admin", "analyze", env.org) + t.Logf("Analyze output:\n%s", analyzeOutput) // Agent runtime files exist (from scaffold). // ADR 35: only non-layered, non-upstream-only files are installed. @@ -409,157 +186,50 @@ func verifyInstalled(t *testing.T, env *e2eEnv, orgCfg *config.OrgConfig, enable "templates/shim-workflow-call.yaml", "CODEOWNERS", } { - _, err := env.client.GetFileContent(ctx, testOrg, forge.ConfigRepoName, path) + _, err := env.client.GetFileContent(ctx, env.org, forge.ConfigRepoName, path) assert.NoError(t, err, "%s should exist in .fullsend", path) } - // Secrets and variables exist for each role. - for _, role := range defaultRoles { - secretName := fmt.Sprintf("FULLSEND_%s_APP_PRIVATE_KEY", strings.ToUpper(role)) - exists, err := env.client.RepoSecretExists(ctx, testOrg, forge.ConfigRepoName, secretName) - assert.NoError(t, err, "checking secret %s", secretName) - assert.True(t, exists, "secret %s should exist", secretName) - - varName := fmt.Sprintf("FULLSEND_%s_CLIENT_ID", strings.ToUpper(role)) - exists, err = env.client.RepoVariableExists(ctx, testOrg, forge.ConfigRepoName, varName) - assert.NoError(t, err, "checking variable %s", varName) - assert.True(t, exists, "variable %s should exist", varName) - } + // Register .fullsend cleanup (in case later phases fail). + registerRepoCleanup(t, env.client, env.org, forge.ConfigRepoName) - // Inference secrets exist if WIF provider was configured. - if os.Getenv("E2E_HALFSEND_WIF_PROVIDER") != "" { - for _, secretName := range []string{"FULLSEND_GCP_WIF_PROVIDER", "FULLSEND_GCP_PROJECT_ID"} { - exists, secErr := env.client.RepoSecretExists(ctx, testOrg, forge.ConfigRepoName, secretName) - assert.NoError(t, secErr, "checking inference secret %s", secretName) - assert.True(t, exists, "inference secret %s should exist", secretName) - } - } + // Phase 2: Merge enrollment PR. + t.Log("=== Phase 2: Merge Enrollment PR ===") + mergeEnrollmentPR(t, env) - // OIDC dispatch variable exists; stale PAT secret should not. - mintURLExists, err := env.client.OrgVariableExists(ctx, testOrg, "FULLSEND_MINT_URL") - assert.NoError(t, err, "checking FULLSEND_MINT_URL org variable") - assert.True(t, mintURLExists, "FULLSEND_MINT_URL org variable should exist") - dispatchExists, err := env.client.OrgSecretExists(ctx, testOrg, "FULLSEND_DISPATCH_TOKEN") - assert.NoError(t, err, "checking stale dispatch token") - assert.False(t, dispatchExists, "FULLSEND_DISPATCH_TOKEN org secret should not exist in OIDC mode") + // Phase 3: Triage dispatch smoke test. + t.Log("=== Phase 3: Triage Dispatch Smoke Test ===") + runTriageDispatchSmokeTest(t, env) - // Enrollment PR exists for test-repo. - prs, err := env.client.ListRepoPullRequests(ctx, testOrg, testRepo) - require.NoError(t, err, "listing PRs for %s", testRepo) - found := false - for _, pr := range prs { - if strings.Contains(pr.Title, "fullsend") { - found = true - t.Logf("Found enrollment PR: %s", pr.URL) - break - } - } - assert.True(t, found, "enrollment PR should exist for %s", testRepo) + // Phase 4: Unenrollment reconciliation. + t.Log("=== Phase 4: Unenrollment ===") + runUnenrollmentTest(t, env) - // Analyze reports installed. - user, err := env.client.GetAuthenticatedUser(ctx) - require.NoError(t, err) - allRepos, err := env.client.ListOrgRepos(ctx, testOrg) - require.NoError(t, err) - hasPrivate := hasPrivateRepos(allRepos) - - analyzeStack := buildTestLayerStack(testOrg, env.client, orgCfg, env.printer, user, hasPrivate, enabledRepos, agentCreds, nil, nil) - reports, err := analyzeStack.AnalyzeAll(ctx) - require.NoError(t, err, "analyzing layers") - for _, report := range reports { - if report.Name == "enrollment" { - // Enrollment creates a PR but doesn't merge it, so the shim - // workflow file doesn't exist on the default branch yet. - assert.Contains(t, []layers.LayerStatus{layers.StatusInstalled, layers.StatusNotInstalled}, - report.Status, "layer %s status: %s (details: %v)", - report.Name, report.Status, report.Details) - continue - } - assert.Equal(t, layers.StatusInstalled, report.Status, - "layer %s should be installed, got %s (details: %v)", - report.Name, report.Status, report.Details) - } -} - -// verifyNotInstalled checks that the config repo is gone and analyze reports -// not-installed for layers with concrete artifacts. -func verifyNotInstalled(t *testing.T, env *e2eEnv) { - t.Helper() - ctx := context.Background() - - _, err := env.client.GetRepo(ctx, testOrg, forge.ConfigRepoName) - assert.True(t, forge.IsNotFound(err), ".fullsend repo should be deleted") - - // Dispatch token org secret should be deleted. - dispatchExists, err := env.client.OrgSecretExists(ctx, testOrg, "FULLSEND_DISPATCH_TOKEN") - assert.NoError(t, err, "checking dispatch token after uninstall") - assert.False(t, dispatchExists, "FULLSEND_DISPATCH_TOKEN org secret should be deleted") - - // OIDC mint URL org variable should be deleted. - mintURLExists, err := env.client.OrgVariableExists(ctx, testOrg, "FULLSEND_MINT_URL") - assert.NoError(t, err, "checking mint URL variable after uninstall") - assert.False(t, mintURLExists, "FULLSEND_MINT_URL org variable should be deleted") - - emptyCfg := config.NewOrgConfig(nil, nil, nil, nil, "") - stack := layers.NewStack( - layers.NewConfigRepoLayer(testOrg, env.client, emptyCfg, env.printer, false), - layers.NewWorkflowsLayer(testOrg, env.client, env.printer, ""), - layers.NewSecretsLayer(testOrg, env.client, nil, env.printer), - layers.NewInferenceLayer(testOrg, env.client, nil, env.printer), - layers.NewBothModesDispatchLayer(testOrg, env.client, &e2eDispatcher{}, env.printer), - layers.NewEnrollmentLayer(testOrg, env.client, nil, nil, env.printer), + // Phase 5: Uninstall via CLI subprocess. + t.Log("=== Phase 5: Uninstall ===") + runCLI(t, env.binary, env.token, + "admin", "uninstall", env.org, + "--yolo", + "--app-set", e2eAppSet, ) - reports, err := stack.AnalyzeAll(ctx) - require.NoError(t, err, "analyzing layers after uninstall") - for _, report := range reports { - switch report.Name { - case "config-repo", "workflows", "dispatch": - assert.Equal(t, layers.StatusNotInstalled, report.Status, - "layer %s should be not-installed, got %s", - report.Name, report.Status) - default: - // Layers with empty config may report "installed" (nothing to track). - t.Logf("layer %s status: %s (accepted)", report.Name, report.Status) - } - } -} - -// vendorBinaryForE2E builds the fullsend binary for the current platform -// (which is linux/amd64 in CI) and uploads it to the config repo so the -// triage workflow uses the code under test rather than a released version. -func vendorBinaryForE2E(t *testing.T, env *e2eEnv) { - t.Helper() - tmpBinary, err := os.CreateTemp("", "fullsend-e2e-*") + time.Sleep(5 * time.Second) + _, err = env.client.GetRepo(ctx, env.org, forge.ConfigRepoName) + require.True(t, forge.IsNotFound(err), ".fullsend repo should be deleted") + mintURLExists, err = env.client.OrgVariableExists(ctx, env.org, "FULLSEND_MINT_URL") require.NoError(t, err) - tmpBinary.Close() - t.Cleanup(func() { os.Remove(tmpBinary.Name()) }) - - // Find the module root (go test runs with cwd set to the test package dir). - modRoot, err := exec.Command("go", "list", "-m", "-f", "{{.Dir}}").Output() - require.NoError(t, err, "finding module root") - - t.Log("Building fullsend binary for vendoring...") - cmd := exec.Command("go", "build", "-o", tmpBinary.Name(), "./cmd/fullsend/") - cmd.Dir = strings.TrimSpace(string(modRoot)) - cmd.Env = append(os.Environ(), "GOOS=linux", "GOARCH=amd64", "CGO_ENABLED=0") - out, err := cmd.CombinedOutput() - require.NoError(t, err, "building fullsend binary: %s", string(out)) - - t.Log("Uploading vendored binary to .fullsend/bin/fullsend...") - err = layers.VendorBinary(context.Background(), env.client, testOrg, tmpBinary.Name()) - require.NoError(t, err, "vendoring binary") - t.Log("Vendored binary uploaded successfully") + require.False(t, mintURLExists, "FULLSEND_MINT_URL should be deleted") + + t.Log("=== E2E test complete ===") } // mergeEnrollmentPR finds and merges the enrollment PR for test-repo so the -// shim workflow is active on the default branch. This must run before both -// the triage smoke test and the unenrollment test. +// shim workflow is active on the default branch. func mergeEnrollmentPR(t *testing.T, env *e2eEnv) { t.Helper() ctx := context.Background() - prs, err := env.client.ListRepoPullRequests(ctx, testOrg, testRepo) + prs, err := env.client.ListRepoPullRequests(ctx, env.org, testRepo) require.NoError(t, err, "listing PRs for %s", testRepo) var enrollmentPR *forge.ChangeProposal @@ -573,10 +243,9 @@ func mergeEnrollmentPR(t *testing.T, env *e2eEnv) { require.NotNil(t, enrollmentPR, "enrollment PR should exist for %s", testRepo) t.Logf("Merging enrollment PR #%d: %s", enrollmentPR.Number, enrollmentPR.URL) - err = env.client.MergeChangeProposal(ctx, testOrg, testRepo, enrollmentPR.Number) + err = env.client.MergeChangeProposal(ctx, env.org, testRepo, enrollmentPR.Number) require.NoError(t, err, "merging enrollment PR") - // Wait for GitHub to process the merge. time.Sleep(5 * time.Second) t.Log("Enrollment PR merged") } @@ -616,12 +285,12 @@ Segmentation fault (core dumped) **Additional context:** This started happening after the v2.3.0 -> v2.3.1 upgrade. Files under 64KB save fine. Files over 64KB save fine if they contain only ASCII characters.` - issue, err := env.client.CreateIssue(ctx, testOrg, testRepo, issueTitle, issueBody) + issue, err := env.client.CreateIssue(ctx, env.org, testRepo, issueTitle, issueBody) require.NoError(t, err, "creating test issue") t.Logf("Created test issue #%d: %s", issue.Number, issue.URL) t.Cleanup(func() { t.Log("Closing test issue...") - if closeErr := env.client.CloseIssue(ctx, testOrg, testRepo, issue.Number); closeErr != nil { + if closeErr := env.client.CloseIssue(ctx, env.org, testRepo, issue.Number); closeErr != nil { t.Logf("warning: could not close test issue: %v", closeErr) } }) @@ -636,7 +305,7 @@ Files over 64KB save fine if they contain only ASCII characters.` var triageRun *forge.WorkflowRun for attempt := 0; attempt < 12; attempt++ { time.Sleep(5 * time.Second) - runs, listErr := env.client.ListWorkflowRuns(ctx, testOrg, forge.ConfigRepoName, "triage.yml") + runs, listErr := env.client.ListWorkflowRuns(ctx, env.org, forge.ConfigRepoName, "triage.yml") if listErr != nil { t.Logf("Attempt %d: error listing workflow runs: %v", attempt+1, listErr) continue @@ -670,7 +339,7 @@ Files over 64KB save fine if they contain only ASCII characters.` deadline := time.Now().Add(12 * time.Minute) for time.Now().Before(deadline) { time.Sleep(15 * time.Second) - run, getErr := env.client.GetWorkflowRun(ctx, testOrg, forge.ConfigRepoName, triageRun.ID) + run, getErr := env.client.GetWorkflowRun(ctx, env.org, forge.ConfigRepoName, triageRun.ID) if getErr != nil { t.Logf("Error polling workflow run: %v", getErr) continue @@ -683,20 +352,37 @@ Files over 64KB save fine if they contain only ASCII characters.` } require.NotNil(t, finalRun, "triage workflow run should have completed within deadline") - // If the run failed, fetch logs for debugging. + // If the run failed, save logs and artifacts for debugging. if finalRun.Conclusion != "success" { - logs, logErr := env.client.GetWorkflowRunLogs(ctx, testOrg, forge.ConfigRepoName, finalRun.ID) + runURL := fmt.Sprintf("https://github.com/%s/%s/actions/runs/%d", env.org, forge.ConfigRepoName, finalRun.ID) + fmt.Fprintf(os.Stderr, "::notice::Triage workflow run %d failed (conclusion: %s). Downloading debug artifacts. Run URL: %s\n", finalRun.ID, finalRun.Conclusion, runURL) + + debugDir := filepath.Join(env.screenshotDir, fmt.Sprintf("triage-run-%d", finalRun.ID)) + _ = os.MkdirAll(debugDir, 0o755) + + // Save workflow logs. + logs, logErr := env.client.GetWorkflowRunLogs(ctx, env.org, forge.ConfigRepoName, finalRun.ID) if logErr != nil { t.Logf("Could not fetch run logs: %v", logErr) } else { + logPath := filepath.Join(debugDir, "workflow-logs.txt") + if writeErr := os.WriteFile(logPath, []byte(logs), 0o644); writeErr != nil { + t.Logf("Could not write logs to %s: %v", logPath, writeErr) + } else { + fmt.Fprintf(os.Stderr, "::notice file=%s::Triage run %d workflow logs saved\n", logPath, finalRun.ID) + } t.Logf("Workflow run logs:\n%s", logs) } - t.Fatalf("Triage workflow run %d concluded with %q, expected success", finalRun.ID, finalRun.Conclusion) + + // Download run artifacts (transcripts, etc). + downloadRunArtifacts(ctx, env.token, env.org, forge.ConfigRepoName, finalRun.ID, debugDir, t) + + t.Fatalf("Triage workflow run %d concluded with %q, expected success. Debug artifacts saved to %s", finalRun.ID, finalRun.Conclusion, debugDir) } // Verify the triage agent posted a comment on the issue. t.Log("Verifying triage agent posted a comment...") - comments, err := env.client.ListIssueComments(ctx, testOrg, testRepo, issue.Number) + comments, err := env.client.ListIssueComments(ctx, env.org, testRepo, issue.Number) require.NoError(t, err, "listing issue comments") assert.NotEmpty(t, comments, "triage agent should have posted at least one comment on the issue") @@ -711,7 +397,7 @@ Files over 64KB save fine if they contain only ASCII characters.` // Verify labels: either needs-info (insufficient) or ready-to-code (sufficient). t.Log("Verifying triage labels...") - labelURL := fmt.Sprintf("https://api.github.com/repos/%s/%s/issues/%d/labels", testOrg, testRepo, issue.Number) + labelURL := fmt.Sprintf("https://api.github.com/repos/%s/%s/issues/%d/labels", env.org, testRepo, issue.Number) labelReq, err := http.NewRequestWithContext(ctx, http.MethodGet, labelURL, nil) require.NoError(t, err) labelReq.Header.Set("Authorization", "Bearer "+env.token) @@ -743,40 +429,172 @@ Files over 64KB save fine if they contain only ASCII characters.` "issue should have a triage label (needs-info, ready-to-code, duplicate, or blocked), got: %v", labelNames) } +// downloadRunArtifacts fetches all artifacts from a workflow run and extracts +// them into destDir. Each artifact is extracted into a subdirectory named after +// the artifact. This captures transcripts, screenshots, and other debug data +// that the agent run uploads. +func downloadRunArtifacts(ctx context.Context, token, org, repo string, runID int, destDir string, t *testing.T) { + t.Helper() + + // List artifacts for the run. + listURL := fmt.Sprintf("https://api.github.com/repos/%s/%s/actions/runs/%d/artifacts", org, repo, runID) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, listURL, nil) + if err != nil { + t.Logf("[artifacts] Could not create request: %v", err) + return + } + req.Header.Set("Authorization", "Bearer "+token) + req.Header.Set("Accept", "application/vnd.github+json") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Logf("[artifacts] Could not list artifacts: %v", err) + return + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Logf("[artifacts] List artifacts returned HTTP %d", resp.StatusCode) + return + } + + var result struct { + Artifacts []struct { + ID int `json:"id"` + Name string `json:"name"` + } `json:"artifacts"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + t.Logf("[artifacts] Could not decode artifact list: %v", err) + return + } + + if len(result.Artifacts) == 0 { + t.Log("[artifacts] No artifacts found for this run") + return + } + + t.Logf("[artifacts] Found %d artifact(s), downloading...", len(result.Artifacts)) + for _, art := range result.Artifacts { + downloadAndExtractArtifact(ctx, token, org, repo, art.ID, art.Name, destDir, t) + } +} + +// downloadAndExtractArtifact downloads a single artifact zip and extracts it. +func downloadAndExtractArtifact(ctx context.Context, token, org, repo string, artifactID int, name, destDir string, t *testing.T) { + t.Helper() + + dlURL := fmt.Sprintf("https://api.github.com/repos/%s/%s/actions/artifacts/%d/zip", org, repo, artifactID) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, dlURL, nil) + if err != nil { + t.Logf("[artifacts] Could not create download request for %s: %v", name, err) + return + } + req.Header.Set("Authorization", "Bearer "+token) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Logf("[artifacts] Could not download %s: %v", name, err) + return + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Logf("[artifacts] Download %s returned HTTP %d", name, resp.StatusCode) + return + } + + // Read the zip into memory (artifacts are typically small). + zipData, err := io.ReadAll(io.LimitReader(resp.Body, 50<<20)) // 50 MB limit + if err != nil { + t.Logf("[artifacts] Could not read %s: %v", name, err) + return + } + + artDir := filepath.Join(destDir, name) + _ = os.MkdirAll(artDir, 0o755) + + zr, err := zip.NewReader(bytes.NewReader(zipData), int64(len(zipData))) + if err != nil { + // Not a zip — save raw content. + rawPath := filepath.Join(destDir, name+".bin") + _ = os.WriteFile(rawPath, zipData, 0o644) + t.Logf("[artifacts] %s is not a zip, saved raw to %s", name, rawPath) + return + } + + for _, f := range zr.File { + outPath := filepath.Join(artDir, f.Name) + + // Prevent zip slip. + if !strings.HasPrefix(filepath.Clean(outPath), filepath.Clean(artDir)+string(os.PathSeparator)) { + t.Logf("[artifacts] Skipping suspicious path in %s: %s", name, f.Name) + continue + } + + if f.FileInfo().IsDir() { + _ = os.MkdirAll(outPath, 0o755) + continue + } + + _ = os.MkdirAll(filepath.Dir(outPath), 0o755) + rc, err := f.Open() + if err != nil { + t.Logf("[artifacts] Could not open %s/%s: %v", name, f.Name, err) + continue + } + data, err := io.ReadAll(rc) + rc.Close() + if err != nil { + t.Logf("[artifacts] Could not read %s/%s: %v", name, f.Name, err) + continue + } + if err := os.WriteFile(outPath, data, 0o644); err != nil { + t.Logf("[artifacts] Could not write %s: %v", outPath, err) + continue + } + } + + fmt.Fprintf(os.Stderr, "::notice::Extracted artifact %q (%d files) to %s\n", name, len(zr.File), artDir) + t.Logf("[artifacts] Extracted %s (%d files) to %s", name, len(zr.File), artDir) +} + // runUnenrollmentTest disables test-repo in config.yaml, runs install to // dispatch reconciliation, verifies the removal PR, merges it, and confirms // the shim is gone from the default branch. -func runUnenrollmentTest(t *testing.T, env *e2eEnv, orgCfg *config.OrgConfig, agentCreds []layers.AgentCredentials, enrolledRepoIDs []int64) { +func runUnenrollmentTest(t *testing.T, env *e2eEnv) { t.Helper() ctx := context.Background() - // Update config.yaml to disable test-repo. + cfgData, err := env.client.GetFileContent(ctx, env.org, forge.ConfigRepoName, "config.yaml") + require.NoError(t, err, "reading config.yaml") + orgCfg, err := config.ParseOrgConfig(cfgData) + require.NoError(t, err, "parsing config.yaml") orgCfg.Repos[testRepo] = config.RepoConfig{Enabled: false} - cfgData, err := orgCfg.Marshal() + updatedCfg, err := orgCfg.Marshal() require.NoError(t, err, "marshaling updated config") - - err = env.client.CreateOrUpdateFile(ctx, testOrg, forge.ConfigRepoName, "config.yaml", "chore: disable test-repo for unenrollment test", cfgData) - require.NoError(t, err, "updating config.yaml with disabled repo") + err = env.client.CreateOrUpdateFile(ctx, env.org, forge.ConfigRepoName, + "config.yaml", "chore: disable test-repo for unenrollment test", updatedCfg) + require.NoError(t, err, "updating config.yaml") t.Logf("Set %s to enabled: false in config.yaml", testRepo) - // Wait for GitHub to process the push. time.Sleep(5 * time.Second) - // Run install with no enabled repos and test-repo as disabled. - user, err := env.client.GetAuthenticatedUser(ctx) - require.NoError(t, err) - allRepos, err := env.client.ListOrgRepos(ctx, testOrg) - require.NoError(t, err) - hasPrivate := hasPrivateRepos(allRepos) - - stack := buildTestLayerStack(testOrg, env.client, orgCfg, env.printer, user, hasPrivate, nil, agentCreds, enrolledRepoIDs, nil) - err = stack.InstallAll(ctx) - require.NoError(t, err, "install with disabled repo should succeed") + reinstallArgs := []string{ + "admin", "install", env.org, + "--skip-app-setup", + "--skip-mint-check", + "--mint-url", env.cfg.mintURL, + "--app-set", e2eAppSet, + "--enroll-none", + } + if env.cfg.gcpProjectID != "" { + reinstallArgs = append(reinstallArgs, "--inference-project", env.cfg.gcpProjectID) + } + runCLI(t, env.binary, env.token, reinstallArgs...) - // Verify removal PR exists. - prs, err := env.client.ListRepoPullRequests(ctx, testOrg, testRepo) + prs, err := env.client.ListRepoPullRequests(ctx, env.org, testRepo) require.NoError(t, err, "listing PRs for %s", testRepo) - var removalPR *forge.ChangeProposal for _, pr := range prs { if pr.Title == "chore: disconnect from fullsend agent pipeline" { @@ -787,61 +605,10 @@ func runUnenrollmentTest(t *testing.T, env *e2eEnv, orgCfg *config.OrgConfig, ag } require.NotNil(t, removalPR, "removal PR should exist for %s", testRepo) t.Logf("Found removal PR #%d: %s", removalPR.Number, removalPR.URL) - - // Merge the removal PR. - err = env.client.MergeChangeProposal(ctx, testOrg, testRepo, removalPR.Number) + err = env.client.MergeChangeProposal(ctx, env.org, testRepo, removalPR.Number) require.NoError(t, err, "merging removal PR") - t.Logf("Merged removal PR #%d", removalPR.Number) - - // Wait for merge to propagate. time.Sleep(5 * time.Second) - - // Verify shim no longer exists on the default branch. - _, err = env.client.GetFileContent(ctx, testOrg, testRepo, ".github/workflows/fullsend.yaml") - assert.True(t, forge.IsNotFound(err), "shim workflow should be removed from %s after merging removal PR", testRepo) - t.Logf("Verified shim is gone from %s", testRepo) - - // Re-enable the repo in config for subsequent test phases. - orgCfg.Repos[testRepo] = config.RepoConfig{Enabled: true} -} - -// --- Utility functions --- - -func buildTestLayerStack( - org string, - client forge.Client, - cfg *config.OrgConfig, - printer *ui.Printer, - user string, - hasPrivate bool, - enabledRepos []string, - agentCreds []layers.AgentCredentials, - enrolledRepoIDs []int64, - inferenceProvider inference.Provider, -) *layers.Stack { - return layers.NewStack( - layers.NewConfigRepoLayer(org, client, cfg, printer, hasPrivate), - layers.NewWorkflowsLayer(org, client, printer, user), - layers.NewSecretsLayer(org, client, agentCreds, printer).WithOIDCMode(), - layers.NewInferenceLayer(org, client, inferenceProvider, printer), - layers.NewOIDCDispatchLayer(org, client, enrolledRepoIDs, &e2eDispatcher{}, printer), - layers.NewEnrollmentLayer(org, client, enabledRepos, cfg.DisabledRepos(), printer), - ) -} - -func repoNameList(repos []forge.Repository) []string { - names := make([]string, len(repos)) - for i, r := range repos { - names[i] = r.Name - } - return names -} - -func hasPrivateRepos(repos []forge.Repository) bool { - for _, r := range repos { - if r.Private { - return true - } - } - return false + _, err = env.client.GetFileContent(ctx, env.org, testRepo, ".github/workflows/fullsend.yaml") + require.True(t, forge.IsNotFound(err), "shim should be removed from %s after unenrollment", testRepo) + t.Log("Verified shim is gone") } diff --git a/e2e/admin/browser.go b/e2e/admin/browser.go deleted file mode 100644 index a522a31ef..000000000 --- a/e2e/admin/browser.go +++ /dev/null @@ -1,435 +0,0 @@ -//go:build e2e - -package admin - -import ( - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "strings" - "time" - - xhtml "golang.org/x/net/html" - - "github.com/playwright-community/playwright-go" -) - -// PlaywrightBrowserOpener implements appsetup.BrowserOpener using a -// Playwright browser page with a pre-authenticated persistent context. -type PlaywrightBrowserOpener struct { - page playwright.Page - logf func(string, ...any) - screenshotDir string -} - -// NewPlaywrightBrowserOpener creates a new PlaywrightBrowserOpener -// using the given Playwright page. -func NewPlaywrightBrowserOpener(page playwright.Page, logf func(string, ...any), screenshotDir string) *PlaywrightBrowserOpener { - return &PlaywrightBrowserOpener{page: page, logf: logf, screenshotDir: screenshotDir} -} - -// Open navigates the Playwright page to the given URL and handles the -// expected interactions based on the page type. -func (b *PlaywrightBrowserOpener) Open(_ context.Context, url string) error { - b.logf("[browser] Open called with URL: %s", url) - - // Local manifest form — fetch via HTTP to avoid cross-origin SameSite - // cookie issues, then submit from within GitHub's origin. - if strings.Contains(url, "127.0.0.1") { - return b.handleLocalFormSubmission(url) - } - - if _, err := b.page.Goto(url, playwright.PageGotoOptions{ - WaitUntil: playwright.WaitUntilStateDomcontentloaded, - Timeout: playwright.Float(10000), - }); err != nil { - saveDebugScreenshot(b.page, b.screenshotDir, "browser-goto-failed", b.logf) - return fmt.Errorf("navigating to %s: %w", url, err) - } - - pageURL := b.page.URL() - b.logf("[browser] After Goto, page URL: %s", pageURL) - - switch { - case strings.Contains(pageURL, "/settings/apps/new"), - strings.Contains(pageURL, "/settings/apps/manifest"): - return b.handleCreateAppPage() - case strings.Contains(pageURL, "/installations/select_target"): - return b.handleSelectTargetPage() - case strings.Contains(pageURL, "/installations/new"): - return b.handleInstallAppPage() - default: - saveDebugScreenshot(b.page, b.screenshotDir, "browser-unexpected-url", b.logf) - return fmt.Errorf("unexpected URL: %s", pageURL) - } -} - -// handleLocalFormSubmission fetches the local form via HTTP, extracts the -// manifest (which already contains redirect_url), then submits from -// GitHub's origin so that session cookies (SameSite=Lax) are included -// in the POST. -func (b *PlaywrightBrowserOpener) handleLocalFormSubmission(localURL string) error { - httpClient := &http.Client{Timeout: 10 * time.Second} - resp, err := httpClient.Get(localURL) - if err != nil { - return fmt.Errorf("fetching local form page: %w", err) - } - defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) - if err != nil { - return fmt.Errorf("reading local form page: %w", err) - } - content := string(body) - - // Extract manifest and form action from the hidden inputs. - // The v6 manifest flow puts redirect_url inside the manifest JSON, - // not as a separate form field. - manifest, err := extractInputValue(content, "manifest") - if err != nil { - return fmt.Errorf("extracting manifest from form: %w", err) - } - actionURL, err := extractFormAction(content) - if err != nil { - return fmt.Errorf("extracting form action: %w", err) - } - - // Ensure hook_attributes exists in the manifest (GitHub requires it). - var manifestMap map[string]any - if jsonErr := json.Unmarshal([]byte(manifest), &manifestMap); jsonErr != nil { - return fmt.Errorf("parsing manifest JSON: %w", jsonErr) - } - if _, ok := manifestMap["hook_attributes"]; !ok { - manifestMap["hook_attributes"] = map[string]any{ - "url": "https://example.com/webhook", - "active": false, - } - patched, jsonErr := json.Marshal(manifestMap) - if jsonErr != nil { - return fmt.Errorf("re-marshaling manifest: %w", jsonErr) - } - manifest = string(patched) - } - - b.logf("[browser] Extracted manifest (%d bytes), action=%s", len(manifest), actionURL) - - // Navigate to a neutral GitHub page first so we're on the same - // origin and session cookies will be sent with the POST. - if _, err := b.page.Goto("https://github.com/settings", playwright.PageGotoOptions{ - WaitUntil: playwright.WaitUntilStateDomcontentloaded, - Timeout: playwright.Float(10000), - }); err != nil { - b.logf("[browser] Warning: pre-navigate to GitHub settings failed: %v", err) - } - - // Submit the form via JS, passing values as arguments to avoid - // any quoting/escaping issues with string interpolation. - _, err = b.page.Evaluate(`([action, manifest]) => { - const form = document.createElement('form'); - form.method = 'post'; - form.action = action; - const m = document.createElement('input'); - m.type = 'hidden'; m.name = 'manifest'; m.value = manifest; - form.appendChild(m); - document.body.appendChild(form); - form.submit(); - }`, []string{actionURL, manifest}) - if err != nil { - saveDebugScreenshot(b.page, b.screenshotDir, "browser-js-submit-failed", b.logf) - return fmt.Errorf("submitting manifest form via JS: %w", err) - } - - // Wait for navigation to the app creation confirmation page. - // GitHub redirects to /settings/apps/manifest or /settings/apps/new. - if err := b.page.WaitForURL("**/settings/apps/**", playwright.PageWaitForURLOptions{ - Timeout: playwright.Float(10000), - }); err != nil { - pageURL := b.page.URL() - if strings.Contains(pageURL, "/settings/apps/") { - // We're there. - } else if strings.Contains(pageURL, "/callback") { - return nil - } else { - saveDebugScreenshot(b.page, b.screenshotDir, "browser-manifest-redirect-failed", b.logf) - return fmt.Errorf("waiting for manifest page: %w (URL: %s)", err, pageURL) - } - } - - return b.handleCreateAppPage() -} - -// handleCreateAppPage clicks "Create GitHub App" on the confirmation page. -func (b *PlaywrightBrowserOpener) handleCreateAppPage() error { - b.logf("[browser] handleCreateAppPage at URL: %s", b.page.URL()) - - // The button text varies: "Create GitHub App" or "Create GitHub App for {org}". - btn := b.page.Locator("button:has-text('Create GitHub App'), input[type='submit'][value*='Create GitHub App']") - if err := btn.First().WaitFor(playwright.LocatorWaitForOptions{ - State: playwright.WaitForSelectorStateVisible, - Timeout: playwright.Float(5000), - }); err != nil { - saveDebugScreenshot(b.page, b.screenshotDir, "browser-create-btn-failed", b.logf) - return fmt.Errorf("waiting for 'Create GitHub App' button: %w", err) - } - - if err := btn.First().Click(playwright.LocatorClickOptions{ - Timeout: playwright.Float(5000), - }); err != nil { - saveDebugScreenshot(b.page, b.screenshotDir, "browser-create-btn-click-failed", b.logf) - return fmt.Errorf("clicking 'Create GitHub App': %w", err) - } - - // WaitForURL checks the current URL first (no race if redirect already - // completed), then listens for future navigations. GitHub's manifest - // flow can be slow (app creation, key generation), so use a 90-second - // timeout (increased from 30s per #287). - b.logf("[browser] Clicked 'Create GitHub App', waiting up to 90s for callback redirect...") - if err := b.page.WaitForURL("**/callback**", playwright.PageWaitForURLOptions{ - Timeout: playwright.Float(90000), - }); err != nil { - pageURL := b.page.URL() - // Strip query params before logging — the callback URL contains a - // temporary authorization code that should not appear in CI logs. - safeURL := pageURL - if idx := strings.Index(safeURL, "?"); idx != -1 { - safeURL = safeURL[:idx] - } - b.logf("[browser] Timeout waiting for callback, current URL: %s", safeURL) - if strings.Contains(pageURL, "/callback") { - return nil - } - saveDebugScreenshot(b.page, b.screenshotDir, "browser-callback-failed", b.logf) - return fmt.Errorf("waiting for callback timed out (current URL: %s)", safeURL) - } - - return nil -} - -// handleSelectTargetPage selects the target organization on GitHub's -// "select installation target" page, then proceeds to the install page. -func (b *PlaywrightBrowserOpener) handleSelectTargetPage() error { - b.logf("[browser] handleSelectTargetPage at URL: %s", b.page.URL()) - - orgLink := b.page.Locator(fmt.Sprintf("a:has-text('%s')", testOrg)) - if err := orgLink.First().Click(playwright.LocatorClickOptions{ - Timeout: playwright.Float(5000), - }); err != nil { - saveDebugScreenshot(b.page, b.screenshotDir, "browser-select-target-failed", b.logf) - return fmt.Errorf("selecting target org %s: %w", testOrg, err) - } - - if err := b.page.WaitForLoadState(playwright.PageWaitForLoadStateOptions{ - State: playwright.LoadStateDomcontentloaded, - }); err != nil { - return fmt.Errorf("waiting for install page after selecting org: %w", err) - } - - b.logf("[browser] Selected org %s, now at: %s", testOrg, b.page.URL()) - return b.handleInstallAppPage() -} - -// handleInstallAppPage clicks "Install" on the GitHub App installation page. -// Retries navigation if the page 404s (GitHub needs time to provision the app). -func (b *PlaywrightBrowserOpener) handleInstallAppPage() error { - pageURL := b.page.URL() - b.logf("[browser] handleInstallAppPage at URL: %s", pageURL) - - // Retry loop: the app page may 404 briefly after creation. - for attempt := range 5 { - // Check if we got a 404 and need to retry. - is404, _ := b.page.Locator("img[alt='404'], h1:has-text('404')").Count() - if is404 > 0 { - b.logf("[browser] Got 404, retrying in %ds (attempt %d/5)", (attempt+1)*2, attempt+1) - time.Sleep(time.Duration((attempt+1)*2) * time.Second) - if _, err := b.page.Goto(pageURL, playwright.PageGotoOptions{ - WaitUntil: playwright.WaitUntilStateDomcontentloaded, - Timeout: playwright.Float(10000), - }); err != nil { - b.logf("[browser] Warning: retry navigation failed: %v", err) - continue - } - continue - } - - btn := b.page.Locator("button[type='submit']:has-text('Install')") - if err := btn.Click(playwright.LocatorClickOptions{ - Timeout: playwright.Float(5000), - }); err != nil { - if attempt < 4 { - b.logf("[browser] Install button not found, retrying (attempt %d/5): %v", attempt+1, err) - time.Sleep(time.Duration((attempt+1)*2) * time.Second) - if _, navErr := b.page.Goto(pageURL, playwright.PageGotoOptions{ - WaitUntil: playwright.WaitUntilStateDomcontentloaded, - Timeout: playwright.Float(10000), - }); navErr != nil { - b.logf("[browser] Warning: retry navigation failed: %v", navErr) - } - continue - } - saveDebugScreenshot(b.page, b.screenshotDir, "browser-install-btn-failed", b.logf) - return fmt.Errorf("clicking 'Install': %w", err) - } - - // Successfully clicked Install. - break - } - - // Wait for URL to change away from the installations/new page. - if err := b.page.WaitForURL("!**/installations/new**", playwright.PageWaitForURLOptions{ - Timeout: playwright.Float(10000), - }); err != nil { - // Fall through to WaitForLoadState. - b.logf("[browser] Warning: WaitForURL after install timed out: %v", err) - } - if err := b.page.WaitForLoadState(playwright.PageWaitForLoadStateOptions{ - State: playwright.LoadStateDomcontentloaded, - }); err != nil { - return fmt.Errorf("waiting for install to complete: %w", err) - } - b.logf("[browser] After install, page URL: %s", b.page.URL()) - - return nil -} - -// deleteAppViaPlaywright navigates to the app's advanced settings and deletes it. -func deleteAppViaPlaywright(page playwright.Page, slug string, logf func(string, ...any), screenshotDir string) error { - url := fmt.Sprintf("https://github.com/organizations/%s/settings/apps/%s/advanced", testOrg, slug) - if _, err := page.Goto(url, playwright.PageGotoOptions{ - WaitUntil: playwright.WaitUntilStateDomcontentloaded, - Timeout: playwright.Float(10000), - }); err != nil { - return fmt.Errorf("navigating to app settings for %s: %w", slug, err) - } - - // Check if we got a 404 (app doesn't exist) — not an error. - is404, _ := page.Locator("img[alt='404'], h1:has-text('404')").Count() - if is404 > 0 { - logf("[cleanup] App %s does not exist (404), skipping", slug) - return nil - } - - deleteBtn := page.Locator("button:has-text('Delete GitHub App')") - if err := deleteBtn.Click(playwright.LocatorClickOptions{ - Timeout: playwright.Float(3000), - }); err != nil { - saveDebugScreenshot(page, screenshotDir, "app-delete-"+slug, logf) - logf("[cleanup] Delete button not found at %s, current URL: %s", url, page.URL()) - return fmt.Errorf("clicking 'Delete GitHub App' for %s: %w", slug, err) - } - - // GitHub requires typing the app name to confirm deletion. - // After clicking "Delete GitHub App", a modal appears with a text input. - // Wait a moment for the modal animation. - time.Sleep(1 * time.Second) - saveDebugScreenshot(page, screenshotDir, "app-confirm-dialog-"+slug, logf) - - confirmInput := page.Locator("input[type='text']") - if err := confirmInput.Last().WaitFor(playwright.LocatorWaitForOptions{ - State: playwright.WaitForSelectorStateVisible, - Timeout: playwright.Float(5000), - }); err != nil { - saveDebugScreenshot(page, screenshotDir, "app-confirm-wait-"+slug, logf) - return fmt.Errorf("waiting for confirmation input for %s: %w", slug, err) - } - - if err := confirmInput.Last().Fill(slug, playwright.LocatorFillOptions{ - Timeout: playwright.Float(5000), - }); err != nil { - saveDebugScreenshot(page, screenshotDir, "app-confirm-input-"+slug, logf) - return fmt.Errorf("filling app name for deletion of %s: %w", slug, err) - } - logf("[cleanup] Typed app name %q into confirmation input", slug) - - // Click the confirmation button — try multiple possible text variants. - confirmBtn := page.Locator("button:has-text('I understand'), button:has-text('Delete this'), button[type='submit'].btn-danger") - if err := confirmBtn.First().Click(playwright.LocatorClickOptions{ - Timeout: playwright.Float(5000), - }); err != nil { - saveDebugScreenshot(page, screenshotDir, "app-confirm-btn-"+slug, logf) - return fmt.Errorf("confirming deletion of %s: %w", slug, err) - } - - if err := page.WaitForLoadState(playwright.PageWaitForLoadStateOptions{ - State: playwright.LoadStateDomcontentloaded, - }); err != nil { - return fmt.Errorf("waiting for deletion of %s: %w", slug, err) - } - - logf("[cleanup] Deleted GitHub App: %s", slug) - return nil -} - -// extractInputValue extracts the value attribute of a hidden input with the -// given name from raw HTML using proper HTML parsing. The html package -// handles entity decoding automatically. -func extractInputValue(rawHTML, name string) (string, error) { - doc, err := xhtml.Parse(strings.NewReader(rawHTML)) - if err != nil { - return "", fmt.Errorf("parsing HTML: %w", err) - } - var value string - var found bool - var walk func(*xhtml.Node) - walk = func(n *xhtml.Node) { - if found { - return - } - if n.Type == xhtml.ElementNode && n.Data == "input" { - var nameAttr, valueAttr string - for _, a := range n.Attr { - if a.Key == "name" { - nameAttr = a.Val - } - if a.Key == "value" { - valueAttr = a.Val - } - } - if nameAttr == name { - value = valueAttr - found = true - } - } - for c := n.FirstChild; c != nil; c = c.NextSibling { - walk(c) - } - } - walk(doc) - if !found { - return "", fmt.Errorf("input %q not found in HTML", name) - } - return value, nil -} - -// extractFormAction extracts the action URL from the first form element -// using proper HTML parsing. -func extractFormAction(rawHTML string) (string, error) { - doc, err := xhtml.Parse(strings.NewReader(rawHTML)) - if err != nil { - return "", fmt.Errorf("parsing HTML: %w", err) - } - var action string - var found bool - var walk func(*xhtml.Node) - walk = func(n *xhtml.Node) { - if found { - return - } - if n.Type == xhtml.ElementNode && n.Data == "form" { - for _, a := range n.Attr { - if a.Key == "action" { - action = a.Val - found = true - } - } - } - for c := n.FirstChild; c != nil; c = c.NextSibling { - walk(c) - } - } - walk(doc) - if !found { - return "", fmt.Errorf("form action not found in HTML") - } - return action, nil -} diff --git a/e2e/admin/cleanup.go b/e2e/admin/cleanup.go index 91f65aeb2..61f935e2f 100644 --- a/e2e/admin/cleanup.go +++ b/e2e/admin/cleanup.go @@ -10,110 +10,61 @@ import ( "strings" "testing" - "github.com/playwright-community/playwright-go" - - "github.com/fullsend-ai/fullsend/internal/appsetup" "github.com/fullsend-ai/fullsend/internal/forge" ) // cleanupStaleResources removes leftover resources from previous test runs. // This is the "teardown-first" part of the dual cleanup strategy. -func cleanupStaleResources(ctx context.Context, client forge.Client, page playwright.Page, token, screenshotDir string, t *testing.T) { +func cleanupStaleResources(ctx context.Context, client forge.Client, token, org string, t *testing.T) { t.Helper() t.Log("[cleanup] Scanning for stale resources from previous runs...") // 1. Delete .fullsend repo if it exists. - _, err := client.GetRepo(ctx, testOrg, forge.ConfigRepoName) + _, err := client.GetRepo(ctx, org, forge.ConfigRepoName) if err == nil { t.Logf("[cleanup] Deleting stale %s repo", forge.ConfigRepoName) - if delErr := client.DeleteRepo(ctx, testOrg, forge.ConfigRepoName); delErr != nil { + if delErr := client.DeleteRepo(ctx, org, forge.ConfigRepoName); delErr != nil { t.Logf("[cleanup] Warning: could not delete %s: %v", forge.ConfigRepoName, delErr) } } // 2. Delete stale FULLSEND_DISPATCH_TOKEN org secret if it exists (legacy PAT mode artifact). - dispatchExists, dispatchErr := client.OrgSecretExists(ctx, testOrg, "FULLSEND_DISPATCH_TOKEN") + dispatchExists, dispatchErr := client.OrgSecretExists(ctx, org, "FULLSEND_DISPATCH_TOKEN") if dispatchErr != nil { t.Logf("[cleanup] Warning: could not check dispatch token org secret: %v", dispatchErr) } else if dispatchExists { t.Log("[cleanup] Deleting stale FULLSEND_DISPATCH_TOKEN org secret") - if delErr := client.DeleteOrgSecret(ctx, testOrg, "FULLSEND_DISPATCH_TOKEN"); delErr != nil { + if delErr := client.DeleteOrgSecret(ctx, org, "FULLSEND_DISPATCH_TOKEN"); delErr != nil { t.Logf("[cleanup] Warning: could not delete dispatch token org secret: %v", delErr) } } - // 3. Delete any stale fullsend GitHub Apps via Playwright. - // First, try deleting by expected slug for each role (catches apps that - // were created but never installed, which don't appear in ListOrgInstallations). - for _, role := range defaultRoles { - slug := testOrg + "-" + role // v6 convention: halfsend-fullsend, etc. - t.Logf("[cleanup] Attempting to delete app %s (if it exists)", slug) - if delErr := deleteAppViaPlaywright(page, slug, t.Logf, screenshotDir); delErr != nil { - t.Logf("[cleanup] App %s not found or could not delete: %v", slug, delErr) - } - - newSlug := appsetup.AppSlug(appsetup.DefaultAppSet, role) // current convention: fullsend-ai-triage, etc. - if newSlug != slug { - t.Logf("[cleanup] Attempting to delete app %s (if it exists)", newSlug) - if delErr := deleteAppViaPlaywright(page, newSlug, t.Logf, screenshotDir); delErr != nil { - t.Logf("[cleanup] App %s not found or could not delete: %v", newSlug, delErr) - } - } - - legacySlug := "fullsend-" + role // legacy convention: fullsend-triage, etc. - if legacySlug != slug && legacySlug != newSlug { - t.Logf("[cleanup] Attempting to delete app %s (if it exists)", legacySlug) - if delErr := deleteAppViaPlaywright(page, legacySlug, t.Logf, screenshotDir); delErr != nil { - t.Logf("[cleanup] App %s not found or could not delete: %v", legacySlug, delErr) - } - } - } - - // Also clean up apps found via installations (catches old naming conventions). - installations, err := client.ListOrgInstallations(ctx, testOrg) - if err != nil { - t.Logf("[cleanup] Warning: could not list installations: %v", err) - } else { - for _, inst := range installations { - // Safe: testOrg is a dedicated E2E org with no production apps. - isStale := strings.HasPrefix(inst.AppSlug, testOrg+"-") || // v6: halfsend-* - strings.HasPrefix(inst.AppSlug, appsetup.DefaultAppSet+"-") || // current: fullsend-ai-* - strings.HasPrefix(inst.AppSlug, "fullsend-") // legacy: fullsend-triage, fullsend-halfsend-*, etc. - if isStale { - t.Logf("[cleanup] Deleting stale installed app: %s", inst.AppSlug) - if delErr := deleteAppViaPlaywright(page, inst.AppSlug, t.Logf, screenshotDir); delErr != nil { - t.Logf("[cleanup] Warning: could not delete app %s: %v", inst.AppSlug, delErr) - } - } - } - } - - // 4. Ensure test-repo exists (needed for enrollment testing). - _, err = client.GetRepo(ctx, testOrg, testRepo) + // 3. Ensure test-repo exists (needed for enrollment testing). + _, err = client.GetRepo(ctx, org, testRepo) if forge.IsNotFound(err) { t.Logf("[cleanup] Creating missing %s repo", testRepo) - if _, createErr := client.CreateRepo(ctx, testOrg, testRepo, "E2E test repo", false); createErr != nil { + if _, createErr := client.CreateRepo(ctx, org, testRepo, "E2E test repo", false); createErr != nil { t.Logf("[cleanup] Warning: could not create %s: %v", testRepo, createErr) } } - // 5. Delete stale enrollment and unenrollment branches from test-repo. - deleteBranch(ctx, token, testOrg, testRepo, "fullsend/onboard", t) - deleteBranch(ctx, token, testOrg, testRepo, "fullsend/offboard", t) + // 4. Delete stale enrollment and unenrollment branches from test-repo. + deleteBranch(ctx, token, org, testRepo, "fullsend/onboard", t) + deleteBranch(ctx, token, org, testRepo, "fullsend/offboard", t) - // 6. Delete shim workflow from test-repo's default branch (left behind + // 5. Delete shim workflow from test-repo's default branch (left behind // when a previous run merged the enrollment PR in Phase 2.5). - deleteShimWorkflow(ctx, token, testOrg, testRepo, t) + deleteShimWorkflow(ctx, token, org, testRepo, t) - // 7. Close any open fullsend-related PRs in test-repo. - prs, err := client.ListRepoPullRequests(ctx, testOrg, testRepo) + // 6. Close any open fullsend-related PRs in test-repo. + prs, err := client.ListRepoPullRequests(ctx, org, testRepo) if err != nil { t.Logf("[cleanup] Warning: could not list PRs: %v", err) } else { for _, pr := range prs { if strings.Contains(pr.Title, "fullsend") { t.Logf("[cleanup] Closing stale PR #%d: %s", pr.Number, pr.Title) - closePR(ctx, token, testOrg, testRepo, pr.Number, t) + closePR(ctx, token, org, testRepo, pr.Number, t) } } } @@ -121,17 +72,6 @@ func cleanupStaleResources(ctx context.Context, client forge.Client, page playwr t.Log("[cleanup] Stale resource scan complete") } -// registerAppCleanup registers a t.Cleanup that deletes the given app slug. -func registerAppCleanup(t *testing.T, page playwright.Page, slug, screenshotDir string) { - t.Helper() - t.Cleanup(func() { - t.Logf("[cleanup] Deleting app %s via Playwright", slug) - if err := deleteAppViaPlaywright(page, slug, t.Logf, screenshotDir); err != nil { - t.Logf("[cleanup] Warning: could not delete app %s: %v", slug, err) - } - }) -} - // deleteBranch deletes a branch from a repo using the GitHub API directly // (forge.Client doesn't have DeleteBranch). func deleteBranch(ctx context.Context, token, org, repo, branch string, t *testing.T) { @@ -276,7 +216,7 @@ func registerRepoCleanup(t *testing.T, client forge.Client, org, repo string) { } t.Logf("[cleanup] Deleting repo %s/%s", org, repo) if delErr := client.DeleteRepo(ctx, org, repo); delErr != nil { - t.Logf("[cleanup] Warning: could not delete %s/%s: %v", org, repo, delErr) + t.Errorf("[cleanup] could not delete %s/%s: %v", org, repo, delErr) } }) } diff --git a/e2e/admin/cli_test.go b/e2e/admin/cli_test.go new file mode 100644 index 000000000..45b2944c5 --- /dev/null +++ b/e2e/admin/cli_test.go @@ -0,0 +1,15 @@ +//go:build e2e + +package admin + +import ( + "os" + "testing" +) + +func TestBuildCLI(t *testing.T) { + binary := buildCLIBinary(t) + if _, err := os.Stat(binary); err != nil { + t.Fatalf("binary not found at %s: %v", binary, err) + } +} diff --git a/e2e/admin/lock.go b/e2e/admin/lock.go index 0b74d5005..52a20c517 100644 --- a/e2e/admin/lock.go +++ b/e2e/admin/lock.go @@ -117,8 +117,13 @@ func acquireLock(ctx context.Context, client forge.Client, token, org, runID str func tryCreateLock(ctx context.Context, client forge.Client, org, runID string, logf func(string, ...any)) (bool, error) { _, err := client.CreateRepo(ctx, org, lockRepo, "E2E test lock — do not delete manually", false) if err != nil { - // Repo already exists (409 or similar) — someone else got it. - return false, nil + if isRepoAlreadyExists(err) { + // Repo already exists — someone else got it. + return false, nil + } + // Unexpected error (rate limit, auth failure, network). Propagate + // so acquireOrg can distinguish "locked" from "broken". + return false, fmt.Errorf("creating lock repo in %s: %w", org, err) } // Use CreateOrUpdateFile since auto_init creates a default README.md. @@ -150,7 +155,7 @@ func tryCreateLock(ctx context.Context, client forge.Client, org, runID string, func releaseLock(ctx context.Context, client forge.Client, org, runID string, t *testing.T) { content, err := client.GetFileContent(ctx, org, lockRepo, "README.md") if err != nil { - t.Logf("[e2e-lock] Could not read lock file during release: %v", err) + t.Errorf("[e2e-lock] could not read lock file during release: %v", err) return } @@ -160,12 +165,37 @@ func releaseLock(ctx context.Context, client forge.Client, org, runID string, t } if err := client.DeleteRepo(ctx, org, lockRepo); err != nil { - t.Logf("[e2e-lock] Failed to release lock: %v", err) + t.Errorf("[e2e-lock] failed to release lock: %v", err) return } t.Logf("[e2e-lock] Lock released (run: %s)", truncateUUID(runID)) } +// tryReclaimStaleLock checks whether the lock on org is stale (older than +// timeout) and force-acquires it if so. Returns true if the lock was +// reclaimed. This runs during the first pass so stale locks from crashed +// runs don't waste pool capacity. +func tryReclaimStaleLock(ctx context.Context, client forge.Client, token, org, runID string, timeout time.Duration, logf func(string, ...any)) bool { + createdAt, err := getRepoCreatedAt(ctx, token, org, lockRepo) + if err != nil { + logf("[org-pool] Could not check lock age for %s: %v", org, err) + return false + } + age := time.Since(createdAt) + if age <= timeout { + logf("[org-pool] %s lock is fresh (age: %s), skipping", org, age.Round(time.Second)) + return false + } + logf("[org-pool] %s lock is stale (age: %s > %s), force-acquiring", org, age.Round(time.Second), timeout) + _ = client.DeleteRepo(ctx, org, lockRepo) + acquired, err := tryCreateLock(ctx, client, org, runID, logf) + if err != nil { + logf("[org-pool] Error force-acquiring %s: %v", org, err) + return false + } + return acquired +} + // truncateUUID returns the first 8 chars of a UUID for log readability. func truncateUUID(u string) string { if len(u) > 8 { @@ -179,3 +209,14 @@ func isValidUUID(s string) bool { _, err := uuid.Parse(s) return err == nil } + +// isRepoAlreadyExists reports whether the error indicates that CreateRepo +// failed because the repository already exists (422 from GitHub API or +// "already exists" from the fake client). +func isRepoAlreadyExists(err error) bool { + if err == nil { + return false + } + msg := err.Error() + return strings.Contains(msg, "already exists") || strings.Contains(msg, "422") +} diff --git a/e2e/admin/lock_test.go b/e2e/admin/lock_test.go index fd35285c1..cd0e568db 100644 --- a/e2e/admin/lock_test.go +++ b/e2e/admin/lock_test.go @@ -4,6 +4,7 @@ package admin import ( "context" + "fmt" "testing" "time" @@ -12,16 +13,18 @@ import ( "github.com/stretchr/testify/require" ) +const testLockOrg = "halfsend-test" + func TestAcquireLock_NoExistingLock(t *testing.T) { fake := forge.NewFakeClient() ctx := context.Background() runID := "test-uuid-1234" - err := acquireLock(ctx, fake, "", testOrg, runID, 5*time.Minute, t.Logf) + err := acquireLock(ctx, fake, "", testLockOrg, runID, 5*time.Minute, t.Logf) require.NoError(t, err) // Verify the lock repo was created with our UUID. - content, err := fake.GetFileContent(ctx, testOrg, lockRepo, "README.md") + content, err := fake.GetFileContent(ctx, testLockOrg, lockRepo, "README.md") require.NoError(t, err) assert.Equal(t, runID, string(content)) } @@ -32,15 +35,15 @@ func TestReleaseLock_OwnedByUs(t *testing.T) { runID := "test-uuid-1234" // Pre-create the lock repo with our UUID. - _, err := fake.CreateRepo(ctx, testOrg, lockRepo, "E2E test lock", false) + _, err := fake.CreateRepo(ctx, testLockOrg, lockRepo, "E2E test lock", false) require.NoError(t, err) - err = fake.CreateFile(ctx, testOrg, lockRepo, "README.md", "acquire lock", []byte(runID)) + err = fake.CreateFile(ctx, testLockOrg, lockRepo, "README.md", "acquire lock", []byte(runID)) require.NoError(t, err) - releaseLock(ctx, fake, testOrg, runID, t) + releaseLock(ctx, fake, testLockOrg, runID, t) // Verify repo was deleted. - _, err = fake.GetRepo(ctx, testOrg, lockRepo) + _, err = fake.GetRepo(ctx, testLockOrg, lockRepo) assert.True(t, forge.IsNotFound(err)) } @@ -49,14 +52,95 @@ func TestReleaseLock_OwnedBySomeoneElse(t *testing.T) { ctx := context.Background() // Pre-create the lock repo with a different UUID. - _, err := fake.CreateRepo(ctx, testOrg, lockRepo, "E2E test lock", false) + _, err := fake.CreateRepo(ctx, testLockOrg, lockRepo, "E2E test lock", false) require.NoError(t, err) - err = fake.CreateFile(ctx, testOrg, lockRepo, "README.md", "acquire lock", []byte("other-uuid")) + err = fake.CreateFile(ctx, testLockOrg, lockRepo, "README.md", "acquire lock", []byte("other-uuid")) require.NoError(t, err) - releaseLock(ctx, fake, testOrg, "our-uuid", t) + releaseLock(ctx, fake, testLockOrg, "our-uuid", t) // Repo should NOT have been deleted (not our lock). - _, err = fake.GetRepo(ctx, testOrg, lockRepo) + _, err = fake.GetRepo(ctx, testLockOrg, lockRepo) assert.NoError(t, err) } + +func TestAcquireOrg_FirstOrgAvailable(t *testing.T) { + fake := forge.NewFakeClient() + ctx := context.Background() + + // Save and restore the real pool. + origPool := orgPool + defer func() { orgPool = origPool }() + orgPool = []string{"test-org-1", "test-org-2", "test-org-3"} + + org, err := acquireOrg(ctx, fake, "", "run-1", 5*time.Second, t.Logf) + require.NoError(t, err) + assert.Contains(t, orgPool, org, "should acquire one of the pool orgs") + + // Verify the lock is held on the acquired org. + content, err := fake.GetFileContent(ctx, org, lockRepo, "README.md") + require.NoError(t, err) + assert.Equal(t, "run-1", string(content)) +} + +func TestAcquireOrg_SkipsLockedOrg(t *testing.T) { + fake := forge.NewFakeClient() + ctx := context.Background() + + origPool := orgPool + defer func() { orgPool = origPool }() + orgPool = []string{"test-org-1", "test-org-2", "test-org-3"} + + // Lock the first org. + fake.CreatedRepos = append(fake.CreatedRepos, forge.Repository{ + Name: lockRepo, + FullName: "test-org-1/" + lockRepo, + }) + fake.FileContents["test-org-1/"+lockRepo+"/README.md"] = []byte("aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee") + + org, err := acquireOrg(ctx, fake, "", "run-2", 5*time.Second, t.Logf) + require.NoError(t, err) + assert.NotEqual(t, "test-org-1", org, "should skip locked test-org-1") + assert.Contains(t, []string{"test-org-2", "test-org-3"}, org, "should acquire an unlocked org") +} + +func TestAcquireOrg_AllLockedTimesOut(t *testing.T) { + fake := forge.NewFakeClient() + ctx := context.Background() + + origPool := orgPool + defer func() { orgPool = origPool }() + orgPool = []string{"test-org-1", "test-org-2"} + + // Lock all orgs by pre-populating directly (same-name repos across + // orgs collide in the fake client's duplicate check). + for _, org := range orgPool { + fake.CreatedRepos = append(fake.CreatedRepos, forge.Repository{ + Name: lockRepo, + FullName: org + "/" + lockRepo, + }) + fake.FileContents[org+"/"+lockRepo+"/README.md"] = []byte("aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee") + } + + // Use a very short timeout so the test doesn't block. + _, err := acquireOrg(ctx, fake, "", "run-3", 1*time.Second, t.Logf) + require.Error(t, err) + assert.Contains(t, err.Error(), "could not acquire any org") +} + +func TestAcquireOrg_PropagatesErrors(t *testing.T) { + fake := forge.NewFakeClient() + ctx := context.Background() + + origPool := orgPool + defer func() { orgPool = origPool }() + orgPool = []string{"test-org-1"} + + // Inject a non-"already exists" error for CreateRepo. + fake.Errors = map[string]error{"CreateRepo": fmt.Errorf("rate limited")} + + // The error from tryCreateLock should be logged and the function + // should fall through to the timeout path. + _, err := acquireOrg(ctx, fake, "", "run-4", 1*time.Second, t.Logf) + require.Error(t, err) +} diff --git a/e2e/admin/prompter.go b/e2e/admin/prompter.go deleted file mode 100644 index c33406548..000000000 --- a/e2e/admin/prompter.go +++ /dev/null @@ -1,25 +0,0 @@ -//go:build e2e - -package admin - -// AutoPrompter implements appsetup.Prompter for non-interactive e2e use. -// It automatically accepts all prompts without human input. -type AutoPrompter struct{} - -// WaitForEnter returns immediately. In e2e tests, Playwright handles -// the browser interactions that the prompt would normally gate on. -func (AutoPrompter) WaitForEnter(_ string) error { - return nil -} - -// Confirm always returns true, accepting any confirmation prompt -// (e.g., reuse existing app). -func (AutoPrompter) Confirm(_ string) (bool, error) { - return true, nil -} - -// ReadLine returns an empty string. In e2e tests, the PEM recovery -// path is not exercised (apps are freshly created each run). -func (AutoPrompter) ReadLine(_ string) (string, error) { - return "", nil -} diff --git a/e2e/admin/testutil.go b/e2e/admin/testutil.go index 91c32b740..5d39593fd 100644 --- a/e2e/admin/testutil.go +++ b/e2e/admin/testutil.go @@ -7,8 +7,12 @@ import ( "encoding/json" "fmt" "io" + "math/rand/v2" "net/http" "os" + "os/exec" + "path/filepath" + "strings" "testing" "time" @@ -17,9 +21,6 @@ import ( ) const ( - // testOrg is the dedicated GitHub org for e2e tests. - testOrg = "halfsend" - // testRepo is a pre-existing repo in the test org for enrollment testing. testRepo = "test-repo" @@ -27,7 +28,8 @@ const ( lockRepo = "e2e-lock" // defaultLockTimeout is how long to wait for the lock before giving up. - defaultLockTimeout = 2 * time.Minute + // This is only used as the fallback if ALL orgs are locked. + defaultLockTimeout = 20 * time.Minute // lockPollInterval is how often to poll while waiting for the lock. lockPollInterval = 30 * time.Second @@ -37,15 +39,90 @@ const ( freshLockThreshold = 1 * time.Minute ) +// orgPool is the set of GitHub orgs available for parallel e2e test runs. +// Each run acquires a lock on one org before proceeding. +var orgPool = []string{ + "halfsend-01", + "halfsend-02", + // "halfsend-03", // not yet enrolled in mint + // "halfsend-04", // not yet enrolled in mint + // "halfsend-05", // not yet enrolled in mint + // "halfsend-06", // not yet enrolled in mint +} + +// acquireOrg scans the org pool for an unlocked org and acquires its lock. +// If all orgs are locked, it falls back to waiting on each org in sequence, +// bounded by a single shared deadline (not per-org). Returns the org name. +func acquireOrg(ctx context.Context, client forge.Client, token, runID string, timeout time.Duration, logf func(string, ...any)) (string, error) { + // Shuffle the pool so concurrent runners don't all compete for the + // same first org (thundering herd). + shuffled := make([]string, len(orgPool)) + copy(shuffled, orgPool) + rand.Shuffle(len(shuffled), func(i, j int) { shuffled[i], shuffled[j] = shuffled[j], shuffled[i] }) + + // First pass: try each org without waiting. If a lock exists but is + // stale (older than timeout), force-acquire it so we don't waste pool + // capacity on crashed runs. + for _, org := range shuffled { + logf("[org-pool] Trying to acquire %s...", org) + acquired, err := tryCreateLock(ctx, client, org, runID, logf) + if err != nil { + logf("[org-pool] Error trying %s: %v", org, err) + continue + } + if acquired { + logf("[org-pool] Acquired %s", org) + return org, nil + } + // Lock exists — check if it's stale and force-acquire if so. + if token != "" { + if reclaimed := tryReclaimStaleLock(ctx, client, token, org, runID, timeout, logf); reclaimed { + return org, nil + } + } + logf("[org-pool] %s is locked, trying next", org) + } + + // All orgs are locked. Fall back to waiting with a shared deadline + // across all orgs so total wait is bounded by timeout, not N*timeout. + logf("[org-pool] All %d orgs are locked, waiting with total timeout %s", len(orgPool), timeout) + deadlineCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + var lastErr error + for _, org := range shuffled { + if deadlineCtx.Err() != nil { + break + } + // Pass remaining context time, not the full timeout, so that + // each org only uses its fair share of the shared deadline. + err := acquireLock(deadlineCtx, client, token, org, runID, timeout, logf) + if err == nil { + return org, nil + } + lastErr = err + logf("[org-pool] Could not acquire %s: %v", org, err) + } + + if lastErr != nil { + return "", fmt.Errorf("could not acquire any org from pool after %s (tried %d orgs): %w", timeout, len(orgPool), lastErr) + } + return "", fmt.Errorf("could not acquire any org from pool after %s (tried %d orgs)", timeout, len(orgPool)) +} + // defaultRoles is the standard set of agent roles. -var defaultRoles = []string{"fullsend", "triage", "coder", "review"} +var defaultRoles = []string{"fullsend", "triage", "coder", "review", "retro", "prioritize"} + +// e2eAppSet is the app set prefix used by the shared public GitHub Apps. +const e2eAppSet = "fullsend-ai" // envConfig holds required environment configuration. type envConfig struct { - sessionFile string - password string - totpSecret string - lockTimeout time.Duration + sessionFile string + password string + totpSecret string + mintURL string + gcpProjectID string + lockTimeout time.Duration } // loadEnvConfig reads and validates required env vars. Calls t.Skip if @@ -65,6 +142,13 @@ func loadEnvConfig(t *testing.T) envConfig { password := os.Getenv("E2E_GITHUB_PASSWORD") totpSecret := os.Getenv("E2E_GITHUB_TOTP_SECRET") + mintURL := os.Getenv("E2E_MINT_URL") + if mintURL == "" { + t.Skip("E2E_MINT_URL not set, skipping e2e test") + } + + gcpProjectID := os.Getenv("E2E_GCP_PROJECT_ID") + lockTimeout := defaultLockTimeout if v := os.Getenv("E2E_LOCK_TIMEOUT"); v != "" { d, err := time.ParseDuration(v) @@ -75,10 +159,12 @@ func loadEnvConfig(t *testing.T) envConfig { } return envConfig{ - sessionFile: sessionFile, - password: password, - totpSecret: totpSecret, - lockTimeout: lockTimeout, + sessionFile: sessionFile, + password: password, + totpSecret: totpSecret, + mintURL: mintURL, + gcpProjectID: gcpProjectID, + lockTimeout: lockTimeout, } } @@ -120,24 +206,48 @@ func getRepoCreatedAt(ctx context.Context, token, org, repo string) (time.Time, return result.CreatedAt, nil } -// e2eDispatcher is a no-op dispatch.Dispatcher for e2e tests. It returns a -// dummy mint URL so the OIDC dispatch layer can create org variables without -// provisioning real cloud infrastructure. -type e2eDispatcher struct{} - -func (d *e2eDispatcher) Name() string { return "e2e-test" } - -func (d *e2eDispatcher) Provision(_ context.Context) (map[string]string, error) { - return map[string]string{"FULLSEND_MINT_URL": "https://e2e-test.example.com/mint"}, nil +// buildCLIBinary compiles the fullsend CLI binary once per test run. +func buildCLIBinary(t *testing.T) string { + t.Helper() + modRoot, err := exec.Command("go", "list", "-m", "-f", "{{.Dir}}").Output() + if err != nil { + t.Fatalf("finding module root: %v", err) + } + binary := filepath.Join(t.TempDir(), "fullsend") + cmd := exec.Command("go", "build", "-o", binary, "./cmd/fullsend/") + cmd.Dir = strings.TrimSpace(string(modRoot)) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("building fullsend binary: %s\n%s", err, out) + } + return binary } -func (d *e2eDispatcher) StoreAgentPEM(_ context.Context, _, _ string, _ []byte) error { return nil } +// runCLI executes the fullsend CLI with the given args, passing GITHUB_TOKEN. +// The working directory is set to the module root so that --vendor-fullsend-binary +// can find ./cmd/fullsend/ (same as a user running from the repo root). +func runCLI(t *testing.T, binary, token string, args ...string) string { + t.Helper() + t.Logf("[cli] fullsend %s", strings.Join(args, " ")) -func (d *e2eDispatcher) OrgSecretNames() []string { return nil } + modRoot, err := exec.Command("go", "list", "-m", "-f", "{{.Dir}}").Output() + if err != nil { + t.Fatalf("finding module root for runCLI: %v", err) + } -func (d *e2eDispatcher) OrgVariableNames() []string { return []string{"FULLSEND_MINT_URL"} } + cmd := exec.Command(binary, args...) + cmd.Dir = strings.TrimSpace(string(modRoot)) + cmd.Env = append(os.Environ(), "GITHUB_TOKEN="+token) + out, runErr := cmd.CombinedOutput() + output := string(out) + t.Logf("[cli] output:\n%s", output) + if runErr != nil { + t.Fatalf("[cli] fullsend %s failed: %v\n%s", strings.Join(args, " "), runErr, output) + } + return output +} -// retryOnNotFound retries an operation up to maxAttempts times with exponential +// retryOnNotFound retries an operation up to maxAttempts times with linear // backoff when it returns a not-found error (GitHub eventual consistency). func retryOnNotFound(ctx context.Context, maxAttempts int, fn func() error) error { var err error diff --git a/hack/setup-new-e2e-org.sh b/hack/setup-new-e2e-org.sh new file mode 100755 index 000000000..91e60833f --- /dev/null +++ b/hack/setup-new-e2e-org.sh @@ -0,0 +1,243 @@ +#!/usr/bin/env bash +# setup-new-e2e-org — provision a halfsend-NN org for e2e testing. +# +# Usage: hack/setup-new-e2e-org NN +# +# Idempotent: safe to run multiple times. Checks each prerequisite and +# only acts on what's missing. Pauses for manual steps and verifies +# before continuing. + +set -euo pipefail + +APP_SET="fullsend-ai" +ROLES=(fullsend triage coder review retro prioritize) +BOT_USER="botsend" + +# open_browser tries to open a URL in the default browser. +open_browser() { + local url="$1" + if command -v xdg-open &>/dev/null; then + xdg-open "${url}" 2>/dev/null || true + elif command -v open &>/dev/null; then + open "${url}" 2>/dev/null || true + fi +} + +# wait_for_user pauses until the user presses Enter. +wait_for_user() { + local msg="${1:-Press Enter to continue...}" + read -rp " ${msg}" /dev/null \ + | grep -qx "${slug}"; then + return 0 + fi + done + return 1 +} + +# --- arg parsing --- +if [[ $# -ne 1 ]] || ! [[ "$1" =~ ^[0-9]+$ ]]; then + echo "Usage: $0 NN" >&2 + echo " NN is the org number, e.g. 01, 02, 03" >&2 + exit 1 +fi + +NN="$1" +ORG="halfsend-${NN}" + +echo "==> Setting up e2e org: ${ORG}" +echo + +# --- check prerequisites --- +if ! command -v gh &>/dev/null; then + echo "Error: gh CLI is required. Install from https://cli.github.com/" >&2 + exit 1 +fi + +if ! gh auth status &>/dev/null; then + echo "Error: not authenticated with gh. Run 'gh auth login' first." >&2 + exit 1 +fi + +# --- 1. check if org exists --- +echo "==> Checking if org ${ORG} exists..." +if ! gh api "/orgs/${ORG}" --silent 2>/dev/null; then + url="https://github.com/organizations/plan" + echo " Org ${ORG} does not exist. Opening creation page..." + open_browser "${url}" + echo + echo " Create the org at: ${url}" + echo " Use the name: ${ORG}" + echo + wait_for_user "Press Enter after creating the org..." + + if ! gh api "/orgs/${ORG}" --silent 2>/dev/null; then + echo " ERROR: org ${ORG} still does not exist." + exit 1 + fi +fi +echo " OK: org ${ORG} exists" +echo + +# --- 2. check botsend membership --- +echo "==> Checking if ${BOT_USER} is an owner of ${ORG}..." +membership_role=$(gh api "/orgs/${ORG}/memberships/${BOT_USER}" --jq '.role' 2>/dev/null || echo "none") + +if [[ "${membership_role}" != "admin" ]]; then + if [[ "${membership_role}" == "member" ]]; then + echo " ${BOT_USER} is a member but not an owner." + url="https://github.com/orgs/${ORG}/people" + echo " Promote them to owner at: ${url}" + open_browser "${url}" + else + echo " ${BOT_USER} is not a member. Sending invitation..." + if gh api "/orgs/${ORG}/invitations" \ + -f login="${BOT_USER}" \ + -f role="admin" \ + --silent 2>/dev/null; then + echo " Invitation sent to ${BOT_USER}." + else + echo " Could not send invitation (may already be pending)." + fi + echo " Accept/check at: https://github.com/orgs/${ORG}/people" + fi + echo + wait_for_user "Press Enter after ${BOT_USER} is an owner..." + + membership_role=$(gh api "/orgs/${ORG}/memberships/${BOT_USER}" --jq '.role' 2>/dev/null || echo "none") + if [[ "${membership_role}" != "admin" ]]; then + echo " ERROR: ${BOT_USER} is still not an owner (role: ${membership_role})." + exit 1 + fi +fi +echo " OK: ${BOT_USER} is an owner" +echo + +# --- 3. check test-repo exists --- +echo "==> Checking if ${ORG}/test-repo exists..." +if gh api "/repos/${ORG}/test-repo" --silent 2>/dev/null; then + echo " OK: test-repo exists" +else + echo " Creating test-repo..." + gh api "/orgs/${ORG}/repos" \ + -f name="test-repo" \ + -f description="E2E test repo" \ + -F private=false \ + --silent + echo " OK: created test-repo" +fi +echo + +# --- 4. check app installations --- +echo "==> Checking app installations..." +org_id=$(gh api "/orgs/${ORG}" --jq '.id') +installed_apps=$(gh api "/orgs/${ORG}/installations" --jq '.installations[].app_slug' 2>/dev/null || echo "") + +for role in "${ROLES[@]}"; do + slug="${APP_SET}-${role}" + if echo "${installed_apps}" | grep -qx "${slug}"; then + echo " OK: ${slug} is installed" + continue + fi + + url="https://github.com/apps/${slug}/installations/new/permissions?target_id=${org_id}&target_type=Organization" + echo " MISSING: ${slug}" + echo " Opening: ${url}" + echo " Grant access to 'All repositories' and click Install." + open_browser "${url}" + echo " Waiting for installation (polling every ${POLL_INTERVAL}s, timeout ${POLL_TIMEOUT}s)..." + + if poll_for_app_install "${ORG}" "${slug}"; then + echo " OK: ${slug} is now installed" + else + echo " Timed out waiting for ${slug}." + wait_for_user "Press Enter if you've installed it manually..." + + if ! gh api "/orgs/${ORG}/installations" --jq '.installations[].app_slug' 2>/dev/null \ + | grep -qx "${slug}"; then + echo " ERROR: ${slug} is still not installed." + exit 1 + fi + echo " OK: ${slug} is now installed" + fi +done + +# --- 5. check mint enrollment --- +MINT_PROJECT="${MINT_PROJECT:-it-gcp-konflux-dev-fullsend}" +MINT_REGION="${MINT_REGION:-us-central1}" +MINT_FUNCTION="${MINT_FUNCTION:-fullsend-mint}" + +echo +echo "==> Checking mint enrollment (project: ${MINT_PROJECT}, region: ${MINT_REGION})..." +mint_ok=true + +if ! command -v gcloud &>/dev/null; then + echo " SKIP: gcloud CLI not found, cannot check mint config." + mint_ok=false +else + mint_env=$(gcloud functions describe "${MINT_FUNCTION}" \ + --region "${MINT_REGION}" \ + --project "${MINT_PROJECT}" \ + --format json 2>/dev/null \ + | jq -r '.serviceConfig.environmentVariables' 2>/dev/null) || mint_env="" + + if [[ -z "${mint_env}" || "${mint_env}" == "null" ]]; then + echo " SKIP: could not read mint function env vars." + echo " You may need to run: gcloud auth login" + mint_ok=false + else + # Check ALLOWED_ORGS + allowed_orgs=$(echo "${mint_env}" | jq -r '.ALLOWED_ORGS // ""') + if echo "${allowed_orgs}" | tr ',' '\n' | grep -qx "${ORG}"; then + echo " OK: ${ORG} is in ALLOWED_ORGS" + else + echo " MISSING: ${ORG} is NOT in ALLOWED_ORGS" + mint_ok=false + fi + + # Check ROLE_APP_IDS + role_app_ids=$(echo "${mint_env}" | jq -r '.ROLE_APP_IDS // ""') + missing_roles=() + for role in "${ROLES[@]}"; do + key="${ORG}/${role}" + if echo "${role_app_ids}" | jq -e --arg k "${key}" '.[$k]' &>/dev/null; then + echo " OK: ${key} is in ROLE_APP_IDS" + else + echo " MISSING: ${key} is NOT in ROLE_APP_IDS" + missing_roles+=("${key}") + mint_ok=false + fi + done + fi +fi + +echo +if [[ "${mint_ok}" == "true" ]]; then + echo "==> ${ORG} is fully ready for e2e testing." +else + echo "==> ${ORG} GitHub setup is complete, but mint enrollment needs attention." + echo " Fix the issues above, then:" +fi +echo +echo " Next steps:" +if [[ "${mint_ok}" != "true" ]]; then + echo " 1. Enroll ${ORG} in the mint: run /mint-enroll in Claude Code" + echo " 2. Uncomment \"${ORG}\" in e2e/admin/testutil.go orgPool" + echo " 3. Run: make e2e-test" +else + echo " 1. Uncomment \"${ORG}\" in e2e/admin/testutil.go orgPool" + echo " 2. Run: make e2e-test" +fi diff --git a/internal/forge/fake.go b/internal/forge/fake.go index 088e3b0a1..9cb80c746 100644 --- a/internal/forge/fake.go +++ b/internal/forge/fake.go @@ -205,13 +205,13 @@ func (f *FakeClient) CreateRepo(_ context.Context, org, name, description string fullName := org + "/" + name // Check for duplicates in pre-populated repos. for _, r := range f.Repos { - if r.FullName == fullName || r.Name == name { + if r.FullName == fullName { return nil, fmt.Errorf("repository already exists: %s", fullName) } } // Check for duplicates in previously created repos. for _, r := range f.CreatedRepos { - if r.FullName == fullName || r.Name == name { + if r.FullName == fullName { return nil, fmt.Errorf("repository already exists: %s", fullName) } }