Refactor FederatedCredentials e2e test to use stable repository with config-based disabling

e2eTests/e2eTestHelper.psm1

@@ -326,6 +326,116 @@ function SetRepositorySecret {
     gh secret set $name -b $value --repo $repository
 }
 
+function CleanupOldWorkflowRuns {
+    Param(
+        [string] $repository,
+        [int] $keepCount = 10
+    )
+
+    if (!$repository) {
+        $repository = $defaultRepository
+    }
+
+    Write-Host -ForegroundColor Yellow "`nCleaning up old workflow runs in $repository (keeping last $keepCount)"
+
+    RefreshToken -repository $repository
+
+    # Get all workflow runs, sorted by created_at descending (newest first)
+    $runs = invoke-gh api "/repos/$repository/actions/runs?per_page=100" -silent -returnValue | ConvertFrom-Json
+
+    if ($runs.workflow_runs.Count -le $keepCount) {
+        Write-Host "Found $($runs.workflow_runs.Count) workflow runs, no cleanup needed"
+        return
+    }
+
+    # Get runs to delete (skip the most recent ones)
+    # Runs are already sorted by created_at descending from the API
+    $runsToDelete = $runs.workflow_runs | Select-Object -Skip $keepCount
+
+    Write-Host "Deleting $($runsToDelete.Count) old workflow runs..."
+    foreach ($run in $runsToDelete) {
+        try {
+            Write-Host "Deleting run $($run.id) ($($run.name) - $($run.status))"
+            invoke-gh api /repos/$repository/actions/runs/$($run.id) --method DELETE -silent | Out-Null
+        }
+        catch {
+            Write-Host "Warning: Failed to delete run $($run.id): $_"
+        }
+    }
+    Write-Host "Cleanup completed"
+}
+
+function ResetRepositoryToSource {
+    Param(
+        [string] $repository,
+        [string] $sourceRepository,
+        [string] $branch = "main"
+    )
+
+    if (!$repository) {
+        $repository = $defaultRepository
+    }
+
+    Write-Host -ForegroundColor Yellow "`nResetting repository $repository to match $sourceRepository"
+
+    RefreshToken -repository $repository
+
+    # Clone the repository locally if not already in it
+    $tempPath = [System.IO.Path]::GetTempPath()
+    $repoPath = Join-Path $tempPath ([System.Guid]::NewGuid().ToString())
+    New-Item $repoPath -ItemType Directory | Out-Null
+
+    Push-Location $repoPath
+    try {
+        Write-Host "Cloning $repository..."
+        invoke-gh repo clone $repository . -- --quiet
+        if ($LASTEXITCODE -ne 0) {
+            throw "Failed to clone repository $repository"
+        }
+
+        # Fetch the source repository content
+        Write-Host "Fetching source repository $sourceRepository..."
+        invoke-git remote add source "https://github.com/$sourceRepository.git"
+        if ($LASTEXITCODE -ne 0) {
+            throw "Failed to add remote source for $sourceRepository"
+        }
+
+        invoke-git fetch source $branch --quiet
+        if ($LASTEXITCODE -ne 0) {
+            throw "Failed to fetch branch $branch from source $sourceRepository"
+        }
+
+        # Reset the current branch to match the source
+        Write-Host "Resetting $branch to match source/$branch..."
+        invoke-git checkout $branch --quiet
+        if ($LASTEXITCODE -ne 0) {
+            throw "Failed to checkout branch $branch"
+        }
+
+        invoke-git reset --hard "source/$branch" --quiet
+        if ($LASTEXITCODE -ne 0) {
+            throw "Failed to reset branch $branch to source/$branch"
+        }
+
+        # Force push to update the repository
+        Write-Host "Force pushing changes..."
+        invoke-git push origin $branch --force --quiet
+        if ($LASTEXITCODE -ne 0) {
+            throw "Failed to push changes to $repository"
+        }
+
+        Write-Host "Repository reset completed successfully"
+    }
+    catch {
+        Write-Host "Error resetting repository: $_"
+        throw
+    }
+    finally {
+        Pop-Location
+        Remove-Item -Path $repoPath -Force -Recurse -ErrorAction SilentlyContinue
+    }
+}
+
 function CreateNewAppInFolder {
     Param(
         [string] $folder,

e2eTests/scenarios/FederatedCredentials/runtest.ps1

@@ -29,15 +29,20 @@ Write-Host -ForegroundColor Yellow @'
 # This test uses the bcsamples-bingmaps.appsource repository and will deliver a new build of the app to AppSource.
 # The bcsamples-bingmaps.appsource repository is setup to use an Azure KeyVault for secrets and app signing.
 #
-# During the test, the bcsamples-bingmaps.appsource repository will be copied to a new repository called tmp-bingmaps.appsource.
+# The test requires a stable temporary repository called tmp-bingmaps.appsource that must be manually created
+# with federated credentials configured before running this test.
+# This is required because federated credentials no longer work with repository name-based matching,
+# so the repository must remain stable to maintain the federated credential configuration.
 # tmp-bingmaps.appsource has access to the same Azure KeyVault as bcsamples-bingmaps.appsource using federated credentials.
 # The bcSamples-bingmaps.appsource repository is setup for continuous delivery to AppSource
 # tmp-bingmaps.appsource also has access to the Entra ID app registration for delivering to AppSource using federated credentials.
 # This test will deliver another build of the latest app version already delivered to AppSource (without go-live)
 #
 # This test tests the following scenario:
 #
-#  - Create a new repository called tmp-bingmaps.appsource (based on bcsamples-bingmaps.appsource)
+#  - Verify that the repository tmp-bingmaps.appsource exists (error out if not)
+#  - Reset the repository to match bcsamples-bingmaps.appsource for deterministic state
+#  - Clean up old workflow runs to ensure proper workflow tracking
 #  - Update AL-Go System Files in branch main in tmp-bingmaps.appsource
 #  - Update version numbers in app.json in tmp-bingmaps.appsource in order to not be lower than the version number in AppSource (and not be higher than the next version from bcsamples-bingmaps.appsource)
 #  - Wait for CI/CD in branch main in repository tmp-bingmaps.appsource
@@ -59,34 +64,84 @@ $repository = "$githubOwner/tmp-bingmaps.appsource"
 $template = "https://github.com/$appSourceTemplate"
 $sourceRepository = 'microsoft/bcsamples-bingmaps.appsource' # E2E test will create a copy of this repository
 
-# Create temp repository from sourceRepository
+# Setup authentication and repository
 SetTokenAndRepository -github:$github -githubOwner $githubOwner -appId $e2eAppId -appKey $e2eAppKey -repository $repository
 
+# Check if the repository already exists
+# This repository must exist with federated credentials already configured
 gh api repos/$repository --method HEAD
-if ($LASTEXITCODE -eq 0) {
-    Write-Host "Repository $repository already exists. Deleting it."
-    gh repo delete $repository --yes | Out-Host
-    Start-Sleep -Seconds 30
+if ($LASTEXITCODE -ne 0) {
+    throw "Repository $repository does not exist. The repository must be created manually with federated credentials configured before running this test."
 }
 
-CreateAlGoRepository `
-    -github:$github `
-    -template "https://github.com/$sourceRepository" `
-    -repository $repository `
-    -addRepoSettings @{"ghTokenWorkflowSecretName" = "e2eghTokenWorkflow" }
+# Repository exists - reuse it and reset to source state
+# This is required because federated credentials no longer work with repository name-based matching,
+# so the repository must remain stable across test runs
+Write-Host "Repository $repository exists. Reusing and resetting to match source."
 
+# Reset the repository to match the source repository
+ResetRepositoryToSource -repository $repository -sourceRepository $sourceRepository -branch 'main'
+
+# Clean up old workflow runs to prevent the list from growing and ensure we wait for the correct run
+CleanupOldWorkflowRuns -repository $repository -keepCount 5
+
+# Always set/update secrets (they may have changed or repo may have been reset)
 SetRepositorySecret -repository $repository -name 'Azure_Credentials' -value $azureCredentials
 
+# Re-apply the custom repository settings that were lost during reset
+$tempPath = [System.IO.Path]::GetTempPath()
+$repoPath = Join-Path $tempPath ([System.Guid]::NewGuid().ToString())
+New-Item $repoPath -ItemType Directory | Out-Null
+Push-Location $repoPath
+try {
+    Write-Host "Re-applying repository settings..."
+    invoke-gh repo clone $repository . -- --quiet
+    $repoSettingsFile = ".github\AL-Go-Settings.json"
+    if (Test-Path $repoSettingsFile) {
+        Add-PropertiesToJsonFile -path $repoSettingsFile -properties @{"ghTokenWorkflowSecretName" = "e2eghTokenWorkflow"}
+        invoke-git add $repoSettingsFile
+        invoke-git commit -m "Update repository settings for test" --quiet
+        invoke-git push --quiet
+    }
+    else {
+        Write-Host "Warning: .github\AL-Go-Settings.json not found after cloning. Settings may not be applied correctly."
+    }
+}
+finally {
+    Pop-Location
+    Remove-Item -Path $repoPath -Force -Recurse -ErrorAction SilentlyContinue
+}
+
 # Upgrade AL-Go System Files to test version
-RunUpdateAlGoSystemFiles -directCommit -wait -templateUrl $template -repository $repository | Out-Null
+# Capture the run object to ensure we wait for the correct workflow run
+$updateRun = RunUpdateAlGoSystemFiles -directCommit -wait -templateUrl $template -repository $repository
 
 # Wait for CI/CD to complete
+# The Update AL-Go System Files workflow triggers a CI/CD workflow via push event
+# We need to wait for the CI/CD workflow that was triggered AFTER the update workflow completed
+Write-Host "Waiting for CI/CD workflow to start (triggered by Update AL-Go System Files)..."
 Start-Sleep -Seconds 60
+
+# Get workflow runs that started after the update workflow
+# Use created_at for consistent timestamp comparison, and add a small buffer for timing precision
+$updateCreatedAt = [DateTime]$updateRun.created_at
 $runs = invoke-gh api /repos/$repository/actions/runs -silent -returnValue | ConvertFrom-Json
-$run = $runs.workflow_runs | Select-Object -First 1
+
+# Find the CI/CD workflow run that started after the update workflow was created
+$run = $runs.workflow_runs | Where-Object { 
+    $_.event -eq 'push' -and [DateTime]$_.created_at -gt $updateCreatedAt 
+} | Select-Object -First 1
+
+if (-not $run) {
+    # Fallback to the first workflow run if we can't find one based on timestamp
+    Write-Host "Warning: Could not find CI/CD run based on timestamp, using first run"
+    $run = $runs.workflow_runs | Select-Object -First 1
+}
+
+Write-Host "Waiting for CI/CD workflow run $($run.id) to complete..."
 WaitWorkflow -repository $repository -runid $run.id -noError
 
-# The CI/CD workflow should fail because the version number of the app in thie repository is lower than the version number in AppSource
+# The CI/CD workflow should fail because the version number of the app in the repository is lower than the version number in AppSource
 # Reason being that major.minor from the original bcsamples-bingmaps.appsource is the same and the build number in the newly created repository is lower than the one in AppSource
 # This error is expected we will grab the version number from AppSource, add one to revision number (by switching to versioningstrategy 3 in the tmp repo) and use it in the next run
 $MatchArr = Test-LogContainsFromRun -repository $repository -runid $run.id -jobName 'Deliver to AppSource' -stepName 'Deliver' -expectedText '(?m)^.*The new version number \((\d+(?:\.\d+){3})\) is lower than the existing version number \((\d+(?:\.\d+){3})\) in Partner Center.*$' -isRegEx