diff --git a/eng/common/pipelines/templates/stages/archetype-auto-release-prepare.yml b/eng/common/pipelines/templates/stages/archetype-auto-release-prepare.yml new file mode 100644 index 000000000000..1312b1ea1718 --- /dev/null +++ b/eng/common/pipelines/templates/stages/archetype-auto-release-prepare.yml @@ -0,0 +1,60 @@ +parameters: + - name: DependsOn + type: object + default: + - Signing + - name: Artifacts + type: object + default: [] + - name: Condition + type: string + default: succeeded() + +stages: + # Post-merge auto-release preparation, shared across language repos (this file lives in the synced + # eng/common tree). Runs after the Signing stage: it resolves the merged PR for Build.SourceVersion, + # requires the auto-release label, maps the PR's changed files to releasable packages via the language + # repo's own package detection (Get-PrPkgProperties -> Get-AllPackageInfoFromRepo), and emits the + # auto-release output variables consumed by each language's release stage. + # + # The stage/job/step names below (AutoReleasePrepare / ResolveAutoReleasePackages / resolve) are the + # fixed cross-language output contract. Downstream release stages read outputs as: + # condition: dependencies.AutoReleasePrepare.outputs['ResolveAutoReleasePackages.resolve.'] + # variables: stageDependencies.AutoReleasePrepare.ResolveAutoReleasePackages.outputs['resolve.'] + # Emitted variables: AutoReleasePrNumber, AutoReleaseLabelPresent, HasAutoReleaseArtifacts, + # AutoReleaseArtifactsJson, and ReleaseArtifact_ per declared artifact. + - stage: AutoReleasePrepare + displayName: Auto-release prepare + dependsOn: ${{ parameters.DependsOn }} + condition: ${{ parameters.Condition }} + variables: + - template: /eng/pipelines/templates/variables/globals.yml + - template: /eng/pipelines/templates/variables/image.yml + jobs: + - job: ResolveAutoReleasePackages + displayName: Resolve releasable packages from merged PR + pool: + name: $(LINUXPOOL) + image: $(LINUXVMIMAGE) + os: linux + steps: + # Package detection reads package properties from source, so a checkout is required. + - checkout: self + fetchDepth: 1 + + - template: /eng/common/pipelines/templates/steps/login-to-github.yml + + - task: PowerShell@2 + name: resolve + displayName: Determine releasable packages + env: + # convertToJson emits multi-line JSON, so pass it via env instead of the command line. + AUTORELEASE_ARTIFACTS: ${{ convertToJson(parameters.Artifacts) }} + # Map the secret token to env so it is not written to the task command line. + GH_TOKEN: $(GH_TOKEN) + inputs: + pwsh: true + filePath: $(System.DefaultWorkingDirectory)/eng/common/scripts/Resolve-AutoReleasePackages.ps1 + arguments: > + -CommitSha "$(Build.SourceVersion)" + -RepoId "$(Build.Repository.Name)" diff --git a/eng/common/scripts/AutoRelease-Operations.ps1 b/eng/common/scripts/AutoRelease-Operations.ps1 new file mode 100644 index 000000000000..511ed0241219 --- /dev/null +++ b/eng/common/scripts/AutoRelease-Operations.ps1 @@ -0,0 +1,142 @@ +# Shared auto-release operations used by language repos to resolve the pull request and +# changed-file set that drive post-merge auto-release. Generic GitHub API calls live in +# Invoke-GitHubAPI.ps1; this file holds the auto-release policy and diff-shaping logic. + +. "${PSScriptRoot}\logging.ps1" +. "${PSScriptRoot}\Invoke-GitHubAPI.ps1" + +# Resolves the auto-release pull request for a commit SHA. +# +# Applies the shared auto-release selection policy: +# 1. Look up the pull requests associated with the commit. +# 2. Keep only pull requests merged into the target branch. +# 3. Select the most recently merged one. +# 4. Re-fetch that pull request by number to read its authoritative merge state and labels. +# 5. Require it to be merged into the target branch and to carry the auto-release label. +# +# Returns a result object: +# PullRequest : the selected PR object, or $null +# PullRequestNumber : the selected PR number, or $null +# IsEligible : $true only when a merged, labeled PR was found +# SkipReason : a human readable reason when IsEligible is $false +function Get-GitHubAutoReleasePullRequestForCommit { + param ( + $RepoOwner, + $RepoName, + $RepoId = "$RepoOwner/$RepoName", + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + $CommitSha, + $TargetBranch = "main", + $RequiredLabel = "auto-release", + [ValidateNotNullOrEmpty()] + [Parameter(Mandatory = $true)] + $AuthToken + ) + + $result = [PSCustomObject]@{ + PullRequest = $null + PullRequestNumber = $null + IsEligible = $false + SkipReason = "" + } + + $associatedPullRequests = @(Get-GitHubPullRequestsForCommit -RepoId $RepoId -CommitSha $CommitSha -AuthToken $AuthToken) + + $mergedToTarget = @( + $associatedPullRequests | + Where-Object { $_.merged_at -and $_.base.ref -eq $TargetBranch } + ) + + if ($mergedToTarget.Count -eq 0) { + $result.SkipReason = "No merged pull request targeting '$TargetBranch' was associated with commit '$CommitSha'." + return $result + } + + $selectedPullRequest = $mergedToTarget | + Sort-Object { [datetime]$_.merged_at } -Descending | + Select-Object -First 1 + + $result.PullRequestNumber = $selectedPullRequest.number + + # Re-fetch the pull request by number to read its authoritative state. The commit -> pulls payload is + # a secondary representation whose labels and merge state can lag the canonical pull request, so + # eligibility (merge target + required label) is decided against this authoritative payload. + $pullRequest = Get-GitHubPullRequest -RepoId $RepoId -PullRequestNumber $selectedPullRequest.number -AuthToken $AuthToken + $result.PullRequest = $pullRequest + + if (-not $pullRequest.merged_at -or $pullRequest.base.ref -ne $TargetBranch) { + $result.SkipReason = "Pull request #$($pullRequest.number) is not merged into '$TargetBranch'." + return $result + } + + $labels = @($pullRequest.labels | ForEach-Object { $_.name }) + if ($RequiredLabel -notin $labels) { + $result.SkipReason = "Pull request #$($pullRequest.number) does not have the required label '$RequiredLabel'." + return $result + } + + $result.IsEligible = $true + return $result +} + +# Converts GitHub pull request file entries into the Azure SDK PR diff object shape +# consumed by package-detection tooling (compatible with Generate-PR-Diff.ps1 output). +# +# Parameters: +# PullRequestNumber : the PR number recorded in the diff object. +# PullRequestFiles : the file entries returned by Get-GitHubPullRequestFiles. +# ExcludePaths : optional paths to record on the diff object. +# +# Returns a PSCustomObject with ChangedFiles, ChangedServices, ExcludePaths, DeletedFiles, PRNumber. +function New-GitHubPullRequestDiffObject { + param ( + [Parameter(Mandatory = $true)] + $PullRequestNumber, + [Parameter(Mandatory = $true)] + [AllowEmptyCollection()] + [array] $PullRequestFiles, + [AllowEmptyCollection()] + [array] $ExcludePaths = @() + ) + + $changedFiles = @() + $deletedFiles = @() + + foreach ($file in $PullRequestFiles) { + $filename = "$($file.filename)" -replace '\\', '/' + + if ($file.status -eq 'removed') { + $deletedFiles += $filename + } + else { + $changedFiles += $filename + } + + # For renames, include the previous path as deleted so package detection sees both sides of the move. + if ($file.status -eq 'renamed' -and $file.previous_filename) { + $deletedFiles += ("$($file.previous_filename)" -replace '\\', '/') + } + } + + $changedFiles = @($changedFiles | Where-Object { $_ } | Sort-Object -Unique) + $deletedFiles = @($deletedFiles | Where-Object { $_ } | Sort-Object -Unique) + + $changedServices = @( + $changedFiles + $deletedFiles | + ForEach-Object { if ($_ -match "^sdk/([^/]+)/") { $Matches[1] } } | + Sort-Object -Unique + ) + + if (-not $ExcludePaths) { + $ExcludePaths = @() + } + + return [PSCustomObject]@{ + ChangedFiles = $changedFiles + ChangedServices = $changedServices + ExcludePaths = @($ExcludePaths) + DeletedFiles = $deletedFiles + PRNumber = "$PullRequestNumber" + } +} diff --git a/eng/common/scripts/Invoke-GitHubAPI.ps1 b/eng/common/scripts/Invoke-GitHubAPI.ps1 index 0e5bace3e48c..1322a080e29e 100644 --- a/eng/common/scripts/Invoke-GitHubAPI.ps1 +++ b/eng/common/scripts/Invoke-GitHubAPI.ps1 @@ -114,6 +114,66 @@ function Get-GitHubPullRequest { -MaximumRetryCount 3 } +# Returns the pull requests associated with a commit SHA. +# See https://docs.github.com/rest/commits/commits#list-pull-requests-associated-with-a-commit +function Get-GitHubPullRequestsForCommit { + param ( + $RepoOwner, + $RepoName, + $RepoId = "$RepoOwner/$RepoName", + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + $CommitSha, + [ValidateNotNullOrEmpty()] + [Parameter(Mandatory = $true)] + $AuthToken + ) + # A commit is associated with ~1 pull request, so a single request is sufficient. + $uri = "$GithubAPIBaseURI/$RepoId/commits/$CommitSha/pulls" + + return Invoke-RestMethod ` + -Method GET ` + -Uri $uri ` + -Headers (Get-GitHubApiHeaders -token $AuthToken) ` + -MaximumRetryCount 3 +} + +# Returns the list of files changed in a pull request, following pagination. +# See https://docs.github.com/rest/pulls/pulls#list-pull-requests-files +function Get-GitHubPullRequestFiles { + param ( + $RepoOwner, + $RepoName, + $RepoId = "$RepoOwner/$RepoName", + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + $PullRequestNumber, + [ValidateNotNullOrEmpty()] + [Parameter(Mandatory = $true)] + $AuthToken + ) + + $headers = Get-GitHubApiHeaders -token $AuthToken + $pageSize = 100 + $page = 1 + $files = @() + + do { + $uri = "$GithubAPIBaseURI/$RepoId/pulls/$PullRequestNumber/files?per_page=$pageSize&page=$page" + $response = Invoke-RestMethod ` + -Method GET ` + -Uri $uri ` + -Headers $headers ` + -MaximumRetryCount 3 + + $response = @($response) + if ($response.Count -gt 0) { $files += $response } + $page++ + } while ($response.Count -eq $pageSize) + + return $files +} + function New-GitHubPullRequest { param ( $RepoOwner, diff --git a/eng/common/scripts/Resolve-AutoReleasePackages.ps1 b/eng/common/scripts/Resolve-AutoReleasePackages.ps1 new file mode 100644 index 000000000000..b72f39519349 --- /dev/null +++ b/eng/common/scripts/Resolve-AutoReleasePackages.ps1 @@ -0,0 +1,233 @@ +<# +.SYNOPSIS +Determines which of a pipeline's packages should be auto-released after a labeled PR merge to main. + +.DESCRIPTION +Language-agnostic. Intended to run in an internal post-merge CI run on 'main'. Given the build's merge +commit, this script: + 1. Uses the shared Get-GitHubAutoReleasePullRequestForCommit policy to resolve the pull request for + the commit: it selects the newest PR merged into the base branch (default 'main') and requires the + 'auto-release' label. + 2. Builds a PR diff object (New-GitHubPullRequestDiffObject) from the PR's changed files and reuses + the repo's existing package-detection logic (Get-PrPkgProperties) to identify the changed packages + (honoring triggering paths, deleted files and service-level changes), excluding + validation-only packages. Get-PrPkgProperties delegates to each repo's own + Get-AllPackageInfoFromRepo (language-settings.ps1), so this script works for any language repo. + 3. Intersects those packages with this pipeline's declared artifacts and emits Azure DevOps output + variables consumed by the release stages. Both consumption styles are emitted: + - per-artifact ReleaseArtifact_ booleans, for pipelines that loop over their + compile-time Artifacts list and gate each entry (e.g. .NET); + - AutoReleaseArtifactsJson, the matched declared-artifact objects serialized as a JSON array, + for pipelines that iterate the releasable set at runtime (e.g. Java). + +The script FAILS CLOSED: on any error, or if no qualifying labeled PR / changed package is found, it +emits AutoReleaseLabelPresent=false, HasAutoReleaseArtifacts=false, AutoReleaseArtifactsJson=[] and +ReleaseArtifact_=false for every artifact, and exits 0 so the CI run is not failed. + +.PARAMETER CommitSha +The build source version (merge commit) to resolve the pull request from. Typically $(Build.SourceVersion). + +.PARAMETER RepoId +The GitHub repository id in '/' form. Typically $(Build.Repository.Name). + +.PARAMETER Artifacts +JSON array of the pipeline's declared artifacts. Each entry must have 'name' and 'safeName'; entries may +also carry a 'groupId' (used to disambiguate name collisions across groups) and any other fields the +consuming stage needs (they are passed through unchanged in AutoReleaseArtifactsJson). +Defaults to the AUTORELEASE_ARTIFACTS environment variable, which the pipeline sets to +'${{ convertToJson(parameters.Artifacts) }}' (passed via env because it is multi-line JSON). + +.PARAMETER AuthToken +GitHub token used for API calls. Defaults to the GH_TOKEN environment variable produced by +login-to-github.yml (passed via env so the secret is not written to the task command line). + +.PARAMETER AutoReleaseLabel +The GitHub PR label that opts a merged PR into auto-release. Defaults to 'auto-release'. + +.PARAMETER BaseBranch +The base branch a PR must have been merged into to qualify. Defaults to 'main'. + +.OUTPUTS +Azure DevOps output variables (reference cross-stage via dependencies..outputs['..']): + - AutoReleasePrNumber : the resolved PR number, or empty + - AutoReleaseLabelPresent : 'true' if the resolved merged PR has the auto-release label + - HasAutoReleaseArtifacts : 'true' if at least one declared package is releasable + - AutoReleaseArtifactsJson : JSON array of the matched declared-artifact objects (or '[]') + - ReleaseArtifact_ : 'true'/'false' per declared artifact +#> +#Requires -Version 7.0 +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)][string] $CommitSha, + [Parameter(Mandatory = $true)][string] $RepoId, + [string] $Artifacts = $env:AUTORELEASE_ARTIFACTS, + [string] $AuthToken = $env:GH_TOKEN, + [string] $AutoReleaseLabel = 'auto-release', + [string] $BaseBranch = 'main' +) + +$ErrorActionPreference = 'Stop' + +# Import shared logic unless a caller (e.g. a test harness) has already provided it. common.ps1 provides +# Get-PrPkgProperties, the GitHub helpers, Set-PipelineVariable, $RepoRoot and language settings; +# AutoRelease-Operations.ps1 provides the shared auto-release PR selection and diff helpers. StrictMode +# is intentionally not enabled here because the shared package-detection code is not written to run under it. +if (-not (Get-Command 'Get-PrPkgProperties' -ErrorAction SilentlyContinue)) { + . (Join-Path $PSScriptRoot "common.ps1") +} +if (-not (Get-Command 'Get-GitHubAutoReleasePullRequestForCommit' -ErrorAction SilentlyContinue)) { + . (Join-Path $PSScriptRoot "AutoRelease-Operations.ps1") +} + +# Parse the declared artifacts. +$declaredArtifacts = @() +try { + $parsed = $Artifacts | ConvertFrom-Json + if ($null -ne $parsed) { $declaredArtifacts = @($parsed) } +} +catch { + Write-Host "##[warning]Failed to parse -Artifacts JSON; treating as empty. $($_.Exception.Message)" +} + +# Fail-closed defaults: nothing releases unless we positively determine otherwise below. +Set-PipelineVariable -Name 'AutoReleasePrNumber' -Value '' -IsOutput +Set-PipelineVariable -Name 'AutoReleaseLabelPresent' -Value 'false' -IsOutput +Set-PipelineVariable -Name 'HasAutoReleaseArtifacts' -Value 'false' -IsOutput +Set-PipelineVariable -Name 'AutoReleaseArtifactsJson' -Value '[]' -IsOutput +foreach ($artifact in $declaredArtifacts) { + if ($artifact.PSObject.Properties['safeName'] -and $artifact.safeName) { + Set-PipelineVariable -Name "ReleaseArtifact_$($artifact.safeName)" -Value 'false' -IsOutput + } +} + +function Invoke-AutoReleaseResolution { + Write-Host "Resolving the auto-release pull request for commit '$CommitSha' in '$RepoId'..." + $release = Get-GitHubAutoReleasePullRequestForCommit ` + -RepoId $RepoId ` + -CommitSha $CommitSha ` + -TargetBranch $BaseBranch ` + -RequiredLabel $AutoReleaseLabel ` + -AuthToken $AuthToken + + if ($release.PullRequestNumber) { + Set-PipelineVariable -Name 'AutoReleasePrNumber' -Value "$($release.PullRequestNumber)" -IsOutput + } + + if (-not $release.IsEligible) { + Write-Host "Skipping auto-release: $($release.SkipReason)" + return + } + + $pr = $release.PullRequest + Write-Host "PR #$($pr.number) is eligible for auto-release (merged into '$BaseBranch' with the '$AutoReleaseLabel' label)." + Set-PipelineVariable -Name 'AutoReleaseLabelPresent' -Value 'true' -IsOutput + + if ($declaredArtifacts.Count -eq 0) { + Write-Host "No declared artifacts for this pipeline. Nothing to auto-release." + return + } + + # Turn the PR's changed files into a diff object (Generate-PR-Diff.ps1 shape) and reuse the repo's + # package-detection logic to identify the changed packages. + Write-Host "Fetching changed files for PR #$($pr.number)..." + $files = @(Get-GitHubPullRequestFiles -RepoId $RepoId -PullRequestNumber $pr.number -AuthToken $AuthToken) + $diff = New-GitHubPullRequestDiffObject -PullRequestNumber $pr.number -PullRequestFiles $files + Write-Host "PR #$($pr.number) changed $($diff.ChangedFiles.Count) file(s) and deleted $($diff.DeletedFiles.Count) file(s)." + + $diffPath = Join-Path ([System.IO.Path]::GetTempPath()) ("autorelease-diff-" + [System.Guid]::NewGuid().ToString('N') + ".json") + $diff | ConvertTo-Json -Depth 10 | Set-Content -Path $diffPath -Encoding utf8 + + try { + $changedPackages = @(Get-PrPkgProperties -InputDiffJson $diffPath) + } + finally { + Remove-Item -Path $diffPath -ErrorAction SilentlyContinue + } + + # Build the releasable key set. Add each changed package's name and, when the group is known, a + # 'group/name' composite so that pipelines whose declared artifacts carry a groupId (e.g. Java) match + # the correct group and are not confused by name collisions across groups. Packages pulled in solely + # for validation are not releasable. + $releasableKeys = New-Object 'System.Collections.Generic.HashSet[string]' ([System.StringComparer]::OrdinalIgnoreCase) + foreach ($package in $changedPackages) { + if ($package.IncludedForValidation) { continue } + + $names = @() + if ($package.Name) { $names += [string]$package.Name } + if ($package.PSObject.Properties['ArtifactName'] -and $package.ArtifactName) { $names += [string]$package.ArtifactName } + + $group = $null + if ($package.PSObject.Properties['Group'] -and $package.Group) { $group = [string]$package.Group } + + foreach ($name in ($names | Sort-Object -Unique)) { + [void]$releasableKeys.Add($name) + if ($group) { [void]$releasableKeys.Add("$group/$name") } + } + } + + $matchedArtifacts = @() + foreach ($artifact in $declaredArtifacts) { + try { + $name = $artifact.name + $safeName = $artifact.safeName + if (-not $name -or -not $safeName) { + Write-Host " Skipping artifact with missing name/safeName." + continue + } + + $groupId = $null + if ($artifact.PSObject.Properties['groupId'] -and $artifact.groupId) { $groupId = [string]$artifact.groupId } + + # Prefer a group-qualified match when the artifact declares a group; otherwise match by name. + if ($groupId) { + $isMatch = $releasableKeys.Contains("$groupId/$name") + } + else { + $isMatch = $releasableKeys.Contains([string]$name) + } + + if ($isMatch) { + Write-Host " [$name] changed by PR #$($pr.number) -> releasable." + Set-PipelineVariable -Name "ReleaseArtifact_$safeName" -Value 'true' -IsOutput + $matchedArtifacts += $artifact + } + else { + Write-Host " [$name] not changed by PR #$($pr.number)." + } + } + catch { + Write-Host "##[warning]Failed to evaluate an artifact; treating as not releasable. $($_.Exception.Message)" + } + } + + if ($matchedArtifacts.Count -gt 0) { + # Pipe (not -InputObject) with -AsArray so a single match still serializes as a JSON array, '[{...}]'. + $artifactsJson = $matchedArtifacts | ConvertTo-Json -Depth 100 -Compress -AsArray + Set-PipelineVariable -Name 'AutoReleaseArtifactsJson' -Value $artifactsJson -IsOutput + Write-Host "Auto-release packages from PR #$($pr.number): $((@($matchedArtifacts | ForEach-Object { $_.name })) -join ', ')" + Set-PipelineVariable -Name 'HasAutoReleaseArtifacts' -Value 'true' -IsOutput + } + else { + Write-Host "PR #$($pr.number) changed no releasable package in this pipeline." + } +} + +try { + Invoke-AutoReleaseResolution +} +catch { + # Re-emit the fail-closed defaults so a failure after any positive signal was set (e.g. after + # AutoReleaseLabelPresent or a ReleaseArtifact_ flag was flipped to 'true') cannot leak a + # partial "release" decision to downstream stages, regardless of how each consumer gates on the outputs. + Write-Host "##[warning]Auto-release resolution failed; skipping auto-release. $($_.Exception.Message)" + Set-PipelineVariable -Name 'AutoReleaseLabelPresent' -Value 'false' -IsOutput + Set-PipelineVariable -Name 'HasAutoReleaseArtifacts' -Value 'false' -IsOutput + Set-PipelineVariable -Name 'AutoReleaseArtifactsJson' -Value '[]' -IsOutput + foreach ($artifact in $declaredArtifacts) { + if ($artifact.PSObject.Properties['safeName'] -and $artifact.safeName) { + Set-PipelineVariable -Name "ReleaseArtifact_$($artifact.safeName)" -Value 'false' -IsOutput + } + } +} + +exit 0 diff --git a/eng/common/scripts/logging.ps1 b/eng/common/scripts/logging.ps1 index ae0c85438659..9178ec9402fb 100644 --- a/eng/common/scripts/logging.ps1 +++ b/eng/common/scripts/logging.ps1 @@ -126,3 +126,20 @@ function ProcessMsBuildLogLine($line) { } return $line } + +function ConvertTo-DevOpsLoggingValue($value) { + if ($null -eq $value) { + return "" + } + + return "$value".Replace('%', '%25').Replace(';', '%3B').Replace(']', '%5D').Replace("`r", '%0D').Replace("`n", '%0A') +} + +function Set-PipelineVariable($Name, $Value = "", [switch]$IsOutput, [switch]$IsSecret) { + $properties = "variable=$Name" + if ($IsSecret) { $properties += ";issecret=true" } + if ($IsOutput) { $properties += ";isOutput=true" } + + $escapedValue = ConvertTo-DevOpsLoggingValue $Value + Write-Host "##vso[task.setvariable $properties]$escapedValue" +}