Skip to content

Commit 0942ffa

Browse files
CopilotEMaher
Copilot
and
EMaher
authored
Harden Phase 7 teardown sequencing for APIM delete/purge races (#122)
* Harden teardown waits before APIM purge * Mask APIM name in teardown timeout error * Strengthen teardown timeout diagnostics and APIM lookup * powershell version update for devcontainers * updating script to wait for delete before purging apim instance --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: Elizabeth Maher <enewman@microsoft.com>
1 parent a495dd7 commit 0942ffa

3 files changed

Lines changed: 138 additions & 29 deletions

File tree

‎.devcontainer/devcontainer-lock.json‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,10 @@
1010
"resolved": "ghcr.io/devcontainers/features/github-cli@sha256:d22f50b70ed75339b4eed1ba9ecde3a1791f90e88d37936517e3bace0bbad671",
1111
"integrity": "sha256:d22f50b70ed75339b4eed1ba9ecde3a1791f90e88d37936517e3bace0bbad671"
1212
},
13-
"ghcr.io/devcontainers/features/powershell:1": {
14-
"version": "1.5.1",
15-
"resolved": "ghcr.io/devcontainers/features/powershell@sha256:df7baa89598c93bfd15808641d9ec9eb03e0ccdf52e5de4cbbce9ab2d9755d18",
16-
"integrity": "sha256:df7baa89598c93bfd15808641d9ec9eb03e0ccdf52e5de4cbbce9ab2d9755d18"
13+
"ghcr.io/devcontainers/features/powershell:2.0.2": {
14+
"version": "2.0.2",
15+
"resolved": "ghcr.io/devcontainers/features/powershell@sha256:82ea3880c5706ac3963f1b5e940a7f5871835393a5472f80008148b995d58d61",
16+
"integrity": "sha256:82ea3880c5706ac3963f1b5e940a7f5871835393a5472f80008148b995d58d61"
1717
}
1818
}
19-
}
19+
}

‎tests/integration/all-resource-types/phases/run-phase7-teardown.ps1‎

Lines changed: 123 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,10 @@ param(
3131
[Parameter(Mandatory)]
3232
[string]$TargetResourceGroup,
3333

34+
[string]$SourceSubscriptionId,
35+
36+
[string]$TargetSubscriptionId,
37+
3438
[string]$Location = 'eastus2',
3539

3640
[switch]$SkipTeardown
@@ -49,39 +53,139 @@ Write-Host "🧹 PHASE 7 — Teardown"
4953

5054
$sourceApimName = $null
5155
$targetApimName = $null
52-
$sourceApimName = az apim list --resource-group $SourceResourceGroup --query "[0].name" -o tsv 2>$null
53-
$targetApimName = az apim list --resource-group $TargetResourceGroup --query "[0].name" -o tsv 2>$null
5456

55-
Write-Host " Deleting $(Protect-ResourceGroupName -Value $SourceResourceGroup)..."
56-
az group delete --name $SourceResourceGroup --yes --no-wait 2>$null
57-
Write-Host " Deleting $(Protect-ResourceGroupName -Value $TargetResourceGroup)..."
58-
az group delete --name $TargetResourceGroup --yes --no-wait 2>$null
57+
function Get-SubscriptionArgs {
58+
param([string]$SubscriptionId)
5959

60-
Write-Host " ⏳ Waiting for resource group deletions to complete for hard-delete..."
61-
$maxWaitMinutes = 15
62-
$waited = 0
63-
$interval = 30
60+
if ([string]::IsNullOrWhiteSpace($SubscriptionId)) {
61+
return @()
62+
}
63+
64+
return @('--subscription', $SubscriptionId)
65+
}
66+
67+
function Get-GroupExists {
68+
param(
69+
[Parameter(Mandatory)]
70+
[string]$ResourceGroup,
71+
[string]$SubscriptionId
72+
)
73+
74+
$subArgs = Get-SubscriptionArgs -SubscriptionId $SubscriptionId
75+
$exists = az group exists --name $ResourceGroup @subArgs -o tsv 2>$null
76+
if ($LASTEXITCODE -ne 0) {
77+
throw "Failed to query existence for resource group '$(Protect-ResourceGroupName -Value $ResourceGroup)'."
78+
}
79+
80+
return $exists -eq 'true'
81+
}
82+
83+
$sourceListArgs = @('apim', 'list', '--resource-group', $SourceResourceGroup, '--query', '[0].name', '-o', 'tsv') + (Get-SubscriptionArgs -SubscriptionId $SourceSubscriptionId)
84+
$targetListArgs = @('apim', 'list', '--resource-group', $TargetResourceGroup, '--query', '[0].name', '-o', 'tsv') + (Get-SubscriptionArgs -SubscriptionId $TargetSubscriptionId)
85+
86+
$sourceApimName = az @sourceListArgs 2>$null
87+
$targetApimName = az @targetListArgs 2>$null
88+
89+
function Wait-ForResourceGroupsDeletion {
90+
param(
91+
[hashtable[]]$Groups,
92+
[int]$TimeoutMinutes = 60,
93+
[int]$IntervalSeconds = 30
94+
)
95+
96+
$waitedSeconds = 0
97+
$timeoutSeconds = $TimeoutMinutes * 60
98+
99+
while ($waitedSeconds -lt $timeoutSeconds) {
100+
$existingGroups = @()
101+
102+
foreach ($group in $Groups) {
103+
if (Get-GroupExists -ResourceGroup $group.ResourceGroup -SubscriptionId $group.SubscriptionId) {
104+
$existingGroups += $group.ResourceGroup
105+
}
106+
}
64107

65-
while ($waited -lt ($maxWaitMinutes * 60)) {
66-
$srcExists = (az group exists --name $SourceResourceGroup -o tsv 2>$null) -eq 'true'
67-
$tgtExists = (az group exists --name $TargetResourceGroup -o tsv 2>$null) -eq 'true'
108+
if ($existingGroups.Count -eq 0) {
109+
Write-Host " ✅ Resource groups deleted"
110+
return
111+
}
68112

69-
if (-not $srcExists -and -not $tgtExists) {
70-
Write-Host " ✅ Resource groups deleted"
71-
break
113+
$maskedNames = $existingGroups | ForEach-Object { Protect-ResourceGroupName -Value $_ }
114+
Write-Host " ... waiting for resource group deletion (${waitedSeconds}s elapsed): $($maskedNames -join ', ')"
115+
Start-Sleep -Seconds $IntervalSeconds
116+
$waitedSeconds += $IntervalSeconds
72117
}
73118

74-
Write-Host " ... waiting for resource group deletion (${waited}s elapsed)"
75-
Start-Sleep -Seconds $interval
76-
$waited += $interval
119+
$maskedGroups = $Groups | ForEach-Object { Protect-ResourceGroupName -Value $_.ResourceGroup }
120+
throw "Timed out waiting for resource group deletion for: $($maskedGroups -join ', ')."
77121
}
78122

123+
function Wait-ForDeletedApimService {
124+
param(
125+
[Parameter(Mandatory)]
126+
[string]$ServiceName,
127+
[Parameter(Mandatory)]
128+
[string]$ServiceLocation,
129+
[int]$TimeoutMinutes = 30,
130+
[int]$IntervalSeconds = 15
131+
)
132+
133+
$waitedSeconds = 0
134+
$timeoutSeconds = $TimeoutMinutes * 60
135+
136+
while ($waitedSeconds -lt $timeoutSeconds) {
137+
az apim deletedservice show --service-name $ServiceName --location $ServiceLocation -o none 2>$null
138+
if ($LASTEXITCODE -eq 0) {
139+
return
140+
}
141+
142+
Write-Host " ... waiting for APIM soft-delete entry ($(Protect-ApimName -Value $ServiceName)) (${waitedSeconds}s elapsed)"
143+
Start-Sleep -Seconds $IntervalSeconds
144+
$waitedSeconds += $IntervalSeconds
145+
}
146+
147+
throw "Timed out waiting for APIM soft-delete entry for '$(Protect-ApimName -Value $ServiceName)' in location '$ServiceLocation'."
148+
}
149+
150+
Write-Host " Deleting $(Protect-ResourceGroupName -Value $SourceResourceGroup)..."
151+
if (Get-GroupExists -ResourceGroup $SourceResourceGroup -SubscriptionId $SourceSubscriptionId) {
152+
$sourceDeleteArgs = @('group', 'delete', '--name', $SourceResourceGroup, '--yes', '--no-wait') + (Get-SubscriptionArgs -SubscriptionId $SourceSubscriptionId)
153+
az @sourceDeleteArgs 2>$null
154+
if ($LASTEXITCODE -ne 0) {
155+
throw "Failed to start deletion for source resource group '$(Protect-ResourceGroupName -Value $SourceResourceGroup)'."
156+
}
157+
}
158+
else {
159+
Write-Host " Source resource group already absent"
160+
}
161+
162+
Write-Host " Deleting $(Protect-ResourceGroupName -Value $TargetResourceGroup)..."
163+
if (Get-GroupExists -ResourceGroup $TargetResourceGroup -SubscriptionId $TargetSubscriptionId) {
164+
$targetDeleteArgs = @('group', 'delete', '--name', $TargetResourceGroup, '--yes', '--no-wait') + (Get-SubscriptionArgs -SubscriptionId $TargetSubscriptionId)
165+
az @targetDeleteArgs 2>$null
166+
if ($LASTEXITCODE -ne 0) {
167+
throw "Failed to start deletion for target resource group '$(Protect-ResourceGroupName -Value $TargetResourceGroup)'."
168+
}
169+
}
170+
else {
171+
Write-Host " Target resource group already absent"
172+
}
173+
174+
Write-Host " ⏳ Waiting for resource group deletions to complete for hard-delete..."
175+
$groups = @(
176+
@{ ResourceGroup = $SourceResourceGroup; SubscriptionId = $SourceSubscriptionId },
177+
@{ ResourceGroup = $TargetResourceGroup; SubscriptionId = $TargetSubscriptionId }
178+
)
179+
Wait-ForResourceGroupsDeletion -Groups $groups
180+
79181
if ($sourceApimName) {
182+
Wait-ForDeletedApimService -ServiceName $sourceApimName -ServiceLocation $Location
80183
Write-Host " 🗑️ Purging soft-deleted APIM: $(Protect-ApimName -Value $sourceApimName)..."
81184
az apim deletedservice purge --service-name $sourceApimName --location $Location 2>$null
82185
}
83186

84187
if ($targetApimName) {
188+
Wait-ForDeletedApimService -ServiceName $targetApimName -ServiceLocation $Location
85189
Write-Host " 🗑️ Purging soft-deleted APIM: $(Protect-ApimName -Value $targetApimName)..."
86190
az apim deletedservice purge --service-name $targetApimName --location $Location 2>$null
87191
}

‎tests/integration/all-resource-types/run-roundtrip-test.ps1‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -257,11 +257,16 @@ catch {
257257
}
258258
finally {
259259
# Phase 7: Teardown apim instances and supporting resources
260-
& $phase7TeardownScript `
261-
-SourceResourceGroup $SourceResourceGroup `
262-
-TargetResourceGroup $TargetResourceGroup `
263-
-Location $Location `
264-
-SkipTeardown:$SkipTeardown
260+
$phase7Args = @{
261+
SourceResourceGroup = $SourceResourceGroup
262+
TargetResourceGroup = $TargetResourceGroup
263+
Location = $Location
264+
SkipTeardown = $SkipTeardown
265+
}
266+
Add-ArgumentIfSet -Hashtable $phase7Args -Key 'SourceSubscriptionId' -Value $SourceSubscriptionId
267+
Add-ArgumentIfSet -Hashtable $phase7Args -Key 'TargetSubscriptionId' -Value $TargetSubscriptionId
268+
269+
& $phase7TeardownScript @phase7Args
265270
}
266271

267272
exit $exitCode

0 commit comments

Comments
 (0)