From 7b4a05573402ba988b58891cb5f0c4c2b3526d2a Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Tue, 5 Nov 2024 16:10:53 -0500 Subject: [PATCH 1/5] Retry container deletion --- eng/common/scripts/Helpers/Resource-Helpers.ps1 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/eng/common/scripts/Helpers/Resource-Helpers.ps1 b/eng/common/scripts/Helpers/Resource-Helpers.ps1 index ce8e40d15f..f5e9d29b4d 100644 --- a/eng/common/scripts/Helpers/Resource-Helpers.ps1 +++ b/eng/common/scripts/Helpers/Resource-Helpers.ps1 @@ -345,7 +345,9 @@ function RemoveStorageAccount($Account) { if ($container.BlobContainerProperties.HasImmutableStorageWithVersioning) { try { # Use AzRm cmdlet as deletion will only work through ARM with the immutability policies defined on the blobs - Remove-AzRmStorageContainer -Name $container.Name -StorageAccountName $Account.StorageAccountName -ResourceGroupName $Account.ResourceGroupName -Force + # If blob deletion takes longer then the container delete + # will fail, so retry a couple times + Retry -Action { Remove-AzRmStorageContainer -Name $container.Name -StorageAccountName $Account.StorageAccountName -ResourceGroupName $Account.ResourceGroupName -Force } -Attempts 2 #$container | Remove-AzStorageContainer } catch { Write-Host "Container removal failed: $($container.Name), account: $($Account.storageAccountName), group: $($Account.ResourceGroupName)" From a51637572e20b7790c8bb558bdb6ea790e86a932 Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Tue, 5 Nov 2024 12:37:54 -0500 Subject: [PATCH 2/5] Do not try to purge keyvaults with purge protection --- eng/common/scripts/Helpers/Resource-Helpers.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/common/scripts/Helpers/Resource-Helpers.ps1 b/eng/common/scripts/Helpers/Resource-Helpers.ps1 index f5e9d29b4d..de843a559f 100644 --- a/eng/common/scripts/Helpers/Resource-Helpers.ps1 +++ b/eng/common/scripts/Helpers/Resource-Helpers.ps1 @@ -123,8 +123,8 @@ filter Remove-PurgeableResources { switch ($r.AzsdkResourceType) { 'Key Vault' { if ($r.EnablePurgeProtection) { - # We will try anyway but will ignore errors. Write-Warning "Key Vault '$($r.VaultName)' has purge protection enabled and may not be purged until $($r.ScheduledPurgeDate)" + continue } # Use `-AsJob` to start a lightweight, cancellable job and pass to `Wait-PurgeableResoruceJob` for consistent behavior. @@ -134,8 +134,8 @@ filter Remove-PurgeableResources { 'Managed HSM' { if ($r.EnablePurgeProtection) { - # We will try anyway but will ignore errors. Write-Warning "Managed HSM '$($r.Name)' has purge protection enabled and may not be purged until $($r.ScheduledPurgeDate)" + continue } # Use `GetNewClosure()` on the `-Action` ScriptBlock to make sure variables are captured. From 690b39c74fb1e310a4c7c500a90c02b7549b7417 Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Tue, 5 Nov 2024 16:43:51 -0500 Subject: [PATCH 3/5] Delete all blobs when container has immutability --- eng/common/scripts/Helpers/Resource-Helpers.ps1 | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/eng/common/scripts/Helpers/Resource-Helpers.ps1 b/eng/common/scripts/Helpers/Resource-Helpers.ps1 index de843a559f..5c30d3b9a7 100644 --- a/eng/common/scripts/Helpers/Resource-Helpers.ps1 +++ b/eng/common/scripts/Helpers/Resource-Helpers.ps1 @@ -345,10 +345,9 @@ function RemoveStorageAccount($Account) { if ($container.BlobContainerProperties.HasImmutableStorageWithVersioning) { try { # Use AzRm cmdlet as deletion will only work through ARM with the immutability policies defined on the blobs - # If blob deletion takes longer then the container delete - # will fail, so retry a couple times - Retry -Action { Remove-AzRmStorageContainer -Name $container.Name -StorageAccountName $Account.StorageAccountName -ResourceGroupName $Account.ResourceGroupName -Force } -Attempts 2 - #$container | Remove-AzStorageContainer + # Add a retry in case blob deletion has not finished in time for container deletion, but not too many that we end up + # getting throttled by ARM/SRP if things are actually in a stuck state + Retry -Attempts 1 -Action { Remove-AzRmStorageContainer -Name $container.Name -StorageAccountName $Account.StorageAccountName -ResourceGroupName $Account.ResourceGroupName -Force } } catch { Write-Host "Container removal failed: $($container.Name), account: $($Account.storageAccountName), group: $($Account.ResourceGroupName)" Write-Warning "Ignoring the error and trying to delete the storage account" @@ -362,7 +361,7 @@ function RemoveStorageAccount($Account) { } } -function EnableBlobDeletion($Blob, $StorageAccountName, $ResourceGroupName) { +function EnableBlobDeletion($Blob, $Container, $StorageAccountName, $ResourceGroupName) { # Some properties like immutability policies require the blob to be # deleted before the container can be deleted $forceBlobDeletion = $false @@ -396,6 +395,10 @@ function EnableBlobDeletion($Blob, $StorageAccountName, $ResourceGroupName) { $Blob.ICloudBlob.BreakLease() } + if ($container.BlobContainerProperties.HasImmutableStorageWithVersioning) { + $forceBlobDeletion = $true + } + return $forceBlobDeletion } From 8a7f40d3dda63eb04c3fa701582f905145c7092e Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Tue, 5 Nov 2024 18:10:01 -0500 Subject: [PATCH 4/5] Skip missing blob container properties --- eng/common/scripts/Helpers/Resource-Helpers.ps1 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/eng/common/scripts/Helpers/Resource-Helpers.ps1 b/eng/common/scripts/Helpers/Resource-Helpers.ps1 index 5c30d3b9a7..42f5b1bc11 100644 --- a/eng/common/scripts/Helpers/Resource-Helpers.ps1 +++ b/eng/common/scripts/Helpers/Resource-Helpers.ps1 @@ -342,6 +342,9 @@ function RemoveStorageAccount($Account) { } foreach ($container in $containers) { + if (!($container | Get-Member 'BlobContainerProperties')) { + continue + } if ($container.BlobContainerProperties.HasImmutableStorageWithVersioning) { try { # Use AzRm cmdlet as deletion will only work through ARM with the immutability policies defined on the blobs @@ -395,7 +398,7 @@ function EnableBlobDeletion($Blob, $Container, $StorageAccountName, $ResourceGro $Blob.ICloudBlob.BreakLease() } - if ($container.BlobContainerProperties.HasImmutableStorageWithVersioning) { + if (($container | Get-Member 'BlobContainerProperties') -and $container.BlobContainerProperties.HasImmutableStorageWithVersioning) { $forceBlobDeletion = $true } From 8003c5bffc3501db61c8beaaa09d7b2ff6c9a2a1 Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Wed, 6 Nov 2024 16:18:40 -0500 Subject: [PATCH 5/5] Fix null container --- eng/common/scripts/Helpers/Resource-Helpers.ps1 | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/eng/common/scripts/Helpers/Resource-Helpers.ps1 b/eng/common/scripts/Helpers/Resource-Helpers.ps1 index 42f5b1bc11..b152a3f218 100644 --- a/eng/common/scripts/Helpers/Resource-Helpers.ps1 +++ b/eng/common/scripts/Helpers/Resource-Helpers.ps1 @@ -313,14 +313,16 @@ function RemoveStorageAccount($Account) { if ($Account.Kind -eq "FileStorage") { return } $containers = New-AzStorageContext -StorageAccountName $Account.StorageAccountName | Get-AzStorageContainer - $blobs = $containers | Get-AzStorageBlob $deleteNow = @() try { - foreach ($blob in $blobs) { - $shouldDelete = EnableBlobDeletion -Blob $blob -StorageAccountName $Account.StorageAccountName -ResourceGroupName $Account.ResourceGroupName - if ($shouldDelete) { - $deleteNow += $blob + foreach ($container in $containers) { + $blobs = $container | Get-AzStorageBlob + foreach ($blob in $blobs) { + $shouldDelete = EnableBlobDeletion -Blob $blob -Container $container -StorageAccountName $Account.StorageAccountName -ResourceGroupName $Account.ResourceGroupName + if ($shouldDelete) { + $deleteNow += $blob + } } } } catch { @@ -398,7 +400,7 @@ function EnableBlobDeletion($Blob, $Container, $StorageAccountName, $ResourceGro $Blob.ICloudBlob.BreakLease() } - if (($container | Get-Member 'BlobContainerProperties') -and $container.BlobContainerProperties.HasImmutableStorageWithVersioning) { + if (($Container | Get-Member 'BlobContainerProperties') -and $Container.BlobContainerProperties.HasImmutableStorageWithVersioning) { $forceBlobDeletion = $true }