diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 69bebdfe..792ceb26 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -1,3 +1,5 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. # =========================================================================== # Integration Test β€” Extractβ†’Publish Round-Trip # =========================================================================== @@ -31,6 +33,7 @@ on: required: true type: choice options: + - Standard - StandardV2 - Premium - PremiumV2 @@ -72,6 +75,8 @@ on: required: true APIM_PUBLISHER_EMAIL: required: true + APIM_SKU: + required: false permissions: id-token: write @@ -84,6 +89,8 @@ concurrency: env: SOURCE_RG: rg-apiops-bvt-source-${{ github.run_id }} TARGET_RG: rg-apiops-bvt-target-${{ github.run_id }} + SOURCE_APIM: apiops-bvt-src-${{ github.run_id }} + TARGET_APIM: apiops-bvt-tgt-${{ github.run_id }} jobs: roundtrip-test: @@ -103,14 +110,8 @@ jobs: - run: npm ci && npm run build - - name: Azure Login (OIDC) - uses: azure/login@v2 - with: - client-id: ${{ secrets.AZURE_CLIENT_ID }} - tenant-id: ${{ secrets.AZURE_TENANT_ID }} - subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - - - name: Run Round-Trip Test + - name: Resolve Workflow Settings + id: settings shell: pwsh run: | $logLevel = '${{ inputs.log_level }}' @@ -119,61 +120,140 @@ jobs: throw "Invalid log_level '$logLevel'. Allowed values: Info, Verbose, Debug." } + $skuName = '${{ secrets.APIM_SKU }}' + if ([string]::IsNullOrWhiteSpace($skuName)) { $skuName = '${{ inputs.sku }}' } + if ([string]::IsNullOrWhiteSpace($skuName)) { $skuName = 'StandardV2' } + + "logLevel=$logLevel" | Out-File -FilePath $env:GITHUB_OUTPUT -Append + "skuName=$skuName" | Out-File -FilePath $env:GITHUB_OUTPUT -Append + + - name: Azure Login + uses: azure/login@v2 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + + - name: Run Round-Trip Phase 1 (Deploy) + id: phase1 + shell: pwsh + run: | $params = @{ SourceResourceGroup = '${{ env.SOURCE_RG }}' TargetResourceGroup = '${{ env.TARGET_RG }}' - SkuName = '${{ inputs.sku }}' + SourceApimName = '${{ env.SOURCE_APIM }}' + TargetApimName = '${{ env.TARGET_APIM }}' + SourceSubscriptionId = '${{ secrets.AZURE_SUBSCRIPTION_ID }}' + TargetSubscriptionId = '${{ secrets.AZURE_SUBSCRIPTION_ID }}' + SkuName = '${{ steps.settings.outputs.skuName }}' Location = '${{ inputs.location }}' - LogLevel = $logLevel + LogLevel = '${{ steps.settings.outputs.logLevel }}' PublisherEmail = '${{ secrets.APIM_PUBLISHER_EMAIL }}' - ExtractOutputDir = './extracted-artifacts' - HardDelete = $true } + ./tests/integration/all-resource-types/phases/run-phase1-deploy.ps1 @params - ./tests/integration/all-resource-types/run-roundtrip-test.ps1 @params + - name: Azure Login - Refresh Before Phase 2 + if: success() + uses: azure/login@v2 + with: + # Phase 1 can run for a long time while APIM activates; refresh login + # before extract/publish so apiops receives a fresh federated session. + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + + - name: Run Round-Trip Phase 2 (Extract) + id: phase2 + if: success() + shell: pwsh + env: + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + run: | + ./tests/integration/all-resource-types/phases/run-phase2-extract.ps1 ` + -SourceSubscriptionId '${{ steps.phase1.outputs.sourceSubscriptionId }}' ` + -SourceResourceGroup '${{ steps.phase1.outputs.sourceResourceGroup }}' ` + -SourceApimName '${{ steps.phase1.outputs.sourceApimName }}' ` + -LogLevel '${{ steps.settings.outputs.logLevel }}' ` + -ExtractOutputDir './extracted-artifacts' + + - name: Run Round-Trip Phase 3 (Validate Extract) + if: success() + shell: pwsh + run: | + $skuName = '${{ steps.phase1.outputs.skuName }}' + $extractOutputDir = '${{ steps.phase2.outputs.ExtractOutputDir }}' + + ./tests/integration/all-resource-types/phases/run-phase3-validate-extract.ps1 ` + -SkuName $skuName ` + -LogLevel '${{ steps.settings.outputs.logLevel }}' ` + -ExtractOutputDir $extractOutputDir + + - name: Run Round-Trip Phase 4 (Create Overrides) + id: phase4 + if: success() + shell: pwsh + run: | + $extractOutputDir = '${{ steps.phase2.outputs.ExtractOutputDir }}' + + ./tests/integration/all-resource-types/phases/run-phase4-create-overrides.ps1 ` + -TargetSubscriptionId '${{ steps.phase1.outputs.targetSubscriptionId }}' ` + -TargetResourceGroup '${{ steps.phase1.outputs.targetResourceGroup }}' ` + -LogLevel '${{ steps.settings.outputs.logLevel }}' ` + -ExtractOutputDir $extractOutputDir + + - name: Run Round-Trip Phase 5 (Publish) + if: success() + shell: pwsh + env: + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + run: | + $extractOutputDir = '${{ steps.phase2.outputs.ExtractOutputDir }}' + + ./tests/integration/all-resource-types/phases/run-phase5-publish.ps1 ` + -TargetSubscriptionId '${{ steps.phase1.outputs.targetSubscriptionId }}' ` + -TargetResourceGroup '${{ steps.phase1.outputs.targetResourceGroup }}' ` + -TargetApimName '${{ steps.phase1.outputs.targetApimName }}' ` + -OverrideFile '${{ steps.phase4.outputs.overrideFile }}' ` + -LogLevel '${{ steps.settings.outputs.logLevel }}' ` + -ExtractOutputDir $extractOutputDir + + - name: Run Round-Trip Phase 6 (Compare) + if: success() + shell: pwsh + run: | + ./tests/integration/all-resource-types/phases/run-phase6-compare.ps1 ` + -SourceSubscriptionId '${{ steps.phase1.outputs.sourceSubscriptionId }}' ` + -SourceResourceGroup '${{ steps.phase1.outputs.sourceResourceGroup }}' ` + -SourceApimName '${{ steps.phase1.outputs.sourceApimName }}' ` + -TargetSubscriptionId '${{ steps.phase1.outputs.targetSubscriptionId }}' ` + -TargetResourceGroup '${{ steps.phase1.outputs.targetResourceGroup }}' ` + -TargetApimName '${{ steps.phase1.outputs.targetApimName }}' ` + -LogLevel '${{ steps.settings.outputs.logLevel }}' - name: Upload Extracted Artifacts if: always() uses: actions/upload-artifact@v4 with: - name: extracted-artifacts-${{ inputs.sku }} - path: ./extracted-artifacts/ + name: extracted-artifacts-${{ steps.phase1.outputs.skuName }} + path: ${{ steps.phase2.outputs.ExtractOutputDir || './extracted-artifacts/' }} if-no-files-found: ignore - - name: Emergency Teardown - if: failure() + - name: Run Round-Trip Phase 7 (Teardown) + if: always() shell: pwsh run: | - $sourceRg = '${{ env.SOURCE_RG }}' - $targetRg = '${{ env.TARGET_RG }}' - $location = '${{ inputs.location }}' - - # Capture APIM names before RG deletion so we can purge soft-deleted services. - $sourceApimName = az apim list --resource-group $sourceRg --query "[0].name" -o tsv 2>$null - $targetApimName = az apim list --resource-group $targetRg --query "[0].name" -o tsv 2>$null - - Write-Host "🚨 Emergency teardown β€” deleting resource groups..." - az group delete --name $sourceRg --yes --no-wait 2>$null - az group delete --name $targetRg --yes --no-wait 2>$null - - Write-Host "⏳ Waiting for resource group deletions before APIM purge..." - $maxWaitSeconds = 900 - $waited = 0 - $interval = 30 - - while ($waited -lt $maxWaitSeconds) { - $srcExists = (az group exists --name $sourceRg -o tsv 2>$null) -eq 'true' - $tgtExists = (az group exists --name $targetRg -o tsv 2>$null) -eq 'true' - if (-not $srcExists -and -not $tgtExists) { - break - } - Start-Sleep -Seconds $interval - $waited += $interval - } + $sourceResourceGroup = '${{ steps.phase1.outputs.sourceResourceGroup }}' + if ([string]::IsNullOrWhiteSpace($sourceResourceGroup)) { $sourceResourceGroup = '${{ env.SOURCE_RG }}' } - foreach ($apimName in @($sourceApimName, $targetApimName)) { - if (-not [string]::IsNullOrWhiteSpace($apimName)) { - Write-Host "πŸ—‘οΈ Purging soft-deleted APIM: $apimName" - az apim deletedservice purge --service-name $apimName --location $location 2>$null - } - } + $targetResourceGroup = '${{ steps.phase1.outputs.targetResourceGroup }}' + if ([string]::IsNullOrWhiteSpace($targetResourceGroup)) { $targetResourceGroup = '${{ env.TARGET_RG }}' } + + $location = '${{ steps.phase1.outputs.location }}' + if ([string]::IsNullOrWhiteSpace($location)) { $location = '${{ inputs.location }}' } + + ./tests/integration/all-resource-types/phases/run-phase7-teardown.ps1 ` + -SourceResourceGroup $sourceResourceGroup ` + -TargetResourceGroup $targetResourceGroup ` + -Location $location diff --git a/.gitignore b/.gitignore index 478d67ee..a7ca923d 100644 --- a/.gitignore +++ b/.gitignore @@ -39,11 +39,11 @@ Desktop.ini .local-extract*/ # Files for integration tests -tests/integration/all-resource-types/logs/** -tests/integration/all-resource-types/extracted-artifacts*/** -tests/integration/all-resource-types/target-apim.json -tests/integration/all-resource-types/source-apim.json -tests/integration/all-resource-types/source-apim-post-activation.bicep +tests/integration/all-resource-types/**/logs/** +tests/integration/all-resource-types/**/extracted-artifacts*/** +tests/integration/all-resource-types/bicep/target-apim.json +tests/integration/all-resource-types/bicep/source-apim.json +tests/integration/all-resource-types/bicep/source-apim-post-activation.bicep # Environment variables .env @@ -57,4 +57,3 @@ tests/integration/all-resource-types/source-apim-post-activation.bicep # Squad: SubSquad activation file (local to this machine) .squad-workstream *.d.ts.map -tests/integration/all-resource-types/source-apim-post-activation.json diff --git a/tests/contract/.gitkeep b/tests/contract/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/integration/all-resource-types/Compare-ApimInstance.ps1 b/tests/integration/all-resource-types/Compare-ApimInstance.ps1 index f0963e25..07ab1050 100644 --- a/tests/integration/all-resource-types/Compare-ApimInstance.ps1 +++ b/tests/integration/all-resource-types/Compare-ApimInstance.ps1 @@ -1,3 +1,5 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. <# .SYNOPSIS Compares two Azure API Management instances via ARM REST API. diff --git a/tests/integration/all-resource-types/DeploymentHelpers.psm1 b/tests/integration/all-resource-types/DeploymentHelpers.psm1 deleted file mode 100644 index b4ab9c0f..00000000 --- a/tests/integration/all-resource-types/DeploymentHelpers.psm1 +++ /dev/null @@ -1,65 +0,0 @@ -Import-Module (Join-Path $PSScriptRoot 'MaskingHelpers.psm1') - -function Write-DeploymentFailureDetails { - [CmdletBinding()] - param( - [Parameter(Mandatory)][string]$ResourceGroupName, - [Parameter(Mandatory)][string]$DeploymentName, - [hashtable]$Replacements = @{} - ) - - $maskedRg = Protect-ResourceGroupName -Value $ResourceGroupName - Write-Host "" - Write-Host "========== Deployment failure details ==========" -ForegroundColor Yellow - Write-Host "Resource group: $maskedRg" -ForegroundColor Yellow - Write-Host "Deployment: $DeploymentName" -ForegroundColor Yellow - Write-Host "Querying failed deployment operations (before teardown)..." -ForegroundColor Yellow - - $query = "[?properties.provisioningState=='Failed'].{resource:properties.targetResource.resourceName, type:properties.targetResource.resourceType, code:properties.statusMessage.error.code, message:properties.statusMessage.error.message, details:properties.statusMessage.error.details}" - - try { - Invoke-MaskedProcess -FilePath 'az' -Replacements $Replacements -Arguments @( - 'deployment', 'operation', 'group', 'list', - '--resource-group', $ResourceGroupName, - '--name', $DeploymentName, - '--query', $query, - '--output', 'json' - ) - } catch { - $maskedErr = Protect-LogLine -Line ($_.Exception.Message) -Replacements $Replacements - Write-Host "Failed to retrieve deployment operations: $maskedErr" -ForegroundColor Red - } - - Write-Host "================================================" -ForegroundColor Yellow -} - -function Wait-ApimActivation { - [CmdletBinding()] - param( - [Parameter(Mandatory)][string]$ResourceGroupName, - [Parameter(Mandatory)][string]$ApimName, - [int]$TimeoutSeconds = 1800, - [int]$PollIntervalSeconds = 20 - ) - - $maskedApim = Protect-ApimName -Value $ApimName - Write-Host "Waiting for APIM '$maskedApim' to finish Activating (timeout ${TimeoutSeconds}s)..." -ForegroundColor Cyan - - $deadline = (Get-Date).AddSeconds($TimeoutSeconds) - $lastState = $null - while ((Get-Date) -lt $deadline) { - $state = az apim show --resource-group $ResourceGroupName --name $ApimName --query provisioningState --output tsv 2>$null - if ($state -ne $lastState) { - Write-Host " provisioningState: $state" -ForegroundColor Gray - $lastState = $state - } - if ($state -eq 'Succeeded') { return $true } - if ($state -eq 'Failed') { throw "APIM '$maskedApim' entered Failed state during activation wait" } - Start-Sleep -Seconds $PollIntervalSeconds - } - throw "APIM '$maskedApim' did not reach Succeeded within ${TimeoutSeconds}s (last state: $lastState)" -} - -Export-ModuleMember -Function ` - Write-DeploymentFailureDetails, ` - Wait-ApimActivation diff --git a/tests/integration/all-resource-types/README.md b/tests/integration/all-resource-types/README.md index b2273b6d..2e089541 100644 --- a/tests/integration/all-resource-types/README.md +++ b/tests/integration/all-resource-types/README.md @@ -1,12 +1,43 @@ -# Kitchen Sink APIM β€” Build Verification Test Infrastructure +# All types APIM Test Assets -This directory contains Bicep templates and scripts to deploy a "kitchen sink" Azure API Management instance for end-to-end build verification testing of the APIOps CLI. +The goal of all-resource-types integration test is to cover all resource types in the [`ResourceType`](src/models/resource-types.ts) enum. -## What Gets Deployed +The round trip test first deploys a source API Management instance with all features described in the [what is deployed](#what-is-deployed) section. The test then extracts everything from the source instance and publishes the artifacts to a black target API management instance. See [round trip phases](#round-trip-phases) for more details. -The kitchen sink APIM instance includes **every resource type and API protocol variation** that APIOps-v2 supports, covering all 33 resource types in the `ResourceType` enum: +## Prerequisites + +- Azure CLI authenticated with `az login` +- Subscription permissions to create/delete APIM and supporting resources +- Bicep support via Azure CLI + +## Quick Commands + +### Run full round trip + +```powershell +cd tests/integration/all-resource-types +./run-roundtrip-test.ps1 -PublisherEmail admin@contoso.com +``` + +### Run full round trip with log: + +#### Bash +```bash +set -o pipefail && log_file="tests/integration/all-resource-types/phases/logs/roundtrip-premium-$(date +%Y%m%d-%H%M%S).log" && echo "Logging to $log_file" && pwsh -NoLogo -NoProfile -File ./tests/integration/all-resource-types/run-roundtrip-test.ps1 -SkuName Premium 2>&1 | tee "$log_file" +``` -### API Protocol Variations +#### Powershell +```powershell +$logFile = "tests/integration/all-resource-types/phases/logs/roundtrip-premium-$((Get-Date).ToString('yyyyMMdd-HHmmss')).log" +Write-Host "Logging to $logFile" +.\tests\integration\all-resource-types\run-roundtrip-test.ps1 -SkuName Premium 2>&1 | Tee-Object -FilePath $logFile +if ($LASTEXITCODE -ne 0) { throw "Round-trip failed with exit code $LASTEXITCODE. See $logFile" } +``` + +## What is Deployed? + +### APIs +An apim instance with the following apis | API | Type | Spec Format | |-----|------|-------------| | `src-rest-openapi` | REST | OpenAPI 3.0 YAML | @@ -46,94 +77,138 @@ The kitchen sink APIM instance includes **every resource type and API protocol v - **Subscriptions**: All-APIs + product-scoped - **Workspace** (Developer v2 only): Workspace with backend, named value, tag, product, API -## Prerequisites +## Round-Trip Phases -- [Azure CLI](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli) installed and authenticated (`az login`) -- An Azure subscription with permissions to create resources -- [Bicep](https://learn.microsoft.com/en-us/azure/azure-resource-manager/bicep/install) (bundled with recent Azure CLI) +**Phase 1: Deploy source + target** (`phases/run-phase1-deploy.ps1`). -## Run +Deploys source and target APIM environments in parallel. -### Deploy +```powershell +# Minimum parameters +./phases/run-phase1-deploy.ps1 -SourceResourceGroup rg-src -TargetResourceGroup rg-tgt -PublisherEmail admin@contoso.com + +# All parameters +./phases/run-phase1-deploy.ps1 -SourceResourceGroup rg-src -TargetResourceGroup rg-tgt -PublisherEmail admin@contoso.com -SkuName StandardV2 -Location eastus2 -LogLevel Verbose -SourceApimName src-apim -TargetApimName tgt-apim -SourceSubscriptionId 11111111-1111-1111-1111-111111111111 -TargetSubscriptionId 22222222-2222-2222-2222-222222222222 +``` + +Script returns resolved names, which can be for later phases, especially in the case minimal parameters are passed to the script. Example return value: ```powershell -# Deploy with defaults (StandardV2 SKU, centralus, auto-generated resource names) -.\deploy-source.ps1 -ResourceGroupName rg-apiops-bvt -PublisherEmail admin@contoso.com +@{ + sourceSubscriptionId = "11111111-1111-1111-1111-111111111111" + sourceResourceGroup = "rg-src" + sourceApimName = "src-apim" + targetSubscriptionId = "22222222-2222-2222-2222-222222222222" + targetResourceGroup = "rg-tgt" + targetApimName = "tgt-apim" + skuName = "StandardV2" + location = "eastus2" +} +``` + +**Phase 2: Extract** (`phases/run-phase2-extract.ps1`). -# Deploy with Developer SKU (classic β€” supports self-hosted gateways, no workspaces) -.\deploy-source.ps1 -ResourceGroupName rg-apiops-bvt -PublisherEmail admin@contoso.com -SkuName Developer +Runs `apiops extract` against the source APIM instance and writes artifacts to the extract directory. + +```powershell +# Minimum parameters +./phases/run-phase2-extract.ps1 -SourceResourceGroup rg-src -SourceApimName src-apim -# Deploy with Premium SKU (classic β€” supports everything including workspaces + gateways) -.\deploy-source.ps1 -ResourceGroupName rg-apiops-bvt -PublisherEmail admin@contoso.com -SkuName Premium +# All parameters +./phases/run-phase2-extract.ps1 -SourceSubscriptionId 11111111-1111-1111-1111-111111111111 -SourceResourceGroup rg-src -SourceApimName src-apim -LogLevel Debug -ExtractOutputDir ./phases/extracted-artifacts ``` -> ⏱️ **APIM provisioning takes 30-45 minutes.** The script will wait for completion. +Script returns the path to the extracted files. Example return value: -### Run APIOps Extract Against It +```powershell +/workspaces/apiops-cli/tests/integration/all-resource-types/phases/extracted-artifacts +``` + +**Phase 3: Validate extract** (`phases/run-phase3-validate-extract.ps1`) -After deployment, the script outputs the exact CLI command: +Validates extracted artifacts against the expected manifest before publish. ```powershell -npx apiops extract \ - --subscription-id \ - --resource-group rg-apiops-bvt \ - --service-name src-apim-bvt \ - --output-dir ./extracted +# Minimum parameters +./phases/run-phase3-validate-extract.ps1 + +# All parameters +./phases/run-phase3-validate-extract.ps1 -SkuName PremiumV2 -LogLevel Verbose -ExtractOutputDir ./phases/extracted-artifacts ``` -### Destroy +**Phase 4: Create target overrides** (`phases/run-phase4-create-overrides.ps1`). + +Generates the target-specific `.overrides.yaml` file used by the publish phase. Script returns the value of the created configuration overrides file. ```powershell -.\deploy-source.ps1 -ResourceGroupName rg-apiops-bvt -Destroy +# Minimum parameters +./phases/run-phase4-create-overrides.ps1 -TargetResourceGroup rg-tgt + +# All parameters +./phases/run-phase4-create-overrides.ps1 -TargetSubscriptionId 22222222-2222-2222-2222-222222222222 -TargetResourceGroup rg-tgt -LogLevel Info -ExtractOutputDir ./phases/extracted-artifacts ``` -## Round-Trip Integration Test +Script returns path to created configuration overrides file. Example return value: -The round-trip test validates the full extractβ†’publish cycle: +```powershell +/workspaces/apiops-cli/tests/integration/all-resource-types/phases/extracted-artifacts/.overrides.yaml +``` -1. **Deploy** an all-resources source APIM (all 33 resource types) -2. **Deploy** a blank target APIM (same SKU, supporting infra only) -3. **Extract** from source using `apiops extract` -4. **Publish** extracted artifacts to target using `apiops publish` -5. **Compare** source vs target via ARM REST API (deep property comparison with normalization) -6. **Teardown** both resource groups +**Phase 5: Publish** (`phases/run-phase5-publish.ps1`). -### Run via GitHub Actions +Publishes extracted artifacts to the target APIM instance using the generated overrides file. -The workflow at `.github/workflows/integration-test.yml` provides a manual trigger (`workflow_dispatch`) with: -- **SKU selection** (StandardV2, Developer, Premium, PremiumV2) -- **Location** (default: centralus) -- **Log level** (Info, Verbose, Debug; default: Verbose) -- **Skip teardown** toggle for debugging +```powershell +# Minimum parameters +./phases/run-phase5-publish.ps1 -TargetResourceGroup rg-tgt -TargetApimName tgt-apim -OverrideFile ./phases/extracted-artifacts/.overrides.yaml -Requires an `integration-test` environment with secrets: -- `AZURE_CLIENT_ID`, `AZURE_TENANT_ID`, `AZURE_SUBSCRIPTION_ID` (OIDC) -- `APIM_PUBLISHER_EMAIL` +# All parameters +./phases/run-phase5-publish.ps1 -TargetSubscriptionId 22222222-2222-2222-2222-222222222222 -TargetResourceGroup rg-tgt -TargetApimName tgt-apim -LogLevel Debug -OverrideFile ./phases/extracted-artifacts/.overrides.yaml -ExtractOutputDir ./phases/extracted-artifacts +``` -### Comparison Script +**Phase 6: Compare source and target API Management instances** (`phases/run-phase6-compare.ps1`). -`compare-apim-instances.ps1` can also be run standalone to diff any two APIM instances: +Compares source and target APIM resources and reports differences or parity. ```powershell -.\compare-apim-instances.ps1 ` - -SourceSubscriptionId "..." -SourceResourceGroup rg-src -SourceApimName apim-src ` - -TargetSubscriptionId "..." -TargetResourceGroup rg-tgt -TargetApimName apim-tgt +# Minimum parameters +./phases/run-phase6-compare.ps1 -SourceResourceGroup rg-src -SourceApimName src-apim -TargetResourceGroup rg-tgt -TargetApimName tgt-apim + +# All parameters +./phases/run-phase6-compare.ps1 -SourceSubscriptionId 11111111-1111-1111-1111-111111111111 -SourceResourceGroup rg-src -SourceApimName src-apim -TargetSubscriptionId 22222222-2222-2222-2222-222222222222 -TargetResourceGroup rg-tgt -TargetApimName tgt-apim -LogLevel Verbose ``` -Exit codes: `0` = match, `1` = differences found, `2` = error. +**Phase 7: Teardown** (`phases/run-phase7-teardown.ps1`). + +Deletes source and target resource groups and purges soft-deleted APIM services. This phase always run, regardless of the success of previous phases, unles `-SkipTeardown` switch is specified. + +```powershell +# Minimum parameters +./phases/run-phase7-teardown.ps1 -SourceResourceGroup rg-src -TargetResourceGroup rg-tgt + +# All parameters +./phases/run-phase7-teardown.ps1 -SourceResourceGroup rg-src -TargetResourceGroup rg-tgt -Location eastus2 -SkipTeardown +``` + +## CI + +Workflow: `.github/workflows/integration-test.yml` (`workflow_dispatch`) + +Required environment secrets: + +- `AZURE_CLIENT_ID` +- `AZURE_TENANT_ID` +- `AZURE_SUBSCRIPTION_ID` +- `APIM_PUBLISHER_EMAIL` -## Files +## File Layout -| File | Purpose | -|------|---------| -| `source-apim.bicep` | Source APIM with all 33 resource types | -| `target-apim.bicep` | Blank target APIM + supporting infra | -| `deploy-source.ps1` | Deploy/destroy the source instance | -| `run-roundtrip-test.ps1` | Master orchestrator for the full test | -| `compare-apim-instances.ps1` | ARM REST comparison script | -| `validate-extracted-artifacts.ps1` | Validate extracted artifact structure | -| `expected-structure.json` | Manifest of expected extracted files | +- `bicep/` source and target templates +- `modules/` shared PowerShell helpers +- `phases/` phase scripts +- `run-roundtrip-test.ps1` full orchestrator -## Cost +## Notes -This deployment incurs costs for APIM and supporting resources (App Insights, Event Hub, Key Vault). Deploy on demand and destroy after testing. See [Azure API Management pricing](https://azure.microsoft.com/pricing/details/api-management/) for details. +- APIM provisioning takes time (typically 30-45 minutes). +- Exit codes used by compare/validation phases: `0` success, `1` diff/validation failure, `2` execution error. diff --git a/tests/integration/all-resource-types/Test-ExtractedArtifact.ps1 b/tests/integration/all-resource-types/Test-ExtractedArtifact.ps1 index 50122309..7d4e9cd8 100644 --- a/tests/integration/all-resource-types/Test-ExtractedArtifact.ps1 +++ b/tests/integration/all-resource-types/Test-ExtractedArtifact.ps1 @@ -1,3 +1,5 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. <# .SYNOPSIS Validates extracted APIM artifacts against expected structure manifest. @@ -19,7 +21,7 @@ Path to expected-structure.json manifest file. .PARAMETER SkuName - APIM SKU name (StandardV2, Developer, Premium, PremiumV2) to handle SKU-variant + APIM SKU name (StandardV2, Developer, Premium, Standard, PremiumV2) to handle SKU-variant resources. Default: StandardV2. .EXAMPLE @@ -237,7 +239,7 @@ function Invoke-DirectoryValidation([string]$basePath, [psobject]$dirSpec, [stri # SKU filter check if (Test-SkuFilter $dirSpec $SkuName) { - Write-Host " ⏭️ Skipping $label (SKU: requires $($dirSpec.skuFilter -join ', '))" + Write-Host " ⏭️ Skipping $label (SKU: supported in $($dirSpec.skuFilter -join ', '))" return } @@ -355,7 +357,7 @@ Write-Host "" if ($manifest.workspaces) { $wsSpec = $manifest.workspaces if (Test-SkuFilter $wsSpec $SkuName) { - Write-Host " ⏭️ Skipping workspaces (SKU: requires $($wsSpec.skuFilter -join ', '))" + Write-Host " ⏭️ Skipping workspaces (SKU: supported in $($wsSpec.skuFilter -join ', '))" } elseif ($wsSpec.expected) { foreach ($ws in $wsSpec.expected) { diff --git a/tests/integration/all-resource-types/source-apim-post-activation.bicep b/tests/integration/all-resource-types/bicep/source-apim-post-activation.bicep similarity index 100% rename from tests/integration/all-resource-types/source-apim-post-activation.bicep rename to tests/integration/all-resource-types/bicep/source-apim-post-activation.bicep diff --git a/tests/integration/all-resource-types/source-apim.bicep b/tests/integration/all-resource-types/bicep/source-apim.bicep similarity index 98% rename from tests/integration/all-resource-types/source-apim.bicep rename to tests/integration/all-resource-types/bicep/source-apim.bicep index 0d33ab9f..24ca0902 100644 --- a/tests/integration/all-resource-types/source-apim.bicep +++ b/tests/integration/all-resource-types/bicep/source-apim.bicep @@ -26,8 +26,8 @@ param publisherEmail string @description('Publisher name shown in the developer portal.') param publisherName string = 'APIOps BVT' -@description('APIM SKU name. Use StandardV2/PremiumV2 for v2 tiers, or Developer/Premium for classic.') -@allowed(['Developer', 'Premium', 'StandardV2', 'PremiumV2']) +@description('APIM SKU name. Use StandardV2/PremiumV2 for v2 tiers, or Developer/Premium/Standard for classic.') +@allowed(['Developer', 'Premium', 'Standard', 'StandardV2', 'PremiumV2']) param skuName string = 'StandardV2' @description('Application Insights name for logger/diagnostic testing.') @@ -46,9 +46,9 @@ param logAnalyticsName string = 'bvt-${uniqueString(resourceGroup().id)}-src-law // Variables // --------------------------------------------------------------------------- -var isClassicSku = skuName == 'Developer' || skuName == 'Premium' +var isClassicSku = skuName == 'Developer' || skuName == 'Premium' || skuName == 'Standard' var apimSkuCapacity = isClassicSku ? 1 : 1 -var supportsSelfHostedGateway = isClassicSku +var supportsSelfHostedGateway = skuName == 'Developer' || skuName == 'Premium' var supportsWorkspaces = skuName == 'Premium' || skuName == 'PremiumV2' // Minimal but valid OpenAPI 3.0 spec @@ -636,7 +636,7 @@ resource globalSchema 'Microsoft.ApiManagement/service/schemas@2025-09-01-previe } } -// Activation-sensitive resources are deployed post-activation by Deploy-SourceApim.ps1. +// Activation-sensitive resources are deployed post-activation by run-phase1-deploy-source.ps1. // --------------------------------------------------------------------------- // TIER 2: Resources with Dependencies @@ -657,7 +657,7 @@ resource diagnostic 'Microsoft.ApiManagement/service/diagnostics@2025-09-01-prev } } -// Service policy is deployed post-activation by Deploy-SourceApim.ps1. +// Service policy is deployed post-activation by run-phase1-deploy-source.ps1. // --- Products --- resource productStarter 'Microsoft.ApiManagement/service/products@2025-09-01-preview' = { @@ -1164,7 +1164,7 @@ resource apiA2a 'Microsoft.ApiManagement/service/apis@2025-09-01-preview' = { // TIER 3: Child Resources // --------------------------------------------------------------------------- -// Product policy is deployed post-activation by Deploy-SourceApim.ps1. +// Product policy is deployed post-activation by run-phase1-deploy-source.ps1. // --- Product API Associations --- resource productStarterApiRest 'Microsoft.ApiManagement/service/products/apis@2025-09-01-preview' = { @@ -1210,9 +1210,9 @@ resource productStarterTag 'Microsoft.ApiManagement/service/products/tags@2025-0 dependsOn: [tagEnv] } -// Product wiki is deployed post-activation by Deploy-SourceApim.ps1. +// Product wiki is deployed post-activation by run-phase1-deploy-source.ps1. -// API policy is deployed post-activation by Deploy-SourceApim.ps1. +// API policy is deployed post-activation by run-phase1-deploy-source.ps1. // --- API Tags --- resource apiRestTagEnv 'Microsoft.ApiManagement/service/apis/tags@2025-09-01-preview' = { @@ -1288,7 +1288,7 @@ resource apiRestTagDescEnv 'Microsoft.ApiManagement/service/apis/tagDescriptions } } -// API wiki is deployed post-activation by Deploy-SourceApim.ps1. +// API wiki is deployed post-activation by run-phase1-deploy-source.ps1. // --- API Release (on revisioned API) --- resource apiRelease 'Microsoft.ApiManagement/service/apis/releases@2025-09-01-preview' = { diff --git a/tests/integration/all-resource-types/target-apim.bicep b/tests/integration/all-resource-types/bicep/target-apim.bicep similarity index 98% rename from tests/integration/all-resource-types/target-apim.bicep rename to tests/integration/all-resource-types/bicep/target-apim.bicep index 97b5da57..3c1c25e6 100644 --- a/tests/integration/all-resource-types/target-apim.bicep +++ b/tests/integration/all-resource-types/bicep/target-apim.bicep @@ -30,7 +30,7 @@ param publisherEmail string param publisherName string = 'APIOps BVT' @description('APIM SKU name. Must match the source instance SKU.') -@allowed(['Developer', 'Premium', 'StandardV2', 'PremiumV2']) +@allowed(['Developer', 'Premium', 'Standard', 'StandardV2', 'PremiumV2']) param skuName string = 'StandardV2' @description('Application Insights name for logger/diagnostic testing.') @@ -49,7 +49,7 @@ param logAnalyticsName string = 'bvt-${uniqueString(resourceGroup().id)}-tgt-law // Variables // --------------------------------------------------------------------------- -var isClassicSku = skuName == 'Developer' || skuName == 'Premium' +var isClassicSku = skuName == 'Developer' || skuName == 'Premium' || skuName == 'Standard' var apimSkuCapacity = isClassicSku ? 1 : 1 // --------------------------------------------------------------------------- diff --git a/tests/integration/all-resource-types/modules/ApiopsCli.psm1 b/tests/integration/all-resource-types/modules/ApiopsCli.psm1 new file mode 100644 index 00000000..131f44c8 --- /dev/null +++ b/tests/integration/all-resource-types/modules/ApiopsCli.psm1 @@ -0,0 +1,49 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +<# +.SYNOPSIS +Maps script log levels to apiops CLI log levels. + +.PARAMETER ScriptLogLevel +Script log level (Info, Verbose, Debug). + +.OUTPUTS +System.String +#> +function Get-ApiopsLogLevel { + param( + [Parameter(Mandatory)] + [string]$ScriptLogLevel + ) + + switch ($ScriptLogLevel) { + 'Info' { return 'info' } + 'Verbose' { return 'warn' } + 'Debug' { return 'debug' } + default { return 'info' } + } +} + +<# +.SYNOPSIS +Builds optional auth args from OIDC environment variables. + +.OUTPUTS +System.String[] +#> +function Get-ApiopsAuthArgs { + $authArgs = @() + + if (-not [string]::IsNullOrWhiteSpace($env:AZURE_CLIENT_ID)) { + $authArgs += @('--client-id', $env:AZURE_CLIENT_ID) + } + + if (-not [string]::IsNullOrWhiteSpace($env:AZURE_TENANT_ID)) { + $authArgs += @('--tenant-id', $env:AZURE_TENANT_ID) + } + + return $authArgs +} + +Export-ModuleMember -Function Get-ApiopsLogLevel, Get-ApiopsAuthArgs \ No newline at end of file diff --git a/tests/integration/all-resource-types/modules/DeploymentOps.psm1 b/tests/integration/all-resource-types/modules/DeploymentOps.psm1 new file mode 100644 index 00000000..15b786eb --- /dev/null +++ b/tests/integration/all-resource-types/modules/DeploymentOps.psm1 @@ -0,0 +1,216 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. +Import-Module (Join-Path $PSScriptRoot 'LogMasking.psm1') + +<# +.SYNOPSIS +Prints masked details for failed ARM deployment operations. + +.PARAMETER ResourceGroupName +Resource group that contains the deployment. + +.PARAMETER DeploymentName +Deployment name to inspect. + +.PARAMETER Replacements +Mask replacement map for output. + +.OUTPUTS +None +#> +function Write-DeploymentFailureDetails { + [CmdletBinding()] + param( + [Parameter(Mandatory)][string]$ResourceGroupName, + [Parameter(Mandatory)][string]$DeploymentName, + [hashtable]$Replacements = @{} + ) + + $maskedRg = Protect-ResourceGroupName -Value $ResourceGroupName + Write-Host "" + Write-Host "========== Deployment failure details ==========" -ForegroundColor Yellow + Write-Host "Resource group: $maskedRg" -ForegroundColor Yellow + Write-Host "Deployment: $DeploymentName" -ForegroundColor Yellow + Write-Host "Querying failed deployment operations (before teardown)..." -ForegroundColor Yellow + + $query = "[?properties.provisioningState=='Failed'].{resource:properties.targetResource.resourceName, type:properties.targetResource.resourceType, code:properties.statusMessage.error.code, message:properties.statusMessage.error.message, details:properties.statusMessage.error.details}" + + try { + Invoke-MaskedProcess -FilePath 'az' -Replacements $Replacements -Arguments @( + 'deployment', 'operation', 'group', 'list', + '--resource-group', $ResourceGroupName, + '--name', $DeploymentName, + '--query', $query, + '--output', 'json' + ) + } catch { + $maskedErr = Protect-LogLine -Line ($_.Exception.Message) -Replacements $Replacements + Write-Host "Failed to retrieve deployment operations: $maskedErr" -ForegroundColor Red + } + + Write-Host "================================================" -ForegroundColor Yellow +} + +<# + .SYNOPSIS + Polls a condition until it succeeds or times out. + + .PARAMETER Probe + Scriptblock invoked on each poll. Return $true when the condition is met. + + .PARAMETER TimeoutSeconds + Maximum wait time in seconds. + + .PARAMETER PollIntervalSeconds + Polling interval in seconds. + + .PARAMETER TimeoutMessage + Error message thrown when the timeout is reached. + + .OUTPUTS + System.Boolean + #> + function Invoke-PolledWait { + [CmdletBinding()] + param( + [Parameter(Mandatory)][scriptblock]$Probe, + [int]$TimeoutSeconds = 300, + [int]$PollIntervalSeconds = 10, + [Parameter(Mandatory)][string]$TimeoutMessage + ) + + $deadline = (Get-Date).AddSeconds($TimeoutSeconds) + while ((Get-Date) -lt $deadline) { + if (& $Probe) { + return $true + } + + Start-Sleep -Seconds $PollIntervalSeconds + } + + throw $TimeoutMessage + } + + <# +.SYNOPSIS +Waits for an APIM instance to reach Succeeded provisioning. + +.PARAMETER ResourceGroupName +Resource group containing the APIM instance. + +.PARAMETER ApimName +APIM service name. + +.PARAMETER TimeoutSeconds +Maximum wait time in seconds. + +.PARAMETER PollIntervalSeconds +Polling interval in seconds. + +.OUTPUTS +System.Boolean +#> +function Wait-ApimActivation { + [CmdletBinding()] + param( + [Parameter(Mandatory)][string]$ResourceGroupName, + [Parameter(Mandatory)][string]$ApimName, + [int]$TimeoutSeconds = 1800, + [int]$PollIntervalSeconds = 20 + ) + + $maskedApim = Protect-ApimName -Value $ApimName + Write-Host "Waiting for APIM '$maskedApim' to finish Activating (timeout ${TimeoutSeconds}s)..." -ForegroundColor Cyan + + $activationState = @{ LastState = $null } + $probe = { + $state = az apim show --resource-group $ResourceGroupName --name $ApimName --query provisioningState --output tsv 2>$null + if ($state -ne $activationState.LastState) { + Write-Host " provisioningState: $state" -ForegroundColor Gray + $activationState.LastState = $state + } + if ($state -eq 'Succeeded') { return $true } + if ($state -eq 'Failed') { throw "APIM '$maskedApim' entered Failed state during activation wait" } + return $false + } + + Invoke-PolledWait ` + -Probe $probe ` + -TimeoutSeconds $TimeoutSeconds ` + -PollIntervalSeconds $PollIntervalSeconds ` + -TimeoutMessage "APIM '$maskedApim' did not reach Succeeded within ${TimeoutSeconds}s (last state: $($activationState.LastState))" +} + +<# +.SYNOPSIS +Waits for an APIM API to become queryable via az CLI. + +.PARAMETER ResourceGroupName +Resource group containing the APIM instance. + +.PARAMETER ApimServiceName +APIM service name. + +.PARAMETER ApiId +The API ID to query. + +.PARAMETER Replacements +Mask replacement map for output. + +.PARAMETER TimeoutSeconds +Maximum wait time in seconds. + +.PARAMETER PollIntervalSeconds +Polling interval in seconds. + +.OUTPUTS +System.Boolean +#> +function Wait-ApimApiQueryable { + [CmdletBinding()] + param( + [Parameter(Mandatory)][string]$ResourceGroupName, + [Parameter(Mandatory)][string]$ApimServiceName, + [Parameter(Mandatory)][string]$ApiId, + [hashtable]$Replacements = @{}, + [int]$TimeoutSeconds = 300, + [int]$PollIntervalSeconds = 10 + ) + + $maskedApim = Protect-ApimName -Value $ApimServiceName + Write-Host "Waiting for APIM API '$ApiId' to become queryable in '$maskedApim' (timeout ${TimeoutSeconds}s)..." -ForegroundColor Cyan + + $subscriptionId = az account show --query id --output tsv 2>$null + if ([string]::IsNullOrWhiteSpace($subscriptionId)) { + throw "Could not resolve active Azure subscription while waiting for API '$ApiId'" + } + + $apiListUri = "https://management.azure.com/subscriptions/$subscriptionId/resourceGroups/$ResourceGroupName/providers/Microsoft.ApiManagement/service/$ApimServiceName/apis?api-version=2025-09-01-preview" + + $probe = { + $probeArgs = @( + 'rest', + '--method', 'get', + '--uri', $apiListUri, + '--query', "length(value[?name=='$ApiId'])", + '--output', 'tsv' + ) + $probeResult = Invoke-MaskedAzCommand -Replacements $Replacements -Arguments $probeArgs + if ($LASTEXITCODE -eq 0 -and "$probeResult" -eq '1') { + return $true + } + + return $false + } + + Invoke-PolledWait ` + -Probe $probe ` + -TimeoutSeconds $TimeoutSeconds ` + -PollIntervalSeconds $PollIntervalSeconds ` + -TimeoutMessage "Timed out waiting for API '$ApiId' to be queryable in APIM '$maskedApim' within ${TimeoutSeconds}s" +} + +Export-ModuleMember -Function ` + Write-DeploymentFailureDetails, ` + Wait-ApimActivation, ` + Wait-ApimApiQueryable diff --git a/tests/integration/all-resource-types/MaskingHelpers.psm1 b/tests/integration/all-resource-types/modules/LogMasking.psm1 similarity index 69% rename from tests/integration/all-resource-types/MaskingHelpers.psm1 rename to tests/integration/all-resource-types/modules/LogMasking.psm1 index 064dbe16..485e6076 100644 --- a/tests/integration/all-resource-types/MaskingHelpers.psm1 +++ b/tests/integration/all-resource-types/modules/LogMasking.psm1 @@ -1,3 +1,5 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. # MaskingHelpers β€” secret-redaction utilities for the round-trip integration test scripts. $script:EnableMasking = $true @@ -28,6 +30,22 @@ $script:BuiltinRedactions = @( Replacement = '' } ) +<# +.SYNOPSIS +Masks generic identifiers while preserving short prefix/suffix. + +.PARAMETER Value +Input string to mask. + +.PARAMETER Prefix +Visible prefix length. + +.PARAMETER Suffix +Visible suffix length. + +.OUTPUTS +System.String +#> function Protect-Identifier { param( [string]$Value, @@ -50,22 +68,74 @@ function Protect-Identifier { return "{0}...{1}" -f $Value.Substring(0, $Prefix), $Value.Substring($Value.Length - $Suffix) } +<# +.SYNOPSIS +Replaces subscription IDs with a stable redaction token. + +.PARAMETER Value +Subscription ID value. + +.OUTPUTS +System.String +#> function Protect-SubscriptionId { param([string]$Value) if (-not $script:EnableMasking) { return $Value } return '' } +<# +.SYNOPSIS +Masks resource group names with minimal context retained. + +.PARAMETER Value +Resource group name. + +.OUTPUTS +System.String +#> function Protect-ResourceGroupName { param([string]$Value) + + if (-not $script:EnableMasking) { return $Value } + if ([string]::IsNullOrWhiteSpace($Value)) { return '' } + + # Preserve the generated suffix for round-trip RGs: --