fixes for azure devops pipelines

Author: EMaher

src/services/identity-guide-service.ts

@@ -20,8 +20,6 @@ export interface IdentityGuideService {
   ): string;
 
   generateAzureDevOpsGuide(
-    subscriptionId: string,
-    resourceGroup: string,
     environments: string[]
   ): string;
 }
@@ -63,16 +61,16 @@ class IdentityGuideServiceImpl implements IdentityGuideService {
   }
 
   generateAzureDevOpsGuide(
-    subscriptionId: string,
-    resourceGroup: string,
     environments: string[]
   ): string {
-    const environmentsArrayPowerShell = environments.map((e) => `"${e}"`).join(', ');
-    const environmentsArrayBash = environments.map((e) => `"${e}"`).join(' ');
+    const environmentsArrayPowerShell = environments
+      .map((environment) => `"${environment}"`)
+      .join(', ');
+    const environmentsArrayBash = environments
+      .map((environment) => `"${environment}"`)
+      .join(' ');
 
     const coreSteps = this.renderTemplate(azureDevOpsIdentitySetupCoreTemplate, {
-      SUBSCRIPTION_ID: subscriptionId,
-      RESOURCE_GROUP: resourceGroup,
       ENVIRONMENTS_ARRAY_POWERSHELL: environmentsArrayPowerShell,
       ENVIRONMENTS_ARRAY_BASH: environmentsArrayBash,
     });

src/services/init-service.ts

@@ -332,6 +332,7 @@ class InitServiceImpl implements InitService {
     // Extract pipeline
     const extractPipelineConfig: ExtractPipelineConfig = {
       artifactDir: config.artifactDir,
+      environments: config.environments,
     };
     const extractContent = generateExtractPipeline(extractPipelineConfig);
     const extractPath = path.join(pipelinesDir, 'run-apim-extractor.yml');
@@ -407,11 +408,7 @@ class InitServiceImpl implements InitService {
         config.environments
       );
     } else {
-      guide = identityGuideService.generateAzureDevOpsGuide(
-        subscriptionId,
-        resourceGroup,
-        config.environments
-      );
+      guide = identityGuideService.generateAzureDevOpsGuide(config.environments);
     }
 
     // Save guide to file

src/templates/azure-devops/extract-pipeline.ts

@@ -2,40 +2,45 @@
 // Licensed under the MIT license.
 /**
  * T045: Azure DevOps extract pipeline template
- * Manual trigger with configuration choice and auto-PR creation
+ * Manual trigger with environment choice, configuration choice, and auto-PR creation
  */
 
 export interface ExtractPipelineConfig {
   artifactDir: string;
+  environments: string[];
 }
 
 export function generateExtractPipeline(config: ExtractPipelineConfig): string {
+  const defaultEnv = config.environments[0];
+  const envValues = config.environments.map((env) => `      - '${env}'`).join('\n');
+  const varGroupBlocks = config.environments
+    .map((env) => `- \${{ if eq(parameters.ENVIRONMENT, '${env}') }}:\n  - group: apim-${env}`)
+    .join('\n');
+
   return `# Azure DevOps Pipeline: Run APIM Extractor
 
 trigger: none
 
 parameters:
+  - name: ENVIRONMENT
+    type: string
+    displayName: 'Choose which environment to extract from'
+    default: '${defaultEnv}'
+    values:
+${envValues}
   - name: CONFIGURATION_YAML_PATH
     type: string
     displayName: 'Choose whether to extract all APIs or use the extraction configuration file'
     default: 'Extract All APIs'
     values:
       - 'Extract All APIs'
       - 'configuration.extractor.yaml'
-  - name: resourceGroup
-    type: string
-    displayName: 'Azure Resource Group'
-    default: $(APIM_RESOURCE_GROUP)
-  - name: serviceName
-    type: string
-    displayName: 'APIM Service Name'
-    default: $(APIM_SERVICE_NAME)
 
 pool:
   vmImage: 'ubuntu-latest'
 
 variables:
-  - group: apim-common
+${varGroupBlocks}
 
 steps:
   - checkout: self
@@ -56,43 +61,32 @@ steps:
     displayName: 'Install dependencies'
 
   - bash: |
-      echo "##vso[task.setvariable variable=RESOURCE_GROUP]\${{ parameters.resourceGroup }}"
-      echo "##vso[task.setvariable variable=SERVICE_NAME]\${{ parameters.serviceName }}"
-    displayName: 'Set parameters as variables'
-
-  - bash: |
-      RESOURCE_GROUP='$(RESOURCE_GROUP)'
-      SERVICE_NAME='$(SERVICE_NAME)'
+      APIM_RESOURCE_GROUP='$(APIM_RESOURCE_GROUP)'
+      APIM_SERVICE_NAME='$(APIM_SERVICE_NAME)'
       SERVICE_CONNECTION='$(AZURE_SERVICE_CONNECTION)'
       SUBSCRIPTION_ID='$(AZURE_SUBSCRIPTION_ID)'
 
-      is_unresolved() {
-        local value="$1"
-        case "$value" in
-          ""|\$\(*\)) return 0 ;;
-          *) return 1 ;;
-        esac
-      }
-
-      if is_unresolved "$RESOURCE_GROUP"; then
-        echo "##vso[task.logissue type=error]RESOURCE_GROUP was not resolved. Verify apim-common defines APIM_RESOURCE_GROUP and is authorized for this pipeline."
+      if [[ -z "$APIM_RESOURCE_GROUP" || "$APIM_RESOURCE_GROUP" == '$('*')' ]]; then
+        echo "##vso[task.logissue type=error]APIM_RESOURCE_GROUP is not set. Ensure variable group 'apim-\${{ parameters.ENVIRONMENT }}' is authorized and defines APIM_RESOURCE_GROUP."
         exit 2
       fi
 
-      if is_unresolved "$SERVICE_NAME"; then
-        echo "##vso[task.logissue type=error]SERVICE_NAME was not resolved. Verify apim-common defines APIM_SERVICE_NAME and is authorized for this pipeline."
+      if [[ -z "$APIM_SERVICE_NAME" || "$APIM_SERVICE_NAME" == '$('*')' ]]; then
+        echo "##vso[task.logissue type=error]APIM_SERVICE_NAME is not set. Ensure variable group 'apim-\${{ parameters.ENVIRONMENT }}' is authorized and defines APIM_SERVICE_NAME."
         exit 2
       fi
 
-      if is_unresolved "$SERVICE_CONNECTION"; then
-        echo "##vso[task.logissue type=error]AZURE_SERVICE_CONNECTION was not resolved. Verify apim-common defines AZURE_SERVICE_CONNECTION and is authorized for this pipeline."
+      if [[ -z "$SERVICE_CONNECTION" || "$SERVICE_CONNECTION" == '$('*')' ]]; then
+        echo "##vso[task.logissue type=error]AZURE_SERVICE_CONNECTION is not set. Ensure variable group 'apim-\${{ parameters.ENVIRONMENT }}' is authorized and defines AZURE_SERVICE_CONNECTION."
         exit 2
       fi
 
-      if is_unresolved "$SUBSCRIPTION_ID"; then
-        echo "##vso[task.logissue type=error]AZURE_SUBSCRIPTION_ID was not resolved. Verify apim-common defines AZURE_SUBSCRIPTION_ID and is authorized for this pipeline."
+      if [[ -z "$SUBSCRIPTION_ID" || "$SUBSCRIPTION_ID" == '$('*')' ]]; then
+        echo "##vso[task.logissue type=error]AZURE_SUBSCRIPTION_ID is not set. Ensure variable group 'apim-\${{ parameters.ENVIRONMENT }}' is authorized and defines AZURE_SUBSCRIPTION_ID."
         exit 2
       fi
+
+      echo "All required variables are configured for environment '\${{ parameters.ENVIRONMENT }}'"
     displayName: 'Validate required variables'
 
   - task: AzureCLI@2
@@ -103,25 +97,11 @@ steps:
       scriptType: 'bash'
       scriptLocation: 'inlineScript'
       inlineScript: |
-        SUBSCRIPTION_ID='$(AZURE_SUBSCRIPTION_ID)'
-
-        is_unresolved() {
-          local value="$1"
-          case "$value" in
-            ""|\$\(*\)) return 0 ;;
-            *) return 1 ;;
-          esac
-        }
-
-        if is_unresolved "$SUBSCRIPTION_ID"; then
-          echo "##vso[task.logissue type=error]AZURE_SUBSCRIPTION_ID was not resolved. Ensure apim-common is authorized and defines AZURE_SUBSCRIPTION_ID."
-          exit 2
-        fi
         npx @peterhauge/apiops-cli extract \\
-          --resource-group "$(RESOURCE_GROUP)" \\
-          --service-name "$(SERVICE_NAME)" \\
+          --resource-group "$(APIM_RESOURCE_GROUP)" \\
+          --service-name "$(APIM_SERVICE_NAME)" \\
           --output ${config.artifactDir} \\
-          --subscription-id "$SUBSCRIPTION_ID"
+          --subscription-id "$(AZURE_SUBSCRIPTION_ID)"
 
   - task: AzureCLI@2
     displayName: 'Run APIM Extract (With Configuration)'
@@ -131,47 +111,51 @@ steps:
       scriptType: 'bash'
       scriptLocation: 'inlineScript'
       inlineScript: |
-        SUBSCRIPTION_ID='$(AZURE_SUBSCRIPTION_ID)'
-
-        is_unresolved() {
-          local value="$1"
-          case "$value" in
-            ""|\$\(*\)) return 0 ;;
-            *) return 1 ;;
-          esac
-        }
-
-        if is_unresolved "$SUBSCRIPTION_ID"; then
-          echo "##vso[task.logissue type=error]AZURE_SUBSCRIPTION_ID was not resolved. Ensure apim-common is authorized and defines AZURE_SUBSCRIPTION_ID."
-          exit 2
-        fi
         npx @peterhauge/apiops-cli extract \\
-          --resource-group "$(RESOURCE_GROUP)" \\
-          --service-name "$(SERVICE_NAME)" \\
+          --resource-group "$(APIM_RESOURCE_GROUP)" \\
+          --service-name "$(APIM_SERVICE_NAME)" \\
           --output ${config.artifactDir} \\
           --filter configuration.extractor.yaml \\
-          --subscription-id "$SUBSCRIPTION_ID"
+          --subscription-id "$(AZURE_SUBSCRIPTION_ID)"
 
   - task: PublishPipelineArtifact@1
     displayName: 'Publish artifacts'
     inputs:
       targetPath: ${config.artifactDir}
       artifactName: apim-artifacts
 
-  - script: |
+  - bash: |
       BRANCH_NAME="apim-extract-$(Build.BuildId)"
+      TARGET_BRANCH="$(Build.SourceBranch)"
+      BUILD_ID="$(Build.BuildId)"
+
       git config user.name "Azure DevOps"
       git config user.email "azuredevops@microsoft.com"
       git checkout -b "$BRANCH_NAME"
       git add ${config.artifactDir}
       if git diff --cached --quiet; then
         echo "No changes to commit"
+        exit 0
+      fi
+      git commit -m "chore: update APIM artifacts from extract"
+      git push origin "$BRANCH_NAME"
+
+      PR_PAYLOAD=$(printf '{"title":"APIM Extract - Update artifacts","description":"Auto-generated by APIM extract pipeline run %s","sourceRefName":"refs/heads/%s","targetRefName":"%s"}' \\
+        "$BUILD_ID" "$BRANCH_NAME" "$TARGET_BRANCH")
+
+      HTTP_STATUS=$(curl -s -o /tmp/pr_response.json -w "%{http_code}" -X POST \\
+        -H "Content-Type: application/json" \\
+        -H "Authorization: Bearer $SYSTEM_ACCESSTOKEN" \\
+        "$(System.TeamFoundationCollectionUri)$(System.TeamProject)/_apis/git/repositories/$(Build.Repository.Name)/pullrequests?api-version=7.1" \\
+        -d "$PR_PAYLOAD")
+
+      if [ "$HTTP_STATUS" = "201" ]; then
+        PR_ID=$(python3 -c "import json; print(json.load(open('/tmp/pr_response.json'))['pullRequestId'])" 2>/dev/null)
+        echo "##vso[task.logissue type=warning]Pull request created: $(System.TeamFoundationCollectionUri)$(System.TeamProject)/_git/$(Build.Repository.Name)/pullrequest/$PR_ID"
       else
-        git commit -m "chore: update APIM artifacts from extract"
-        git push origin "$BRANCH_NAME"
-        echo "##vso[task.logissue type=warning]Branch '$BRANCH_NAME' pushed. Please create a pull request to merge the changes."
+        echo "##vso[task.logissue type=warning]Could not auto-create PR (HTTP $HTTP_STATUS). Please create a PR from '$BRANCH_NAME' to '$TARGET_BRANCH' manually."
       fi
-    displayName: 'Create branch with changes'
+    displayName: 'Create branch and open pull request'
     env:
       SYSTEM_ACCESSTOKEN: $(System.AccessToken)
 `;

src/templates/azure-devops/publish-pipeline.ts

@@ -11,14 +11,15 @@ export interface PublishPipelineConfig {
 }
 
 export function generatePublishPipeline(config: PublishPipelineConfig): string {
+  const defaultEnvironment = config.environments[0] ?? 'dev';
   const envValues = config.environments.map((env) => `      - '${env}'`).join('\n');
 
-  const stages = config.environments.map((env, idx) => {
-    const dependsOn = idx === 0 ? '' : `  dependsOn: Publish_${config.environments[idx - 1]}\n`;
+  const stages = config.environments.map((env) => {
+    const envUpper = env.toUpperCase();
 
-    return `${dependsOn}- stage: Publish_${env}
+    return `- stage: Publish_${env}
   displayName: 'Publish to ${env}'
-  condition: or(eq('\${{ parameters.ENVIRONMENT }}', '${env}'), eq('\${{ parameters.ENVIRONMENT }}', 'all'))
+  condition: eq('\${{ parameters.ENVIRONMENT }}', '${env}')
   variables:
     - group: apim-${env}
   jobs:
@@ -37,29 +38,49 @@ export function generatePublishPipeline(config: PublishPipelineConfig): string {
               - task: UseNode@1
                 displayName: 'Setup Node.js'
                 inputs:
-                  versionSpec: '22.x'
+                  version: '22.x'
 
-              - script: npm ci
+              - script: |
+                  node -v
+                  npm -v
+                displayName: 'Verify Node.js version'
+
+              - script: |
+                  if [[ -f package-lock.json || -f npm-shrinkwrap.json ]]; then
+                    npm ci
+                  else
+                    npm install
+                  fi
                 displayName: 'Install dependencies'
 
               - task: replacetokens@6
                 displayName: 'Substitute tokens in configuration.${env}.yaml'
                 inputs:
                   sources: 'configuration.${env}.yaml'
+                  tokenPattern: 'custom'
                   tokenPrefix: '{#['
                   tokenSuffix: ']#}'
+                  missingVarAction: 'keep'
+
+              - script: |
+                  if grep -q '{#\\[' configuration.${env}.yaml; then
+                    echo "Unresolved tokens remain in configuration.${env}.yaml"
+                    grep -o '{#\\[[^]]*\\]#}' configuration.${env}.yaml | sort -u
+                    exit 1
+                  fi
+                displayName: 'Validate token substitution (${env})'
 
               - task: AzureCLI@2
                 displayName: 'Publish to ${env} (incremental - last commit only)'
                 condition: ne('\${{ parameters.COMMIT_ID_CHOICE }}', 'publish-all-artifacts-in-repo')
                 inputs:
-                  azureSubscription: '$(AZURE_SERVICE_CONNECTION_${env.toUpperCase()})'
+                  azureSubscription: 'AZURE_SERVICE_CONNECTION_${envUpper}'
                   scriptType: 'bash'
                   scriptLocation: 'inlineScript'
                   inlineScript: |
                     npx apiops publish \\
-                      --resource-group $(APIM_RESOURCE_GROUP_${env.toUpperCase()}) \\
-                      --service-name $(APIM_SERVICE_NAME_${env.toUpperCase()}) \\
+                      --resource-group $(APIM_RESOURCE_GROUP_${envUpper}) \\
+                      --service-name $(APIM_SERVICE_NAME_${envUpper}) \\
                       --source ${config.artifactDir} \\
                       --overrides configuration.${env}.yaml \\
                       --commit-id $(Build.SourceVersion) \\
@@ -69,13 +90,13 @@ export function generatePublishPipeline(config: PublishPipelineConfig): string {
                 displayName: 'Publish to ${env} (all artifacts)'
                 condition: eq('\${{ parameters.COMMIT_ID_CHOICE }}', 'publish-all-artifacts-in-repo')
                 inputs:
-                  azureSubscription: '$(AZURE_SERVICE_CONNECTION_${env.toUpperCase()})'
+                  azureSubscription: 'AZURE_SERVICE_CONNECTION_${envUpper}'
                   scriptType: 'bash'
                   scriptLocation: 'inlineScript'
                   inlineScript: |
                     npx apiops publish \\
-                      --resource-group $(APIM_RESOURCE_GROUP_${env.toUpperCase()}) \\
-                      --service-name $(APIM_SERVICE_NAME_${env.toUpperCase()}) \\
+                      --resource-group $(APIM_RESOURCE_GROUP_${envUpper}) \\
+                      --service-name $(APIM_SERVICE_NAME_${envUpper}) \\
                       --source ${config.artifactDir} \\
                       --overrides configuration.${env}.yaml \\
                       --subscription-id $(AZURE_SUBSCRIPTION_ID)
@@ -106,9 +127,8 @@ parameters:
   - name: ENVIRONMENT
     type: string
     displayName: 'Choose which environment to publish to'
-    default: 'all'
+    default: '${defaultEnvironment}'
     values:
-      - 'all'
 ${envValues}
 
 stages: