Pipelines access key vault and storage through private endpoints

builds/checkin/e2e-checkin.yaml

@@ -45,27 +45,36 @@ stages:
       builtImages: $[stageDependencies.CheckBuildImages.check_source_change_runtime.outputs['check_files.RUNTIMECHANGES']]
       builtPackages: $[stageDependencies.CheckBuildPackages.check_source_change_edgelet.outputs['check_files.EDGELETCHANGES']]
     jobs:
-      - template: ../e2e/templates/get-storage-uri.yaml
-        parameters:
-          azureSubscription: $(az.subscription)
-
       - job: ubuntu_2204_amd64
         displayName: Ubuntu 22.04 amd64
-        dependsOn: Token
-        condition: succeeded('Token')
         variables:
           verbose: false
           os: linux
           arch: amd64
           identityArtifactName: aziot-identity-ubuntu22.04-amd64
           artifactName: iotedged-ubuntu22.04-amd64
-          sas_uri: $[ dependencies.Token.outputs['generate.sas_uri'] ]
         pool:
           name: $(pool.linux.name)
           demands:
             - ImageOverride -equals agent-aziotedge-ubuntu-22.04-msmoby
         steps:
+        - template: ../e2e/templates/e2e-vault-secrets.yaml
+          parameters:
+            azureSubscription: $(az.subscription)
+            keyVaultName: $(kv.name)
+        - template: ../e2e/templates/storage-token.yaml
+          parameters:
+            azureSubscription: $(az.subscription)
         - template: ../e2e/templates/e2e-setup.yaml
+          parameters:
+            iotHubResourceId: '$(iotHubResourceId)'
+            rootCaCertificate: '$(rootCaCertificate)'
+            rootCaKey: '$(rootCaKey)'
         - template: ../e2e/templates/e2e-run.yaml
           parameters:
+            containerRegistryPassword: '$(containerRegistryPassword)'
+            dpsGroupKeySymmetric: '$(dpsGroupKeySymmetric)'
+            eventHubCompatibleEndpoint: '$(eventHubCompatibleEndpoint)'
+            iotHubConnectionString: '$(iotHubConnectionString)'
+            rootCaPassword: '$(rootCaPassword)'
             sas_uri: $(sas_uri)

builds/e2e/connectivity.yaml

@@ -30,9 +30,17 @@ resources:
     branch: 'main'
 
 jobs:
-  - template: templates/get-storage-uri.yaml
-    parameters:
-      azureSubscription: $(azure.subscription)
+  - job: Token
+    displayName: 'Get SAS URI for Blob Storage Account'
+    pool:
+      name: $(pool.linux.name)
+      demands:
+        - ImageOverride -equals agent-aziotedge-ubuntu-22.04-msmoby
+    steps:
+      - template: templates/storage-token.yaml
+        parameters:
+          azureSubscription: $(azure.subscription)
+          crossJobVariables: true
 
 ################################################################################
   - job: linux_amd64_moby

builds/e2e/isa-95-smoke-test.yaml

@@ -270,11 +270,23 @@ stages:
           Start-Sleep -s 30
         displayName: Wait for network to lock
       - template: templates/nested-get-root-ca.yaml
+      - template: templates/e2e-vault-secrets.yaml
+        parameters:
+          azureSubscription: $(az.subscription)
+          keyVaultName: $(kv.name)
       - template: templates/e2e-setup.yaml
+        parameters:
+          iotHubResourceId: '$(iotHubResourceId)'
+          rootCaCertificate: '$(rootCaCertificate)'
+          rootCaKey: '$(rootCaKey)'
       - template: templates/e2e-run.yaml
         parameters:
-          EventHubCompatibleEndpoint: '$(IotHub-EventHubConnStr)'
-          IotHubConnectionString: '$(IotHub-ConnStr)'
+          containerRegistryPassword: '$(containerRegistryPassword)'
+          dpsGroupKeySymmetric: '$(dpsGroupKeySymmetric)'
+          eventHubCompatibleEndpoint: '$(IotHub-EventHubConnStr)'
+          iotHubConnectionString: '$(IotHub-ConnStr)'
+          rootCaPassword: '$(rootCaPassword)'
+          sas_uri: $(sas_uri)
           test_type: $(test_type)
       - template: templates/nested-isa95-unlock.yaml
         parameters:

builds/e2e/nested-e2e.yaml

@@ -38,9 +38,10 @@ stages:
 - stage: RunNestedTests
   dependsOn: LockAgents
   jobs:
-  - template: templates/get-storage-uri.yaml
+  - template: templates/e2e-secrets.yaml
     parameters:
       azureSubscription: $(az.subscription)
+      keyVaultName: $(kv.name)
   - template: templates/nested-parent-vm-setup.yaml
     parameters:
       upstream.protocol: mqtt
@@ -62,6 +63,12 @@ stages:
       arch: amd64
       identityArtifactName: aziot-identity-ubuntu24.04-amd64
       artifactName: iotedged-ubuntu24.04-amd64
+      containerRegistryPassword: $[ dependencies.Token.outputs['secrets.containerRegistryPassword'] ]
+      dpsGroupKeySymmetric: $[ dependencies.Token.outputs['secrets.dpsGroupKeySymmetric'] ]
+      iotHubResourceId: $[ dependencies.Token.outputs['secrets.iotHubResourceId'] ]
+      rootCaCertificate: $[ dependencies.Token.outputs['secrets.rootCaCertificate'] ]
+      rootCaKey: $[ dependencies.Token.outputs['secrets.rootCaKey'] ]
+      rootCaPassword: $[ dependencies.Token.outputs['secrets.rootCaPassword'] ]
       sas_uri: $[ dependencies.Token.outputs['generate.sas_uri'] ]
       nestededge: true
     pool:
@@ -77,13 +84,20 @@ stages:
       - template: templates/nested-get-root-ca.yaml
       - template: templates/nested-get-device-name.yaml
       - template: templates/e2e-setup.yaml
+        parameters:
+          iotHubResourceId: '$(iotHubResourceId)'
+          rootCaCertificate: '$(rootCaCertificate)'
+          rootCaKey: '$(rootCaKey)'
       - template: templates/e2e-clear-docker-cached-images.yaml
       - template: templates/e2e-run.yaml
         parameters:
-          EventHubCompatibleEndpoint: '$(IotHub-EventHubConnStr)'
-          IotHubConnectionString: '$(IotHub-ConnStr)'
-          test_type: nestededge_mqtt
+          containerRegistryPassword: '$(containerRegistryPassword)'
+          dpsGroupKeySymmetric: '$(dpsGroupKeySymmetric)'
+          eventHubCompatibleEndpoint: '$(IotHub-EventHubConnStr)'
+          iotHubConnectionString: '$(IotHub-ConnStr)'
+          rootCaPassword: '$(rootCaPassword)'
           sas_uri: $(sas_uri)
+          test_type: nestededge_mqtt
       - template: templates/nested-deploy-config.yaml
         parameters:
           deviceId: $(lvl5DeviceId)
@@ -100,10 +114,13 @@ stages:
           name: 4
       - template: templates/e2e-run.yaml
         parameters:
-          EventHubCompatibleEndpoint: '$(IotHub-EventHubConnStr)'
-          IotHubConnectionString: '$(IotHub-ConnStr)'
-          test_type: nestededge_amqp
+          containerRegistryPassword: '$(containerRegistryPassword)'
+          dpsGroupKeySymmetric: '$(dpsGroupKeySymmetric)'
+          eventHubCompatibleEndpoint: '$(IotHub-EventHubConnStr)'
+          iotHubConnectionString: '$(IotHub-ConnStr)'
+          rootCaPassword: '$(rootCaPassword)'
           sas_uri: $(sas_uri)
+          test_type: nestededge_amqp
 
 - stage: Cleanup
   condition: always()

builds/e2e/templates/e2e-run.yaml

@@ -1,8 +1,11 @@
 parameters:
-  EventHubCompatibleEndpoint: '$(TestEventHubCompatibleEndpoint)'
-  IotHubConnectionString: '$(TestIotHubConnectionString)'
-  test_type: ''
+  containerRegistryPassword: ''
+  dpsGroupKeySymmetric: ''
+  eventHubCompatibleEndpoint: ''
+  iotHubConnectionString: ''
+  rootCaPassword: ''
   sas_uri: ''
+  test_type: ''
 
 steps:
 # This E2E test pipeline uses the following filters in order to skip certain tests:
@@ -30,6 +33,12 @@ steps:
       $filter += '&Category!=FlakyOnArm'
       $filter += '&Category!=Amd64Only'
     }
+    if ('$(arch)' -eq 'arm32v7' -And '$(minimal)' -eq 'false')
+    {
+      # TestUploadModuleLogs and TestUploadSupportBundle use a storage account that is only accessible via private link.
+      # If the tests are running from custom on-premises agents (e.g., Raspberry Pis) then they won't have access.
+      $filter += '&FullyQualifiedName!~TestUploadModuleLogs&FullyQualifiedName!~TestUploadSupportBundle'
+    }
     if ($test_type -eq 'nestededge_mqtt')
     {
       $filter += '&Category!=SingleNodeOnly'
@@ -68,12 +77,12 @@ steps:
 
   displayName: Run tests ${{ parameters.test_type }}
   env:
-    E2E_DPS_GROUP_KEY: $(TestDpsGroupKeySymmetric)
-    E2E_EVENT_HUB_ENDPOINT: ${{ parameters['EventHubCompatibleEndpoint'] }}
-    E2E_IOT_HUB_CONNECTION_STRING: ${{ parameters['IotHubConnectionString'] }}
-    E2E_REGISTRIES__0__PASSWORD: $(TestContainerRegistryPassword)
-    E2E_ROOT_CA_PASSWORD: $(TestRootCaPassword)
-    E2E_BLOB_STORE_SAS: ${{ parameters['sas_uri'] }} 
+    E2E_DPS_GROUP_KEY: ${{ parameters.dpsGroupKeySymmetric }}
+    E2E_EVENT_HUB_ENDPOINT: ${{ parameters.eventHubCompatibleEndpoint }}
+    E2E_IOT_HUB_CONNECTION_STRING: ${{ parameters.iotHubConnectionString }}
+    E2E_REGISTRIES__0__PASSWORD: ${{ parameters.containerRegistryPassword }}
+    E2E_ROOT_CA_PASSWORD: ${{ parameters.rootCaPassword }}
+    E2E_BLOB_STORE_SAS: ${{ parameters.sas_uri }}
     no_proxy: 'localhost'
 
 - task: PublishTestResults@2

builds/e2e/templates/e2e-secrets.yaml

@@ -0,0 +1,21 @@
+parameters:
+    azureSubscription: ''
+    keyVaultName: ''
+
+jobs:
+  - job: Token
+    displayName: 'Get SAS URI for Blob Storage Account'
+    pool:
+      name: $(pool.linux.name)
+      demands:
+        - ImageOverride -equals agent-aziotedge-ubuntu-22.04-msmoby
+    steps:
+      - template: e2e-vault-secrets.yaml
+        parameters:
+          azureSubscription: ${{ parameters.azureSubscription }}
+          crossJobVariables: true
+          keyVaultName: ${{ parameters.keyVaultName }}
+      - template: storage-token.yaml
+        parameters:
+          azureSubscription: ${{ parameters.azureSubscription }}
+          crossJobVariables: true

builds/e2e/templates/e2e-setup.yaml

@@ -1,24 +1,14 @@
+parameters:
+  iotHubResourceId: ''
+  rootCaCertificate: ''
+  rootCaKey: ''
+
 steps:
   - checkout: self
     clean: true
     fetchDepth: 100
     submodules: recursive
 
-  - task: AzureKeyVault@2
-    displayName: Get secrets
-    inputs:
-      azureSubscription: $(az.subscription)
-      keyVaultName: $(kv.name)
-      secretsFilter: >-
-        TestContainerRegistryPassword,
-        TestDpsGroupKeySymmetric,
-        TestEventHubCompatibleEndpoint,
-        TestIotHubConnectionString,
-        TestIotHubResourceId,
-        TestRootCaCertificate,
-        TestRootCaKey,
-        TestRootCaPassword
-
   - bash: | 
       echo "Built Images Result is '$(builtImages)'"
       echo "Built Packages Result is '$(builtPackages)'"
@@ -108,14 +98,14 @@ steps:
   - pwsh: |
       $certsDir = '$(System.ArtifactsDirectory)/certs'
       New-Item "$certsDir" -ItemType Directory -Force | Out-Null
-      $env:ROOT_CERT | Out-File -Encoding Utf8 "$certsDir/rsa_root_ca.cert.pem"
-      $env:ROOT_KEY | Out-File -Encoding Utf8 "$certsDir/rsa_root_ca.key.pem"
+      $env:ROOT_CERT -replace '\\x0A', "`n" | Out-File -Encoding Utf8 "$certsDir/rsa_root_ca.cert.pem"
+      $env:ROOT_KEY -replace '\\x0A', "`n" | Out-File -Encoding Utf8 "$certsDir/rsa_root_ca.key.pem"
       Write-Output "##vso[task.setvariable variable=certsDir]$certsDir"
     displayName: Install CA keys
     env:
-      ROOT_CERT: $(TestRootCaCertificate)
-      ROOT_KEY: $(TestRootCaKey)
-  
+      ROOT_CERT: ${{ parameters.rootCaCertificate }}
+      ROOT_KEY: ${{ parameters.rootCaKey }}
+
   - pwsh: |
       $testDir = '$(Build.SourcesDirectory)/test/Microsoft.Azure.Devices.Edge.Test'
       dotnet build $testDir
@@ -238,4 +228,4 @@ steps:
       $context | ConvertTo-Json | Out-File -Encoding Utf8 '$(binDir)/context.json'
     displayName: Create test arguments file (context.json)
     env:
-      IOT_HUB_RESOURCE_ID: $(TestIotHubResourceId)
+      IOT_HUB_RESOURCE_ID: ${{ parameters.iotHubResourceId }}

builds/e2e/templates/e2e-vault-secrets.yaml

@@ -0,0 +1,49 @@
+parameters:
+  - name: azureSubscription
+    type: string
+    default: ''
+  - name: crossJobVariables
+    type: boolean
+    default: false
+  - name: keyVaultName
+    type: string
+    default: ''
+
+steps:
+  - task: AzureKeyVault@2
+    displayName: Get secrets
+    inputs:
+      azureSubscription: ${{ parameters.azureSubscription }}
+      keyVaultName: ${{ parameters.keyVaultName }}
+      secretsFilter: >-
+        TestContainerRegistryPassword,
+        TestDpsGroupKeySymmetric,
+        TestEventHubCompatibleEndpoint,
+        TestIotHubConnectionString,
+        TestIotHubResourceId,
+        TestRootCaCertificate,
+        TestRootCaKey,
+        TestRootCaPassword
+  - bash: |
+      # Azure Pipelines doesn't seem to handle multi-line task variables, so encode to one line
+      # See https://developercommunity.visualstudio.com/t/multiple-lines-variable-in-build-and-release/365667
+      readarray -t certlines < <(echo '$(TestRootCaCertificate)')
+      cert=$(printf '\\x0A%s' "${certlines[@]//$'\r'}") # Prepend each line with hex-escaped newline
+      cert=${cert:4} # Remove leading newline
+
+      # Azure Pipelines doesn't seem to handle multi-line task variables, so encode to one line
+      # See https://developercommunity.visualstudio.com/t/multiple-lines-variable-in-build-and-release/365667
+      readarray -t keylines < <(echo '$(TestRootCaKey)')
+      key=$(printf '\\x0A%s' "${keylines[@]//$'\r'}") # Prepend each line with hex-escaped newline
+      key=${key:4} # Remove leading newline
+
+      echo "##vso[task.setvariable variable=containerRegistryPassword;issecret=true;isoutput=${{ parameters.crossJobVariables }}]$(TestContainerRegistryPassword)"
+      echo "##vso[task.setvariable variable=dpsGroupKeySymmetric;issecret=true;isoutput=${{ parameters.crossJobVariables }}]$(TestDpsGroupKeySymmetric)"
+      echo "##vso[task.setvariable variable=eventHubCompatibleEndpoint;issecret=true;isoutput=${{ parameters.crossJobVariables }}]$(TestEventHubCompatibleEndpoint)"
+      echo "##vso[task.setvariable variable=iotHubConnectionString;issecret=true;isoutput=${{ parameters.crossJobVariables }}]$(TestIotHubConnectionString)"
+      echo "##vso[task.setvariable variable=iotHubResourceId;issecret=false;isoutput=${{ parameters.crossJobVariables }}]$(TestIotHubResourceId)"
+      echo "##vso[task.setvariable variable=rootCaCertificate;issecret=true;isoutput=${{ parameters.crossJobVariables }}]$cert"
+      echo "##vso[task.setvariable variable=rootCaKey;issecret=true;isoutput=${{ parameters.crossJobVariables }}]$key"
+      echo "##vso[task.setvariable variable=rootCaPassword;issecret=true;isoutput=${{ parameters.crossJobVariables }}]$(TestRootCaPassword)"
+    name: secrets
+    displayName: 'Make Key Vault secrets available to pipeline'