Download installApps and installTestApps as part of "Download Project Dependencies"

Actions/DownloadProjectDependencies/DownloadProjectDependencies.Action.ps1

@@ -9,6 +9,8 @@
     [string] $token
 )
 
+Import-Module (Join-Path -Path $PSScriptRoot -ChildPath "DownloadProjectDependencies.psm1" -Resolve) -DisableNameChecking
+
 function DownloadDependenciesFromProbingPaths {
     param(
         $baseFolder,
@@ -123,6 +125,10 @@ Write-Host "::group::Downloading project dependencies from probing paths"
 $downloadedDependencies += DownloadDependenciesFromProbingPaths -baseFolder $baseFolder -project $project -destinationPath $destinationPath -token $token
 Write-Host "::endgroup::"
 
+Write-Host "::group::Downloading dependencies from settings (installApps and installTestApps)"
+$settingsDependencies = Get-DependenciesFromInstallApps -DestinationPath $destinationPath
+Write-Host "::endgroup::"
+
 $downloadedApps = @()
 $downloadedTestApps = @()
 
@@ -137,6 +143,10 @@ $downloadedDependencies | ForEach-Object {
     }
 }
 
+# Add dependencies from settings
+$downloadedApps += $settingsDependencies.Apps
+$downloadedTestApps += $settingsDependencies.TestApps
+
 OutputMessageAndArray -message "Downloaded dependencies (Apps)" -arrayOfStrings $downloadedApps
 OutputMessageAndArray -message "Downloaded dependencies (Test Apps)" -arrayOfStrings $downloadedTestApps
 

Actions/DownloadProjectDependencies/DownloadProjectDependencies.psm1

@@ -0,0 +1,154 @@
+<#
+    .SYNOPSIS
+    Downloads a file from a URL to a specified download path.
+    .DESCRIPTION
+    Downloads a file from a URL to a specified download path.
+    It handles URL decoding and sanitizes the file name.
+    If the downloaded file is a zip file, it extracts the .app files from it.
+    .PARAMETER Url
+    The URL of the file to download.
+    .PARAMETER DownloadPath
+    The path where the file should be downloaded.
+    .OUTPUTS
+    An array of paths to the downloaded/extracted .app files.
+#>
+function Get-AppFilesFromUrl {
+    Param(
+        [string] $Url,
+        [string] $DownloadPath
+    )
+    # Get the file name from the URL
+    $urlWithoutQuery = $Url.Split('?')[0].TrimEnd('/')
+    $rawFileName = [System.IO.Path]::GetFileName($urlWithoutQuery)
+    $decodedFileName = [Uri]::UnescapeDataString($rawFileName)
+    $decodedFileName = [System.IO.Path]::GetFileName($decodedFileName)
+
+    # Sanitize file name by removing invalid characters
+    $sanitizedFileName = $decodedFileName.Split([System.IO.Path]::getInvalidFileNameChars()) -join ""
+    $sanitizedFileName = $sanitizedFileName.Trim()
+
+    if ([string]::IsNullOrWhiteSpace($sanitizedFileName)) {
+        # Assume the file is an .app file if no valid name could be determined
+        $sanitizedFileName = "$([Guid]::NewGuid().ToString()).app"
+    }
+
+    # Get the final file path
+    $downloadedFile = Join-Path $DownloadPath $sanitizedFileName
+    if (Test-Path -LiteralPath $downloadedFile) {
+        OutputDebug -message "Overwriting existing file '$sanitizedFileName'. Multiple dependencies may resolve to the same filename."
+    }
+
+    # Download with retry logic
+    Invoke-CommandWithRetry -ScriptBlock {
+        Invoke-WebRequest -Method GET -UseBasicParsing -Uri $Url -OutFile $downloadedFile | Out-Null
+    } -RetryCount 3 -FirstDelay 5 -MaxWaitBetweenRetries 10
+
+    # Check if the downloaded file is a zip file
+    $extension = [System.IO.Path]::GetExtension($downloadedFile).ToLowerInvariant()
+    if ($extension -eq '.zip') {
+        Write-Host "Extracting .app files from zip archive: $sanitizedFileName"
+
+        # Extract to runner temp folder
+        $extractPath = Join-Path $env:RUNNER_TEMP ([System.IO.Path]::GetFileNameWithoutExtension($sanitizedFileName))
+        Expand-Archive -Path $downloadedFile -DestinationPath $extractPath -Force
+        Remove-Item -Path $downloadedFile -Force
+
+        # Find all .app files in the extracted folder and copy them to the download path
+        $appFiles = @()
+        foreach ($appFile in (Get-ChildItem -Path $extractPath -Filter '*.app' -Recurse)) {
+            $destFile = Join-Path $DownloadPath $appFile.Name
+            Copy-Item -Path $appFile.FullName -Destination $destFile -Force
+            $appFiles += $destFile
+        }
+
+        # Clean up the extracted folder
+        Remove-Item -Path $extractPath -Recurse -Force
+
+        if ($appFiles.Count -eq 0) {
+            throw "Zip archive '$sanitizedFileName' does not contain any .app files"
+        }
+        Write-Host "Found $($appFiles.Count) .app file(s) in zip archive"
+        return $appFiles
+    }
+
+    return @($downloadedFile)
+}
+
+<#
+    .SYNOPSIS
+    Downloads dependencies from URLs specified in installApps and installTestApps settings.
+    .DESCRIPTION
+    Reads the installApps and installTestApps arrays from the repository settings.
+    For each entry that is a URL (starts with http:// or https://):
+    - Resolves any secret placeholders in the format ${{ secretName }} by looking up the secret value
+    - Downloads the app file to the specified destination path
+    For entries that are not URLs (local paths), they are returned as-is.
+    .PARAMETER DestinationPath
+    The path where the app files should be downloaded.
+    .OUTPUTS
+    A hashtable with Apps and TestApps arrays containing the resolved local file paths.
+#>
+function Get-DependenciesFromInstallApps {
+    Param(
+        [string] $DestinationPath
+    )
+
+    $settings = $env:Settings | ConvertFrom-Json | ConvertTo-HashTable
+
+    # ENV:Secrets is not set when running Pull_Request trigger
+    if ($env:Secrets) {
+        $secrets = $env:Secrets | ConvertFrom-Json | ConvertTo-HashTable
+    }
+    else {
+        $secrets = @{}
+    }
+
+    $install = @{
+        "Apps" = @($settings.installApps)
+        "TestApps" = @($settings.installTestApps)
+    }
+
+    # Check if the installApps and installTestApps settings are empty
+    if (($settings.installApps.Count -eq 0) -and ($settings.installTestApps.Count -eq 0)) {
+        Write-Host "No installApps or installTestApps settings found."
+        return $install
+    }
+
+    # Replace secret names in install.apps and install.testApps and download files from URLs
+    foreach($list in @('Apps','TestApps')) {
+        $install."$list" = @($install."$list" | ForEach-Object {
+            $appFile = $_
+
+            # If the app file is not a URL, return it as is
+            if ($appFile -notlike 'http*://*') {
+                Write-Host "install$($list) contains a local path: $appFile"
+                return $appFile
+            }
+
+            # Else, check for secrets in the URL and replace them
+            $appFileUrl = $appFile
+            $pattern = '.*(\$\{\{\s*([^}]+?)\s*\}\}).*'
+            if ($appFile -match $pattern) {
+                $secretName = $matches[2]
+                if (-not $secrets.ContainsKey($secretName)) {
+                    throw "Setting: install$($list) references unknown secret '$secretName' in URL: $appFile"
+                }
+                $appFileUrl = $appFileUrl.Replace($matches[1],[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($secrets."$secretName")))
+            }
+
+            # Download the file (may return multiple .app files if it's a zip)
+            try {
+                Write-Host "Downloading from URL: $appFile"
+                $appFiles = Get-AppFilesFromUrl -Url $appFileUrl -DownloadPath $DestinationPath
+            } catch {
+                throw "Setting: install$($list) contains an inaccessible URL: $appFile. Error was: $($_.Exception.Message)"
+            }
+
+            return $appFiles
+        })
+    }
+
+    return $install
+}
+
+Export-ModuleMember -Function Get-AppFilesFromUrl, Get-DependenciesFromInstallApps

Actions/RunPipeline/RunPipeline.ps1

@@ -193,13 +193,13 @@ try {
     }
 
     $install = @{
-        "Apps" = $settings.installApps
-        "TestApps" = $settings.installTestApps
+        "Apps" = @()
+        "TestApps" = @()
     }
 
     if ($installAppsJson -and (Test-Path $installAppsJson)) {
         try {
-            $install.Apps += @(Get-Content -Path $installAppsJson -Raw | ConvertFrom-Json)
+            $install.Apps = Get-Content -Path $installAppsJson | ConvertFrom-Json
         }
         catch {
             throw "Failed to parse JSON file at path '$installAppsJson'. Error: $($_.Exception.Message)"
@@ -208,7 +208,7 @@ try {
 
     if ($installTestAppsJson -and (Test-Path $installTestAppsJson)) {
         try {
-            $install.TestApps += @(Get-Content -Path $installTestAppsJson -Raw | ConvertFrom-Json)
+            $install.TestApps = Get-Content -Path $installTestAppsJson | ConvertFrom-Json
         }
         catch {
             throw "Failed to parse JSON file at path '$installTestAppsJson'. Error: $($_.Exception.Message)"
@@ -220,30 +220,6 @@ try {
         $install.TestApps = $install.TestApps | ForEach-Object { $_.TrimStart("(").TrimEnd(")") }
     }
 
-    # Replace secret names in install.apps and install.testApps
-    foreach($list in @('Apps','TestApps')) {
-        $install."$list" = @($install."$list" | ForEach-Object {
-            $pattern = '.*(\$\{\{\s*([^}]+?)\s*\}\}).*'
-            $url = $_
-            if ($url -match $pattern) {
-                $finalUrl = $url.Replace($matches[1],[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($secrets."$($matches[2])")))
-            }
-            else {
-                $finalUrl = $url
-            }
-            # Check validity of URL
-            if ($finalUrl -like 'http*://*') {
-                try {
-                    Invoke-WebRequest -Method Head -UseBasicParsing -Uri $finalUrl | Out-Null
-                }
-                catch {
-                    throw "Setting: install$($list) contains an inaccessible URL: $($url). Error was: $($_.Exception.Message)"
-                }
-            }
-            return $finalUrl
-        })
-    }
-
     # Analyze app.json version dependencies before launching pipeline
 
     # Analyze InstallApps and InstallTestApps before launching pipeline

RELEASENOTES.md

@@ -11,6 +11,13 @@
 - AL-Go repositories with large amounts of projects may run into issues with too large environment variables
 - Discussion 1855 Add trigger 'workflow_call' to workflow 'Update AL-Go System Files' for reusability
 
+### Improving error detection and build reliability when downloading project dependencies
+
+The `DownloadProjectDependencies` action now downloads app files from URLs specified in the `installApps` and `installTestApps` settings upfront, rather than validating URLs at build time. This change provides:
+- Earlier detection of inaccessible or misconfigured URLs
+- Clearer error messages when secrets are missing or URLs are invalid
+- Warnings for potential issues like duplicate filenames or insecure HTTP connections
+
 ### Set default values for workflow inputs
 
 The `workflowDefaultInputs` setting now also applies to `workflow_call` inputs when an input with the same name exists for `workflow_dispatch`.