Merge pull request #621 from microsoft/dev

fix: Dev to main

Author: Roopan-Microsoft

.github/workflows/deploy-KMGeneric.yml

@@ -124,7 +124,7 @@ jobs:
           az deployment group create \
             --resource-group ${{ env.RESOURCE_GROUP_NAME }} \
             --template-file infra/main.bicep \
-            --parameters solutionName=${{env.SOLUTION_PREFIX}} contentUnderstandingLocation="swedencentral" secondaryLocation="${{ env.AZURE_LOCATION }}" imageTag=${{ steps.determine_tag.outputs.tagname }} gptDeploymentCapacity=150 azureAiServiceLocation="${{ env.AZURE_LOCATION }}" createdBy="Pipeline" tags="{'SecurityControl':'Ignore','Purpose':'Deploying and Cleaning Up Resources for Validation','CreatedDate':'$current_date'}"
+            --parameters solutionName=${{env.SOLUTION_PREFIX}} location="${{ env.AZURE_LOCATION }}" contentUnderstandingLocation="swedencentral" secondaryLocation="${{ env.AZURE_LOCATION }}" gptDeploymentCapacity=150 aiServiceLocation="${{ env.AZURE_LOCATION }}" createdBy="Pipeline" tags="{'SecurityControl':'Ignore','Purpose':'Deploying and Cleaning Up Resources for Validation','CreatedDate':'$current_date'}"
 
      
 

documents/AVMPostDeploymentGuide.md

@@ -0,0 +1,43 @@
+# AVM Post Deployment Guide
+
+> **📋 Note**: This guide is specifically for post-deployment steps after using the AVM template. For complete deployment instructions, see the main [Deployment Guide](./DeploymentGuide.md).
+
+---
+
+This document provides guidance on post-deployment steps after deploying the Conversation Knowledge Mining solution accelerator from the [AVM (Azure Verified Modules) repository](https://github.com/Azure/bicep-registry-modules/tree/main/avm/ptn/sa/conversation-knowledge-mining).
+
+## Prerequisites
+
+- **Deployed Infrastructure** - A successful Conversation Knowledge Mining solution accelerator deployment from the [AVM repository](https://github.com/Azure/bicep-registry-modules/tree/main/avm/ptn/sa/conversation-knowledge-mining)
+
+## Post Deployment Steps
+
+### 1. Access the Application
+
+1. Navigate to the [Azure Portal](https://portal.azure.com)
+2. Open the resource group created during deployment
+3. Locate the App Service with name starting with `app-`
+4. Copy the **URL** from the Overview page
+5. Open the URL in your browser to access the application
+
+### 2. Configure Authentication (Optional)
+
+If you want to enable authentication, configure it by following the [App Authentication Guide](./AppAuthentication.md).
+
+### 3. Verify Data Processing
+
+- Check that sample data has been uploaded to the storage account
+- Verify that the AI Search index has been created and populated
+- Confirm that the application loads without errors
+
+## Getting Started
+
+### Sample Questions
+
+Try these questions in the application to explore the solution capabilities:
+
+- "Total number of calls by date for the last 7 days"
+- "Show average handling time by topics in minutes"
+- "What are the top 7 challenges users reported?"
+- "Give a summary of billing issues"
+- "When customers call in about unexpected charges, what types of charges are they seeing?"

documents/CustomizeData.md

@@ -10,6 +10,9 @@ If you would like to update the solution to leverage your own data please follow
 
 1. Navigate to the storage account in the resource group you are using for this solution. 
 2. Open the `data` container
+
+> **Note for WAF-aligned deployments:** If your deployment uses private networking, you'll need to log into a VM within the virtual network to upload files. See [VM login instructions](#how-to-login-to-vm-using-azure-bastion) below.
+
 3. If you have audio files, upload them to `custom_audiodata` folder. If you have call transcript files, upload them to `custom_transcripts` folder.
 4. Navigate to the terminal and run the `run_process_data_scripts.sh` to process the new data into the solution with the following commands. 
     ```shell
@@ -21,17 +24,14 @@ If you would like to update the solution to leverage your own data please follow
     ```
     a. resourcegroupname_param - the name of the resource group.
 
-> Note (WAF‑aligned deployments): If you deployed the solution with the WAF / private networking option enabled, you must run the data processing script **from inside the deployed VM (jumpbox / processing VM)** so it can reach the private endpoints. Follow these steps:
->
-> 1. Connect to the VM (Azure Bastion, SSH, or RDP depending on OS).
-> 2. Ensure the repo (or the `infra/scripts` folder) is present. If not, clone or pull it.
-> 3. Open a Bash-compatible shell (Git Bash on Windows, or native bash on Linux).
-> 4. Run `az login` (add `--tenant <tenantId>` if required by your org policy).
-> 5. Navigate to `infra/scripts` and execute:
->    ```bash
->    bash run_process_data_scripts.sh <resource-group-name>
->    ```
-> 6. Replace `<resource-group-name>` with the name of the resource group you deployed (same value used for `resourcegroupname_param`).
->
-> Tip: If Azure CLI is not installed on the VM, install it first (see official docs) before running the script.
+## How to Login to VM Using Azure Bastion
+
+For WAF-aligned deployments with private networking:
+
+1. Navigate to your VM in the Azure portal
+2. Click **Connect** → **Bastion**
+3. Enter your VM credentials (username and password) and click **Connect**
+4. Wait for the Bastion connection to establish - this may take a few moments
+5. Once connected, you'll have access to the VM desktop/terminal interface
+
 

infra/process_data_scripts.bicep

@@ -2,25 +2,33 @@ param solutionLocation string
 param keyVaultName string
 param managedIdentityResourceId string
 param managedIdentityClientId string
+param storageAccount string
+param enablePrivateNetworking bool = false
+param subnetId string = ''
 
 var baseUrl = 'https://raw.githubusercontent.com/microsoft/Conversation-Knowledge-Mining-Solution-Accelerator/main/'
 
-resource process_data_scripts 'Microsoft.Resources/deploymentScripts@2020-10-01' = {
-  kind:'AzureCLI'
-  name: 'process_data_scripts'
-  location: solutionLocation // Replace with your desired location
-  identity: {
-    type: 'UserAssigned'
-    userAssignedIdentities: {
-      '${managedIdentityResourceId}' : {}
-    }
-  }
-  properties: {
+module uploadFiles 'br/public:avm/res/resources/deployment-script:0.5.1' = {
+  name: take('avm.res.resources.deployment-script.uploadFiles', 64)
+  params: {
+    kind: 'AzureCLI'
+    name: 'process_data_scripts'
     azCliVersion: '2.52.0'
-    primaryScriptUri: '${baseUrl}infra/scripts/process_data_scripts.sh' 
-    arguments: '${baseUrl} ${keyVaultName} ${managedIdentityClientId}' // Specify any arguments for the script
-    timeout: 'PT1H' // Specify the desired timeout duration
-    retentionInterval: 'PT1H' // Specify the desired retention interval
-    cleanupPreference:'OnSuccess'
+    cleanupPreference: 'Always'
+    location: solutionLocation
+    managedIdentities: {
+      userAssignedResourceIds: [
+        managedIdentityResourceId
+      ]
+    }
+    retentionInterval: 'P1D'
+    runOnce: true
+    primaryScriptUri: '${baseUrl}infra/scripts/process_data_scripts.sh'
+    arguments: '${baseUrl} ${keyVaultName} ${managedIdentityClientId}'
+    storageAccountResourceId: storageAccount
+    subnetResourceIds: (enablePrivateNetworking && !empty(subnetId)) ? [
+      subnetId
+    ] : null
+    timeout: 'PT1H'
   }
 }

infra/scripts/checkquota_ckmv2.sh

@@ -54,6 +54,7 @@ for REGION in "${REGIONS[@]}"; do
 
         if [ -z "$MODEL_INFO" ]; then
             echo "⚠️ WARNING: No quota information found for model: $MODEL in $REGION. Skipping."
+            INSUFFICIENT_QUOTA=true
             continue
         fi
 

infra/scripts/checkquota_km.sh

@@ -56,6 +56,7 @@ for REGION in "${REGIONS[@]}"; do
 
         if [ -z "$MODEL_INFO" ]; then
             echo "⚠️ WARNING: No quota information found for model: $MODEL in $REGION. Skipping."
+            INSUFFICIENT_QUOTA=true
             continue
         fi
 

infra/scripts/run_process_data_scripts.sh

@@ -36,24 +36,92 @@ sqlServerLocation=$(az sql server list --resource-group "$resourceGroupName" --q
 # === Retrieve the principal ID of the first user-assigned identity with name starting with 'id-' ===
 managedIdentityClientId=$(az identity list --resource-group "$resourceGroupName" --query "[?starts_with(name, 'id-') && !starts_with(name, 'id-sql-')].clientId | [0]" -o tsv)
 
+# === Check for VNet deployment ===
+echo "Checking for VNet deployment in resource group: $resourceGroupName"
+vnetResourceId=$(az network vnet list --resource-group "$resourceGroupName" --query "[0].id" -o tsv)
+
+# === Get resource group location ===
+rgLocation=$(az group show --name "$resourceGroupName" --query "location" -o tsv)
+
+# === Find storage account (always needed) ===
+echo "Looking for storage account in resource group..."
+storageAccountResourceId=$(az storage account list --resource-group "$resourceGroupName" --query "[0].id" -o tsv)
+
+if [ -z "$storageAccountResourceId" ]; then
+    echo "ERROR: No storage account found in resource group $resourceGroupName"
+    exit 1
+else
+    echo "Using storage account: $storageAccountResourceId"
+fi
+
+if [ -z "$vnetResourceId" ]; then
+    echo "No VNet found in resource group. Private networking is disabled."
+    enablePrivateNetworking="false"
+    subnetId=""
+    solutionLocation="$sqlServerLocation"
+    echo "Using SQL Server location for solution: $solutionLocation"
+else
+    echo "VNet found: $vnetResourceId"
+    echo "VNet detected - enabling private networking."
+    enablePrivateNetworking="true"
+    solutionLocation="$rgLocation"
+    echo "Using Resource Group location for solution: $solutionLocation"
+    
+    # === Find the deployment script subnet ===
+    echo "Looking for deployment-scripts subnet..."
+    subnetId=$(az network vnet subnet list --resource-group "$resourceGroupName" --vnet-name $(basename "$vnetResourceId") --query "[?name=='deployment-scripts'].id | [0]" -o tsv)
+    
+    if [ -z "$subnetId" ]; then
+        echo "Warning: deployment-scripts subnet not found. Checking for alternative subnet names..."
+        # Try alternative names
+        subnetId=$(az network vnet subnet list --resource-group "$resourceGroupName" --vnet-name $(basename "$vnetResourceId") --query "[?contains(name, 'deployment') || contains(name, 'script')].id | [0]" -o tsv)
+    fi
+    
+    if [ -z "$subnetId" ]; then
+        echo "Warning: No deployment script subnet found. Private networking will be disabled for deployment script."
+        enablePrivateNetworking="false"
+        subnetId=""
+    else
+        echo "Using deployment script subnet: $subnetId"
+    fi
+fi
+
 # === Validate that all required resources were found ===
-if [[ -z "$keyVaultName" || -z "$sqlServerLocation" || -z "$managedIdentityResourceId" || ! "$managedIdentityResourceId" =~ ^/subscriptions/ ]]; then
+if [[ -z "$keyVaultName" || -z "$solutionLocation" || -z "$managedIdentityResourceId" || ! "$managedIdentityResourceId" =~ ^/subscriptions/ ]]; then
   echo "ERROR: Could not find required resources in resource group $resourceGroupName or managedIdentityResourceId is invalid"
   exit 1
 fi
 
-echo "Using SQL Server Location: $sqlServerLocation"
+echo "Using Solution Location: $solutionLocation"
 echo "Using Key Vault: $keyVaultName"
 echo "Using Managed Identity Resource Id: $managedIdentityResourceId"
 echo "Using Managed Identity ClientId Id: $managedIdentityClientId"
+echo "Enable Private Networking: $enablePrivateNetworking"
+echo "Subnet ID: $subnetId"
+echo "Storage Account Resource ID: $storageAccountResourceId"
 
 # === Deploy resources using the specified Bicep template ===
 echo "Deploying Bicep template..."
 
+# Build base parameters
+deploymentParams="solutionLocation=$solutionLocation keyVaultName=$keyVaultName managedIdentityResourceId=$managedIdentityResourceId managedIdentityClientId=$managedIdentityClientId storageAccount=$storageAccountResourceId"
+
+# Add networking parameters if VNet is deployed
+if [ "$enablePrivateNetworking" = "true" ]; then
+    deploymentParams="$deploymentParams enablePrivateNetworking=true"
+    if [ -n "$subnetId" ]; then
+        deploymentParams="$deploymentParams subnetId=$subnetId"
+    fi
+else
+    deploymentParams="$deploymentParams enablePrivateNetworking=false"
+fi
+
+echo "Deployment parameters: $deploymentParams"
+
 # MSYS_NO_PATHCONV disables path conversion in Git Bash for Windows
 MSYS_NO_PATHCONV=1 az deployment group create \
   --resource-group "$resourceGroupName" \
   --template-file "$bicepFile" \
-  --parameters solutionLocation="$sqlServerLocation" keyVaultName="$keyVaultName" managedIdentityResourceId="$managedIdentityResourceId" managedIdentityClientId="$managedIdentityClientId"
+  --parameters $deploymentParams
 
 echo "Deployment completed."