connection_monitor.azcli

############################################################################
# Created by Jose Moreno
# April 2021
#
# Creates 2 web servers and two polling VMs that will use connection monitor
#   to check availability of each web server.
#
# Tested with zsh
############################################################################


# Variables
rg=monitortest
location1=westeurope
location2=eastus
vm_size=Standard_B1s

# Create RG
az group create -n $rg -l $location1

# Create Web servers
vm_cloudinit_filename=/tmp/cloudinit.txt
cat <<EOF > $vm_cloudinit_filename
#cloud-config
runcmd:
  - apt update && apt install -y python3-pip
  - pip3 install flask
  - wget https://raw.githubusercontent.com/erjosito/azcli/master/myip.py -O /root/myip.py
  - python3 /root/myip.py &
EOF
# Location1
vm_name="web-$location1"
vnet_name="vnet-$location1"
az network vnet create -g $rg -n $vnet_name --address-prefix "10.1.0.0/24" --subnet-name vm --subnet-prefix "10.1.0.0/26" -l $location1
az vm create -n $vm_name -g $rg --image UbuntuLTS --generate-ssh-keys --size $vm_size \
   --vnet-name $vnet_name --subnet vm --public-ip-address "${vm_name}-pip" \
   --custom-data $vm_cloudinit_filename -l $location1
az vm extension set --vm-name $vm_name -g $rg -n NetworkWatcherAgentLinux --publisher Microsoft.Azure.NetworkWatcher --version 1.4
# Location2
vm_name="web-$location2"
vnet_name="vnet-$location2"
az network vnet create -g $rg -n $vnet_name --address-prefix "10.2.0.0/24" --subnet-name vm --subnet-prefix "10.2.0.0/26" -l $location2
az vm create -n $vm_name -g $rg --image UbuntuLTS --generate-ssh-keys --size $vm_size \
   --vnet-name $vnet_name --subnet vm --public-ip-address "${vm_name}-pip" \
   --custom-data $vm_cloudinit_filename -l $location2
az vm extension set --vm-name $vm_name -g $rg -n NetworkWatcherAgentLinux --publisher Microsoft.Azure.NetworkWatcher --version 1.4

# Create pollers
vm_name="poller-$location1"
vnet_name="vnet-$location1"
az vm create -n $vm_name -g $rg -l $location1 --image ubuntuLTS --generate-ssh-keys \
    --public-ip-address "${vm_name}-pip" --vnet-name $vnet_name --size $vm_size --subnet vm
az vm extension set --vm-name $vm_name -g $rg -n NetworkWatcherAgentLinux --publisher Microsoft.Azure.NetworkWatcher --version 1.4
vm_name="poller-$location2"
vnet_name="vnet-$location2"
az vm create -n $vm_name -g $rg -l $location2 --image ubuntuLTS --generate-ssh-keys \
    --public-ip-address "${vm_name}-pip" --vnet-name $vnet_name --size $vm_size --subnet vm
az vm extension set --vm-name $vm_name -g $rg -n NetworkWatcherAgentLinux --publisher Microsoft.Azure.NetworkWatcher --version 1.4

# Peer the vnets
az network vnet peering create -n 1to2 -g $rg --vnet-name "vnet-${location1}" --remote-vnet "vnet-${location2}" --allow-vnet-access --allow-forwarded-traffic
az network vnet peering create -n 2to1 -g $rg --vnet-name "vnet-${location2}" --remote-vnet "vnet-${location1}" --allow-vnet-access --allow-forwarded-traffic

# VM IDs
web1_vm_id=$(az vm show -g $rg --query id -o tsv -n "web-${location1}") && echo $web1_vm_id
web2_vm_id=$(az vm show -g $rg --query id -o tsv -n "web-${location2}") && echo $web2_vm_id
poller1_vm_id=$(az vm show -g $rg --query id -o tsv -n "poller-${location1}") && echo $poller1_vm_id
poller2_vm_id=$(az vm show -g $rg --query id -o tsv -n "poller-${location2}") && echo $poller2_vm_id

# Create connection monitors
# Location1
monitor_name="poller-$location1"
test_name="web-$location1"
az network watcher connection-monitor create -n $monitor_name -g $rg -l $location1 \
    --test-group-name $test_name --endpoint-source-type AzureVM --endpoint-dest-type AzureVM \
    --endpoint-source-resource-id $poller1_vm_id --endpoint-source-name poller1 \
    --endpoint-dest-resource-id $web1_vm_id --endpoint-dest-name web1 \
    --test-config-name HttpConfig --protocol Http --http-port 8080 \
    --http-method GET --https-prefer false --http-path '/api/healthcheck'
az network watcher connection-monitor endpoint add --connection-monitor $monitor_name -l $location1 \
    --resource-id $web2_vm_id --name web2 --type AzureVM --dest-test-groups $test_name
# Location2
monitor_name="poller-$location2"
test_name="web-$location2"
az network watcher connection-monitor create -n $monitor_name -g $rg -l $location2 \
    --test-group-name $test_name --endpoint-source-type AzureVM --endpoint-dest-type AzureVM \
    --endpoint-source-resource-id $poller2_vm_id --endpoint-source-name poller2 \
    --endpoint-dest-resource-id $web1_vm_id --endpoint-dest-name web1 \
    --test-config-name HttpConfig --protocol Http --http-port 8080 \
    --http-method GET --https-prefer false --http-path '/api/healthcheck'
az network watcher connection-monitor endpoint add --connection-monitor $monitor_name -l $location2 \
    --resource-id $web2_vm_id --name web2 --type AzureVM --dest-test-groups $test_name

# Diagnostics
az network watcher connection-monitor list -l $location1 -o table # -g flag not supported
az network watcher connection-monitor list -l $location2 -o table # -g flag not supported

#########################
# Some useful functions #
#########################

# Get all VMs containing the string "vm" (that excludes the NVAs)
vm_list=$(az vm list -g $rg --query "[?contains(name,'vm')].name" -o tsv | sort -u)
# Build an associative array with all VM IDs
declare -A vm_id_list
echo "Getting VM IDs..."
while IFS= read -r vm_name
do
    vm_id=$(az vm show -g $rg --query id -o tsv -n $vm_name)
    vm_id_list+=([$vm_name]="$vm_id")
done <<< "$vm_list"

# Install the network watcher extension in every VM (Linux assumed)
while IFS= read -r vm
do
    echo "Installing Network Watcher extension in VM $vm..."
    az vm extension set --vm-name $vm -g $rg -n NetworkWatcherAgentLinux --publisher Microsoft.Azure.NetworkWatcher --version 1.4 -o none
done <<< "$vm_list"

# Install Apache web server in every VM (Ubuntu assumed)
while IFS= read -r vm
do
    echo "Installing Apache in VM $vm..."
    nic_id=$(az vm show -n $vm -g "$rg" --query 'networkProfile.networkInterfaces[0].id' -o tsv)
    pip_id=$(az network nic show --ids $nic_id --query 'ipConfigurations[0].publicIpAddress.id' -o tsv)
    pip_ip=$(az network public-ip show --ids $pip_id --query ipAddress -o tsv)
    ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip_ip "sudo apt install -y apache2"
done <<< "$vm_list"

# Create connection monitors
while IFS= read -r src_vm
do
    test_name="${src_vm}"
    location=$(az vm show -n $src_vm -g $rg --query location -o tsv)
    monitor_name="${src_vm}-${location}"
    echo "Creating connection monitor for source $src_vm in $location..."
    # The monitor is created with an HTTP config to ubuntu.com
    az network watcher connection-monitor create -n $monitor_name -g $rg -l $location \
        --test-group-name $test_name --endpoint-source-type AzureVM --endpoint-dest-type ExternalAddress \
        --endpoint-source-resource-id "$vm_id_list[$src_vm]" --endpoint-source-name $src_vm \
        --endpoint-dest-address "ubuntu.com" --endpoint-dest-name ubuntucom \
        --test-config-name Http --protocol Http --http-method GET --https-prefer false -o none
    # An ICMP config is added
    echo "Adding ICMP config to connection monitor $monitor_name..."
    az network watcher connection-monitor test-configuration add --connection-monitor $monitor_name -l $location \
        -n Icmp --protocol Icmp --icmp-disable-trace-route false --test-groups $test_name -o none
    # Plus the rest of the VMs are added as targets
    while IFS= read -r dst_vm
    do
        if [[ "$src_vm" != "$dst_vm" ]]
        then
            echo "Adding destination $dst_vm to connection monitor $monitor_name..."
            az network watcher connection-monitor endpoint add --connection-monitor $monitor_name -l $location \
                --resource-id "$vm_id_list[$dst_vm]" --name $dst_vm --type AzureVM --dest-test-groups $test_name -o none
        fi
    done <<< "$vm_list"
done <<< "$vm_list"

# Represents results in table form (configurable interval, default 5m)
function monitor_results() {
    interval=5m
    table_width=12
    # Get location list
    echo "Getting a list of locations..."
    location_list=$(az vm list -g $rg --query "[?contains(name,'vm')].location" -o tsv | sort -u)
    echo "$(echo $location_list | wc -l) locations with VMs found in resource group $rg"
    # Get list of log analytics workspace IDs
    declare -a logws_id_list
    logws_id_list=()
    while IFS= read -r location
    do
        logws_id_list_location=$(az network watcher connection-monitor list -l $location -o tsv --query '[].outputs[].workspaceSettings.workspaceResourceId')
        logws_id_list_location=$(echo $logws_id_list_location | sort -u)  # Unique values
        logws_id_list+=$logws_id_list_location
    done <<< "$location_list"
    # Remove duplicates
    # logws_id_list=$(echo $(for i in ${logws_id_list[@]}; do echo "$i"; done) | sort -u)
    echo "${#logws_id_list[@]} workspaces found"
    # Get list of log analytics customerIds
    logws_customer_id_list_string=$(for logws_id in "${logws_id_list[@]}"; do logws_rg=$(echo $logws_id | cut -d/ -f 5); logws_name=$(echo $logws_id | cut -d/ -f 9); az monitor log-analytics workspace show -n $logws_name -g $logws_rg --query customerId -o tsv; done)
    declare -a logws_customer_id_list
    logws_customer_id_list=()
    while IFS= read -r customer_id; do
        logws_customer_id_list+=$customer_id
    done <<< "$logws_customer_id_list_string"
    echo "${#logws_customer_id_list[@]} workspace customer IDs found"
    # Get list of tests, sources and destinations
    echo "Getting sources..."
    src_query="NWConnectionMonitorTestResult | where TimeGenerated > ago(1h) | distinct SourceName"
    src_list=$(for logws_customer_id in "${logws_customer_id_list[@]}"; do az monitor log-analytics query -w $logws_customer_id --analytics-query $src_query --query '[].SourceName' -o tsv; done)
    src_list=$(echo $src_list | sort -u)
    echo "$(echo $src_list | wc -l) sources found"
    echo "Getting destinations..."
    dst_query="NWConnectionMonitorTestResult | where TimeGenerated > ago(1h) | distinct DestinationName"
    dst_list=$(for logws_customer_id in "${logws_customer_id_list[@]}"; do az monitor log-analytics query -w $logws_customer_id --analytics-query $dst_query --query '[].DestinationName' -o tsv; done)
    dst_list=$(echo $dst_list | sort -u)
    echo "$(echo $dst_list | wc -l) destinations found"
    echo "Getting test configurations..."
    cfg_query="NWConnectionMonitorTestResult | where TimeGenerated > ago(1h) | distinct TestConfigurationName"
    cfg_list=$(for logws_customer_id in "${logws_customer_id_list[@]}"; do az monitor log-analytics query -w $logws_customer_id --analytics-query $cfg_query --query '[].TestConfigurationName' -o tsv; done)
    cfg_list=$(echo $cfg_list | sort -u)
    echo "$(echo $cfg_list | wc -l) test configurations found"
    # Get connectivity data with a union query to all workspaces
    if (( ${#logws_customer_id_list[@]} == 1 )); then
        line1='NWConnectionMonitorTestResult'
    else
        line1='union '
        for logws_customer_id in "${logws_customer_id_list[@]}"; do
            if [[ "$line1" != "union " ]]; then
                line1+=', '
            fi
            line1+='workspace("'$logws_customer_id'").NWConnectionMonitorTestResult'
        done
    fi
    line2="| where TimeGenerated > ago($interval) | project SourceName, DestinationName, TestResult, TestConfigurationName | summarize Pass=countif(TestResult == \"Pass\"), NoPass=countif(TestResult != \"Pass\") by SourceName, DestinationName, TestConfigurationName | project PassPercent=Pass/(Pass+NoPass), SourceName, DestinationName, TestConfigurationName"
    query="${line1} ${line2}"
    monitor_json=$(az monitor log-analytics query -w $logws_customer_id --analytics-query $query -o json)
    # For each source, print the latest results to the destination. Do a separate table per configuration test
    cfgs=''
    while IFS= read -r cfg
    do
        if [[ -n "$cfgs" ]]; then
            cfgs+='/'
        fi
        cfgs+="$cfg"
    done <<< "$cfg_list"
    echo " **** Test configurations: $cfgs ****"
    printf "%${table_width}s" "Source"
    while IFS= read -r dst
    do
        printf "%${table_width}s" $dst
    done <<< "$dst_list"
    printf "\n"
    while IFS= read -r src
    do
        printf "%${table_width}s" "$src"
        while IFS= read -r dst
        do
            if [[ "$src" != "$dst" ]]
            then
                # pass_query="NWConnectionMonitorTestResult | where TimeGenerated > ago($interval) | where SourceName==\"$src\" and DestinationName == \"$dst\" and TestConfigurationName == \"$cfg\" | project TestResult | summarize Pass=countif(TestResult == \"Pass\"), NoPass=countif(TestResult != \"Pass\") | project PassPercent=Pass/(Pass+NoPass)"
                # # Browse over the workspaces until something other than "None" arrives
                # pass_percent="None"
                # for logws_customer_id in "${logws_customer_id_list[@]}"
                # do
                #     if [[ "$pass_percent" == "None" ]]
                #     then
                #         # echo "**DEBUG*** Sending query $pass_query to workspace $logws_customer_id..."
                #         pass_percent=$(az monitor log-analytics query -w $logws_customer_id --analytics-query $pass_query --query '[].PassPercent' -o tsv)
                #     fi
                # done
                results=''
                while IFS= read -r cfg
                do
                    if [[ -n "$results" ]]; then
                        results+='/'
                    fi
                    pass_percent=$(echo $monitor_json | jq -r '.[] | select(.SourceName == "'$src'") | select(.DestinationName == "'$dst'") | select(.TestConfigurationName == "'$cfg'") | .PassPercent')
                    if [[ -z "$pass_percent" ]]; then
                        pass_percent='ND'
                    fi
                    results+="$pass_percent"
                done <<< "$cfg_list"
                printf "%${table_width}s" "$results"
            else
                printf "%${table_width}s" "N/A"
            fi
        done <<< "$dst_list"
        printf "\n"
    done <<< "$src_list"
}
monitor_results

###########
# Cleanup #
###########

# az group delete -n $rg -y --no-wait
# az network watcher connection-monitor delete -n poller-$location1 -l $location1
# az network watcher connection-monitor delete -n poller-$location2 -l $location2