routeserver.azcli

#############################################################################
# Creates a basic 1-vnet setup with ARS, CSR as NVA and a VM
# Optionally creates additional components such as Linux-based appliances,
#   peered vnets, ER/VPN VNGs and simulated onprem environments.
# This script has multiple sections, it is not intended to use all sections
#   at once, but to pick and choose what you need for your particular test.
#
# July 2021, Jose Moreno
#############################################################################


# Control
nva_type=linux      # can be csr or linux
redundant_nva=no    # this script only supports the CSR for redundant CSRs
create_spokes=yes   # can be yes or no

# Variables
rg=routeserver
location=westeurope
vnet_name=hub
vnet_prefix=172.16.0.0/16
vnet_prefix_long='172.16.0.0 255.255.0.0'
rs_subnet_name=RouteServerSubnet
rs_subnet_prefix=172.16.1.0/24
rs_name=hubrs
hub_vm_subnet_name=vm
hub_vm_subnet_prefix=172.16.3.0/24
# GatewaySubnet
gw_subnet_prefix=172.16.254.0/24
# CSR
publisher=cisco
offer=cisco-csr-1000v
sku=16_12-byol
csr_username=$(whoami)
csr_password=Microsoft123!
hub_csrext_subnet_name=csrext
hub_csrext_subnet_prefix=172.16.11.0/24
hub_csrext_subnet_prefix_long_reversemask='172.16.11.0 0.0.0.255'
hub_csrext_nsg_name=hubcsrext-nsg
hub_csrint_subnet_name=csrint
hub_csrint_subnet_prefix=172.16.12.0/24
hub_csrint_nsg_name=hubcsrint-nsg
hub_csr_asn=65001
hub_csr1_name=csr1
hub_csr1_bgp_ip=172.16.12.11
hub_csr2_name=csr2
hub_csr2_bgp_ip=172.16.12.12
# Linux NVA
hub_linuxnva_asn=65002
hub_linuxnva_subnet_name=linuxnva
hub_linuxnva_subnet_prefix=172.16.10.0/24
hub_linuxnva_name=ubuntufw
hub_linuxnva_pip_name="${hub_linuxnva_name}-pip"
hub_linuxnva_cloudinit_file=/tmp/linuxnva_cloudinit.txt
# Spoke 1
spoke1_name=spoke1
spoke1_prefix=172.17.0.0/16
spoke1_vm_subnet_name=vm
spoke1_vm_subnet_prefix=172.17.0.0/24
spoke1_csr_subnet_name=csr
spoke1_csr_subnet_prefix=172.17.1.0/24
spoke1_csr_name=spoke1csr
spoke1_csr_asn=65001
# Spoke 2
spoke2_name=spoke2
spoke2_prefix=172.18.0.0/16
spoke2_vm_subnet_name=vm
spoke2_vm_subnet_prefix=172.18.0.0/24
# Azure VPN GW
vpngw_name=vpngw
vpngw_asn=65515
vpngw_pip="${vpngw_name}-pip"
# Azure ExpressRoute GW
ergw_name=ergw
ergw_pip="${ergw_name}-pip"
er_location=germanywestcentral
er_pop=Frankfurt
er_provider=Megaport
er_circuit_name=tester
# Onprem (simulated with Linux NVA)
onprem_vnet_name=onprem
onprem_vnet_prefix=192.168.0.0/16
onprem_nva_subnet_name=onprem
onprem_nva_subnet_prefix=192.168.0.0/24
onprem_gw_subnet_name=GatewaySubnet
onprem_gw_subnet_prefix=192.168.1.0/24
onprem_linuxnva_asn=65102
onprem_linuxnva_name=onpremnva
onprem_linuxnva_pip=${onprem_linuxnva_name}-pip
onprem_linuxnva_ip=192.168.0.20

####################
# Helper functions #
####################

# Auxiliary function to get the first IP of a subnet (default gateway)
function first_ip(){
    subnet=$1
    IP=$(echo $subnet | cut -d/ -f 1)
    IP_HEX=$(printf '%.2X%.2X%.2X%.2X\n' `echo $IP | sed -e 's/\./ /g'`)
    NEXT_IP_HEX=$(printf %.8X `echo $(( 0x$IP_HEX + 1 ))`)
    NEXT_IP=$(printf '%d.%d.%d.%d\n' `echo $NEXT_IP_HEX | sed -r 's/(..)/0x\1 /g'`)
    echo "$NEXT_IP"
}

# Update home PC IP
function update_myip() {
  myip=$(curl -s4 ifconfig.co) && echo $myip
  az network nsg rule update -n SSHInbound --nsg-name $hub_csrext_nsg_name -g $rg \
    --protocol 'TCP' --access Allow --priority 1020 --direction Inbound \
    --source-address-prefixes "${myip}/32" --source-port-ranges '*' \
    --destination-address-prefixes VirtualNetwork --destination-port-ranges '22'
  az network route-table route update --route-table-name hubvmrt -g $rg --address-prefix "${myip}/32" --name mypc
  az network route-table route update --route-table-name spokevmrt -g $rg --address-prefix "${myip}/32" --name mypc
}

#####################
# Hub VNet with ARS #
#####################

# Create Vnet
az group create -n $rg -l $location
az network vnet create -g $rg -n $vnet_name --address-prefix $vnet_prefix --subnet-name $rs_subnet_name --subnet-prefix $rs_subnet_prefix

# Create additional subnets (no subnet can be created while the route server is being provisioned, same as VNGs)
az network vnet subnet create -n $hub_csrext_subnet_name --address-prefix $hub_csrext_subnet_prefix --vnet-name $vnet_name -g $rg
az network vnet subnet create -n $hub_csrint_subnet_name --address-prefix $hub_csrint_subnet_prefix --vnet-name $vnet_name -g $rg
az network vnet subnet create -n $hub_vm_subnet_name --address-prefix $hub_vm_subnet_prefix --vnet-name $vnet_name -g $rg
az network vnet subnet create -n GatewaySubnet --address-prefix $gw_subnet_prefix --vnet-name $vnet_name -g $rg

# Create Route Server
rs_subnet_id=$(az network vnet subnet show -n $rs_subnet_name --vnet-name $vnet_name -g $rg --query id -o tsv)
rs_pip_name="${rs_name}-pip"
az network public-ip create -g $rg -n $rs_pip_name --allocation-method Static --sku Standard
az network routeserver create -n $rs_name -g $rg --hosted-subnet $rs_subnet_id -l $location --public-ip-address $rs_pip_name
az network routeserver update -n $rs_name -g $rg --allow-b2b-traffic true   # Optional
# az network routeserver delete -n $rs_name -g $rg -y  # Danger Zone!

# Get info (once created)
rs_ip1=$(az network routeserver show -n $rs_name -g $rg --query 'virtualRouterIps[0]' -o tsv) && echo $rs_ip1
rs_ip2=$(az network routeserver show -n $rs_name -g $rg --query 'virtualRouterIps[1]' -o tsv) && echo $rs_ip2
rs_asn=$(az network routeserver show -n $rs_name -g $rg --query 'virtualRouterAsn' -o tsv) && echo $rs_asn

# Create test VM in hub
az vm create -n hubvm -g $rg -l $location --image ubuntuLTS --generate-ssh-keys \
    --public-ip-address hubvm-pip --vnet-name $vnet_name --size Standard_B1s --subnet $hub_vm_subnet_name
hub_vm_ip=$(az network public-ip show -n hubvm-pip --query ipAddress -o tsv -g $rg) && echo $hub_vm_ip
hub_vm_nic_id=$(az vm show -n hubvm -g "$rg" --query 'networkProfile.networkInterfaces[0].id' -o tsv) && echo $hub_vm_nic_id
hub_vm_private_ip=$(az network nic show --ids $hub_vm_nic_id --query 'ipConfigurations[0].privateIpAddress' -o tsv) && echo $hub_vm_private_ip

####################
#       CSR        #
####################

# Default gateways
if [[ "$nva_type" == "csr" ]]; then
    hub_csrext_default_gw=$(first_ip $hub_csrext_subnet_prefix) && echo $hub_csrext_default_gw
    hub_csrint_default_gw=$(first_ip $hub_csrint_subnet_prefix) && echo $hub_csrint_default_gw

    # Create hub CSR1 as NVA with 2 NICs (easier for NAT)
    version=$(az vm image list -p $publisher -f $offer -s $sku --all --query '[0].version' -o tsv)
    az vm image terms accept --urn ${publisher}:${offer}:${sku}:${version}
    # NSGs (104.21.25.86 and 172.67.133.228 are the addresses of ifconfig.co)
    az network nsg create -n $hub_csrext_nsg_name -g $rg -l $location
    myip=$(curl -s4 ifconfig.co) && echo $myip
    az network nsg rule create -n Internet2VnetInbound --nsg-name $hub_csrext_nsg_name -g $rg \
      --protocol '*' --access Allow --priority 1010 --direction Inbound \
      --source-address-prefixes 8.8.8.8/32 104.21.25.86/32 172.67.133.228/32 --source-port-ranges '*' \
      --destination-address-prefixes VirtualNetwork --destination-port-ranges '*'
    az network nsg rule create -n SSHInbound --nsg-name $hub_csrext_nsg_name -g $rg \
      --protocol 'TCP' --access Allow --priority 1020 --direction Inbound \
      --source-address-prefixes "${myip}/32" --source-port-ranges '*' \
      --destination-address-prefixes VirtualNetwork --destination-port-ranges '22'
    az network nsg create -n $hub_csrint_nsg_name -g $rg -l $location
    az network nsg rule create -n Vnet2InternetInbound --nsg-name $hub_csrint_nsg_name -g $rg \
      --protocol '*' --access Allow --priority 1010 --direction Inbound \
      --source-address-prefixes VirtualNetwork --source-port-ranges '*' \
      --destination-address-prefixes Internet --destination-port-ranges '*'
    az network nsg rule create -n Internet2VnetOutbound --nsg-name $hub_csrint_nsg_name -g $rg \
      --protocol '*' --access Allow --priority 1010 --direction Outbound \
      --source-address-prefixes 8.8.8.8/32 104.21.25.86/32 172.67.133.228/32 --source-port-ranges '*' \
      --destination-address-prefixes VirtualNetwork --destination-port-ranges '*'
    # PIP
    az network public-ip create -g $rg -n "${hub_csr1_name}-pip" --sku basic --allocation-method Static
    # NICs
    az network nic create -n "${hub_csr1_name}-nic0" -g $rg --vnet-name $vnet_name --subnet $hub_csrext_subnet_name --network-security-group "$hub_csrext_nsg_name" --public-ip-address "${hub_csr1_name}-pip" --ip-forwarding
    az network nic create -n "${hub_csr1_name}-nic1" -g $rg --vnet-name $vnet_name --subnet $hub_csrint_subnet_name --network-security-group "$hub_csrint_nsg_name" --private-ip-address $hub_csr1_bgp_ip --ip-forwarding
    # VM
    az vm create -n $hub_csr1_name -g $rg -l $location \
        --image ${publisher}:${offer}:${sku}:${version} \
        --admin-username "$csr_username" --admin-password $csr_password --authentication-type all --generate-ssh-keys \
        --nics "${hub_csr1_name}-nic0" "${hub_csr1_name}-nic1"
    #hub_csr1_nic_id=$(az vm show -n $hub_csr1_name -g "$rg" --query 'networkProfile.networkInterfaces[0].id' -o tsv)
    #az network nic update --ids $hub_csr1_nic_id --ip-forwarding

    # Route table without gateway route propagation on the external nic
    az network route-table create -n csrext -g $rg -l $location --disable-bgp-route-propagation
    az network vnet subnet update --name $hub_csrext_subnet_name --route-table csrext --vnet-name $vnet_name --resource-group $rg

    # Test access over SSH
    hub_csr1_ip=$(az network public-ip show -n "${hub_csr1_name}-pip" --query ipAddress -o tsv -g $rg) && echo $hub_csr1_ip
    ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no "$hub_csr1_ip" "sh ip int b"

    # Configure hub CSR1 with BGP
    ssh -o BatchMode=yes -o StrictHostKeyChecking=no "$hub_csr1_ip" <<EOF
config t
    no ip domain lookup
    interface Loopback 0
        ip address 1.1.1.1 255.255.255.255
    router bgp ${hub_csr_asn}
        network 1.1.1.1 mask 255.255.255.255
        neighbor $rs_ip1 remote-as $rs_asn
        neighbor $rs_ip1 ebgp-multihop 2
        neighbor $rs_ip1 update-source GigabitEthernet2
        neighbor $rs_ip1 timers 5 15
        neighbor $rs_ip2 remote-as $rs_asn
        neighbor $rs_ip2 ebgp-multihop 2
        neighbor $rs_ip2 update-source GigabitEthernet2
        neighbor $rs_ip2 timers 5 15
    ip route 10.0.0.0 255.0.0.0 $hub_csrint_default_gw
    ip route $vnet_prefix_long $hub_csrint_default_gw
    ip route 172.16.0.0 255.240.0.0 $hub_csrint_default_gw
    ip route 192.168.0.0 255.255.0.0 $hub_csrint_default_gw
    ip route $rs_ip1 255.255.255.255 $hub_csrint_default_gw
    ip route $rs_ip2 255.255.255.255 $hub_csrint_default_gw
end
wr mem
EOF

    # Route Server is reachable over ICMP :)
    ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_csr1_ip "ping $rs_ip1"
    # Create peering
    az network routeserver peering create --routeserver $rs_name -g $rg --peer-ip $hub_csr1_bgp_ip --peer-asn $hub_csr_asn -n $hub_csr1_name

    # Show peerings and effective routes on the VM nic (at this point only the 1.1.1.1 should be there)
    az network routeserver peering list --routeserver $rs_name -g $rg -o table
    az network nic show-effective-route-table --ids $hub_vm_nic_id -o table

    # OPTIONAL: Inject default route from CSR1, and configure NAT
    ssh -o BatchMode=yes -o StrictHostKeyChecking=no "$hub_csr1_ip" <<EOF
config t
    interface GigabitEthernet1
        ip nat outside
    interface GigabitEthernet2
        ip nat inside
    access-list 100 remark SOURCES TO BE NATTED
    access-list 100 deny ip $hub_csrext_subnet_prefix_long_reversemask any
    access-list 100 permit ip 10.0.0.0 0.255.255.255 any
    ip nat inside source list 100 interface GigabitEthernet1 overload
    ip route 0.0.0.0 0.0.0.0  $hub_csrext_default_gw
    ip route 0.0.0.0 128.0.0.0  $hub_csrext_default_gw
    ip route 128.0.0.0 128.0.0.0  $hub_csrext_default_gw
    router bgp ${hub_csr_asn}
        network 0.0.0.0 mask 0.0.0.0
        network 0.0.0.0 mask 128.0.0.0
        network 128.0.0.0 mask 128.0.0.0
        default-information originate
end
wr mem
EOF

    # Bypass the NVA for SSH access
    az network route-table create -n hubvmrt -g $rg -l $location
    az network vnet subnet update -g $rg --vnet-name $vnet_name -n $hub_vm_subnet_name --route-table hubvmrt
    myip=$(curl -s4 ifconfig.co) && echo $myip
    az network route-table route create --route-table-name hubvmrt -g $rg --address-prefix "${myip}/32" --name mypc  --next-hop-type Internet

    # Test egress access from the VM
    az network routeserver peering list-learned-routes -n $hub_csr1_name --routeserver $rs_name -g $rg --query 'RouteServiceRole_IN_0' -o table
    az network nic show-effective-route-table --ids $hub_vm_nic_id -o table
    ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no "$hub_vm_ip" "curl -s4 ifconfig.co"

    # Measure route injection time
    # Create route in CSR
    ssh -o BatchMode=yes -o StrictHostKeyChecking=no "$hub_csr1_ip" <<EOF
config t
    interface Loopback 10
        ip address 10.10.10.10 255.255.255.255
    router bgp ${hub_csr_asn}
        network 10.10.10.10 mask 255.255.255.255
end
EOF
    start_time=$(date +%s)
    echo "Interface created in CSR, now waiting for route to appear..."
    route=$(az network nic show-effective-route-table --ids $hub_vm_nic_id -o json | jq -r '.value | map(select(.addressPrefix[0] | contains("10.10.10.10/32"))) | .[]' 2>/dev/null)
    until [[ -n "$route" ]]
    do
        route=$(az network nic show-effective-route-table --ids $hub_vm_nic_id -o json | jq -r '.value | map(select(.addressPrefix[0] | contains("10.10.10.10/32"))) | .[]' 2>/dev/null)
    done
    finish_time=$(date +%s)
    run_time=$(($finish_time - $start_time))
    ((minutes=run_time/60))
    ((seconds=run_time%60))
    echo "Route appeared in the effective route table of the hub VM after $minutes minutes and $seconds seconds"
    # Result: around 20-45s

    # Measure route removal time
    ssh -o BatchMode=yes -o StrictHostKeyChecking=no "$hub_csr1_ip" <<EOF
config t
    no interface Loopback10
    router bgp ${hub_csr_asn}
        no network 10.10.10.10 mask 255.255.255.255
end
EOF
    start_time=$(date +%s)
    echo "Interface deleted in CSR, now waiting for route to disappear..."
    route=$(az network nic show-effective-route-table --ids $hub_vm_nic_id -o json | jq -r '.value | map(select(.addressPrefix[0] | contains("10.10.10.10/32"))) | .[]' 2>/dev/null)
    until [[ -z "$route" ]]
    do
        route=$(az network nic show-effective-route-table --ids $hub_vm_nic_id -o json | jq -r '.value | map(select(.addressPrefix[0] | contains("10.10.10.10/32"))) | .[]' 2>/dev/null)
    done
    finish_time=$(date +%s)
    run_time=$(($finish_time - $start_time))
    ((minutes=run_time/60))
    ((seconds=run_time%60))
    echo "Route disappeared from the effective route table of the hub VM after $minutes minutes and $seconds seconds"
    # Result: around 20-30s

# CSR2
    if [[ "$redundant_nva" == "yes" ]]; then
        # PIP
        az network public-ip create -g $rg -n "${hub_csr2_name}-pip" --sku basic --allocation-method Static
        # NICs
        az network nic create -n "${hub_csr2_name}-nic0" -g $rg --vnet-name $vnet_name --subnet $hub_csrext_subnet_name --network-security-group "$hub_csrext_nsg_name" --public-ip-address "${hub_csr2_name}-pip" --ip-forwarding
        az network nic create -n "${hub_csr2_name}-nic1" -g $rg --vnet-name $vnet_name --subnet $hub_csrint_subnet_name --network-security-group "$hub_csrint_nsg_name" --private-ip-address $hub_csr2_bgp_ip --ip-forwarding
        # VM
        az vm create -n $hub_csr2_name -g $rg -l $location \
            --image ${publisher}:${offer}:${sku}:${version} \
            --admin-username "$csr_username" --admin-password $csr_password --authentication-type all --generate-ssh-keys \
            --nics "${hub_csr2_name}-nic0" "${hub_csr2_name}-nic1"
        # Configure hub CSR2 with BGP
        hub_csr2_ip=$(az network public-ip show -n "${hub_csr2_name}-pip" --query ipAddress -o tsv -g $rg) && echo $hub_csr2_ip
        ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no "$hub_csr2_ip" "sh ip int b"
        ssh -o BatchMode=yes -o StrictHostKeyChecking=no "$hub_csr2_ip" <<EOF
config t
    no ip domain lookup
    interface Loopback0
        ip address 2.2.2.2 255.255.255.255
    interface GigabitEthernet1
        ip nat outside
    interface GigabitEthernet2
        ip nat inside
    access-list 100 remark SOURCES TO BE NATTED
    access-list 100 deny ip $hub_csrext_subnet_prefix_long_reversemask any
    access-list 100 permit ip 10.0.0.0 0.255.255.255 any
    ip nat inside source list 100 interface GigabitEthernet1 overload
    router bgp ${hub_csr_asn}
        default-information originate
        network 0.0.0.0 mask 0.0.0.0
        network 0.0.0.0 mask 128.0.0.0
        network 128.0.0.0 mask 128.0.0.0
        network 2.2.2.2 mask 255.255.255.255
        neighbor $rs_ip1 remote-as $rs_asn
        neighbor $rs_ip1 ebgp-multihop 2
        neighbor $rs_ip1 update-source GigabitEthernet2
        neighbor $rs_ip1 route-map ToRS out
        neighbor $rs_ip1 timers 5 15
        neighbor $rs_ip2 remote-as $rs_asn
        neighbor $rs_ip2 ebgp-multihop 2
        neighbor $rs_ip2 update-source GigabitEthernet2
        neighbor $rs_ip2 route-map ToRS out
        neighbor $rs_ip2 timers 5 15
    route-map ToRS
        set as-path prepend ${hub_csr_asn} ${hub_csr_asn}
    ip route 0.0.0.0 0.0.0.0  $hub_csrext_default_gw
    ip route 0.0.0.0 128.0.0.0  $hub_csrext_default_gw
    ip route 128.0.0.0 128.0.0.0  $hub_csrext_default_gw
    ip route 10.0.0.0 255.0.0.0 $hub_csrint_default_gw
    ip route 172.16.0.0 255.240.0.0 $hub_csrint_default_gw
    ip route 192.168.0.0 255.255.0.0 $hub_csrint_default_gw
    ip route $rs_ip1 255.255.255.255 $hub_csrint_default_gw
    ip route $rs_ip2 255.255.255.255 $hub_csrint_default_gw
end
wr mem
EOF

        # Route Server is reachable over ICMP
        ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_csr2_ip "ping $rs_ip1"
        # Create peering
        az network routeserver peering create --routeserver $rs_name -g $rg --peer-ip $hub_csr2_bgp_ip --peer-asn $hub_csr_asn -n $hub_csr2_name
        # az network routeserver peering delete --routeserver $rs_name -g $rg -n $hub_csr2_name -y -o none  # In case you need to recreate the BGP peering

        # Verify that the RS is learning the routes from both CSRs
        ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_csr1_ip "sh ip bgp summ"
        ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_csr2_ip "sh ip bgp summ"
        az network routeserver peering list --routeserver $rs_name -g $rg -o table
        az network routeserver peering list-learned-routes -n $hub_csr1_name --routeserver $rs_name -g $rg --query 'RouteServiceRole_IN_0' -o table
        az network routeserver peering list-learned-routes -n $hub_csr2_name --routeserver $rs_name -g $rg --query 'RouteServiceRole_IN_0' -o table
        az network nic show-effective-route-table --ids $hub_vm_nic_id -o table

        # Simulate outage of CSR1 and measure time for CSR2 to take over (looking at the effective routes)
        function bring_down_csr1() {
          ssh -o BatchMode=yes -o StrictHostKeyChecking=no "$hub_csr1_ip" <<EOF
          config t
            int GigabitEthernet2
            shutdown
        EOF
          start_time=$(date +%s)
          echo "Internal interface shutdown in CSR1, now waiting for the effective route to switch to CSR2 ($hub_csr2_bgp_ip)..."
          nexthop=$(az network nic show-effective-route-table --ids $hub_vm_nic_id -o json | jq -r '.value | map(select(.addressPrefix[0] | contains("0.0.0.0/0"))) | .[].nextHopIpAddress[]')  && echo $nexthop
          until [[ "$nexthop" == "$hub_csr2_bgp_ip" ]]
          do
            nexthop=$(az network nic show-effective-route-table --ids $hub_vm_nic_id -o json | jq -r '.value | map(select(.addressPrefix[0] | contains("0.0.0.0/0"))) | .[].nextHopIpAddress[]' 2>/dev/null)
            echo "Next hop for 0.0.0.0/0 is $nexthop"
          done
          finish_time=$(date +%s)
          run_time=$(($finish_time - $start_time))
          ((minutes=run_time/60))
          ((seconds=run_time%60))
          echo "Effective route pointing now to $nexthop after $minutes minutes and $seconds seconds"
        }

        # Restore CSR1 connectivity
        function bring_up_csr1() {
          ssh -o BatchMode=yes -o StrictHostKeyChecking=no "$hub_csr1_ip" <<EOF
          config t
            int GigabitEthernet2
            no shutdown
        EOF
          start_time=$(date +%s)
          echo "Internal interface restored in CSR1, now waiting for the effective route to switch back to CSR1 ($hub_csr1_bgp_ip)..."
          nexthop=$(az network nic show-effective-route-table --ids $hub_vm_nic_id -o json | jq -r '.value | map(select(.addressPrefix[0] | contains("0.0.0.0/0"))) | .[].nextHopIpAddress[]' 2>/dev/null)
          until [[ "$nexthop" == "$hub_csr1_bgp_ip" ]]
          do
            nexthop=$(az network nic show-effective-route-table --ids $hub_vm_nic_id -o json | jq -r '.value | map(select(.addressPrefix[0] | contains("0.0.0.0/0"))) | .[].nextHopIpAddress[]' 2>/dev/null)
            echo "Next hop for 0.0.0.0/0 is $nexthop"
          done
          finish_time=$(date +%s)
          run_time=$(($finish_time - $start_time))
          ((minutes=run_time/60))
          ((seconds=run_time%60))
          echo "Effective route pointing now to $nexthop after $minutes minutes and $seconds seconds"
        }

        bring_down_csr1
        bring_up_csr1
    fi
fi

################
#    Spokes    #
################

# Create RT to bypass the NVA for SSH connections from the local PC
az network route-table create -n spokevmrt -g $rg -l $location
myip=$(curl -s4 ifconfig.co) && echo $myip
az network route-table route create --route-table-name spokevmrt -g $rg --address-prefix "${myip}/32" --name mypc  --next-hop-type Internet

# Spoke1 with Ubuntu VM
az network vnet create -g $rg -n $spoke1_name --address-prefix $spoke1_prefix --subnet-name $spoke1_vm_subnet_name --subnet-prefix $spoke1_vm_subnet_prefix -l $location
az network vnet subnet create -n $spoke1_csr_subnet_name --address-prefix $spoke1_csr_subnet_prefix --vnet-name $spoke1_name -g $rg
az network vnet subnet update -g $rg --vnet-name $spoke1_name -n $spoke1_vm_subnet_name --route-table spokevmrt
# az network vnet peering delete -n hubtospoke1 -g $rg --vnet-name $vnet_name
# az network vnet peering delete -n spoke1tohub -g $rg --vnet-name $spoke1_name
az network vnet peering create -n hubtospoke1 -g $rg --vnet-name $vnet_name --remote-vnet $spoke1_name --allow-vnet-access --allow-forwarded-traffic --allow-gateway-transit
az network vnet peering create -n spoke1tohub -g $rg --vnet-name $spoke1_name --remote-vnet $vnet_name --allow-vnet-access --allow-forwarded-traffic --use-remote-gateways
az vm create -n spoke1-vm -g $rg -l $location --image ubuntuLTS --generate-ssh-keys \
    --public-ip-address spoke1-vm-pip --vnet-name $spoke1_name --size Standard_B1s --subnet $spoke1_vm_subnet_name
spoke1_ip=$(az network public-ip show -n spoke1-vm-pip --query ipAddress -o tsv -g $rg) && echo $spoke1_ip
spoke1_nic_id=$(az vm show -n spoke1-vm -g "$rg" --query 'networkProfile.networkInterfaces[0].id' -o tsv)
spoke1_private_ip=$(az network nic show --ids $spoke1_nic_id --query 'ipConfigurations[0].privateIpAddress' -o tsv) && echo $spoke1_private_ip

# Spoke 2 with Ubuntu VM
az network vnet create -g $rg -n $spoke2_name --address-prefix $spoke2_prefix --subnet-name $spoke2_vm_subnet_name --subnet-prefix $spoke2_vm_subnet_prefix -l $location
az network vnet subnet update -g $rg --vnet-name $spoke2_name -n $spoke2_vm_subnet_name --route-table spokevmrt
az network vnet peering create -n hubtospoke2 -g $rg --vnet-name $vnet_name --remote-vnet $spoke2_name --allow-vnet-access --allow-forwarded-traffic --allow-gateway-transit
az network vnet peering create -n spoke2tohub -g $rg --vnet-name $spoke2_name --remote-vnet $vnet_name --allow-vnet-access --allow-forwarded-traffic --use-remote-gateways
az vm create -n spoke2-vm -g $rg -l $location --image ubuntuLTS --generate-ssh-keys \
    --public-ip-address spoke2-vm-pip --vnet-name $spoke2_name --size Standard_B1s --subnet $spoke2_vm_subnet_name
spoke2_ip=$(az network public-ip show -n spoke2-vm-pip --query ipAddress -o tsv -g $rg) && echo $spoke2_ip
spoke2_nic_id=$(az vm show -n spoke2-vm -g "$rg" --query 'networkProfile.networkInterfaces[0].id' -o tsv)
spoke2_private_ip=$(az network nic show --ids $spoke2_nic_id --query 'ipConfigurations[0].privateIpAddress' -o tsv) && echo $spoke2_private_ip

# Effective routes
az network nic show-effective-route-table --ids $spoke1_nic_id -o table
az network nic show-effective-route-table --ids $spoke2_nic_id -o table

# Check routing on NVA
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_csr1_ip "show ip route static"

# Connectivity from spoke to NVA (1.1.1.1 is a loopback on CSR1, advertised to ARS via BGP)
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $spoke1_ip "ping 1.1.1.1 -c 5"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $spoke2_ip "ping 1.1.1.1 -c 5"

# Verify spoke-to-spoke connectivity
# ICMP
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $spoke1_ip "ping $spoke2_private_ip -c 5"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $spoke2_ip "ping $spoke1_private_ip -c 5"
# SSH from spoke to spoke (using one of them as jump host)
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no -J $spoke1_ip $spoke2_private_ip "ip a"

# CSR in spoke1 (using the hub NSG, I am lazy...)
az network route-table create -n spoke1csr -g $rg -l $location --disable-bgp-route-propagation
az network vnet subnet update -g $rg --vnet-name $spoke1_name -n $spoke1_csr_subnet_name --route-table spoke1csr  # To prevent ARS routes to be injected here
az network public-ip create -g $rg -n "${spoke1_csr_name}-pip" --sku basic --allocation-method Static
az network nic create -n "${spoke1_csr_name}-nic0" -g $rg --vnet-name $spoke1_name --subnet $spoke1_csr_subnet_name --public-ip-address "${spoke1_csr_name}-pip" --network-security-group "$hub_csrext_nsg_name" --ip-forwarding
az vm create -n $spoke1_csr_name -g $rg -l $location \
    --image ${publisher}:${offer}:${sku}:${version} \
    --admin-username "$csr_username" --admin-password $csr_password --authentication-type all --generate-ssh-keys \
    --nics "${spoke1_csr_name}-nic0"
# Configure spoke1 CSR with BGP
spoke1_csr_ip=$(az network public-ip show -n "${spoke1_csr_name}-pip" --query ipAddress -o tsv -g $rg) && echo $spoke1_csr_ip
spoke1_csr_default_gw=$(first_ip $spoke1_csr_subnet_prefix) && echo $spoke1_csr_default_gw
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no "$spoke1_csr_ip" "sh ip int b"
ssh -o BatchMode=yes -o StrictHostKeyChecking=no "$spoke1_csr_ip" <<EOF
config t
    no ip domain lookup
    interface Loopback0
        ip address 22.22.22.22 255.255.255.255
    router bgp ${spoke1_csr_asn}
        network 22.22.22.22 mask 255.255.255.255
        neighbor $rs_ip1 remote-as $rs_asn
        neighbor $rs_ip1 ebgp-multihop 2
        neighbor $rs_ip1 update-source GigabitEthernet1
        neighbor $rs_ip1 timers 5 15
        neighbor $rs_ip2 remote-as $rs_asn
        neighbor $rs_ip2 ebgp-multihop 2
        neighbor $rs_ip2 update-source GigabitEthernet1
        neighbor $rs_ip2 timers 5 15
    ip route $rs_ip1 255.255.255.255 $hub_csrint_default_gw
    ip route $rs_ip2 255.255.255.255 $hub_csrint_default_gw
end
wr mem
EOF
# Add BGP peering to ARS
spoke1_csr_private_ip=$(az network nic show -n "${spoke1_csr_name}-nic0" -g $rg  --query 'ipConfigurations[0].privateIpAddress' -o tsv) && echo $spoke1_csr_private_ip
az network routeserver peering create --routeserver $rs_name -g $rg --peer-ip $spoke1_csr_private_ip --peer-asn $spoke1_csr_asn -n $spoke1_csr_name

# Since the NVA is in spoke1, you want to vnet-peer spoke1 and spoke2, otherwise the effective route in spoke2 would show up as none
az network vnet peering create -n spoke2tospoke1 -g $rg --vnet-name $spoke2_name --remote-vnet $spoke1_name --allow-vnet-access --allow-forwarded-traffic
az network vnet peering create -n spoke1tospoke1 -g $rg --vnet-name $spoke1_name --remote-vnet $spoke2_name --allow-vnet-access --allow-forwarded-traffic


#############
# Linux NVA #
#############

# Cloud init file to install StrongSwan (not used in this lab) and BIRD 1.6
cat <<EOF > $hub_linuxnva_cloudinit_file
#cloud-config
runcmd:
  - apt update && apt install -y bird strongswan libcharon-extra-plugins strongswan-pki
  - sysctl -w net.ipv4.ip_forward=1
  - sudo iptables -t nat -A POSTROUTING ! -d '10.0.0.0/8' -o eth0 -j MASQUERADE
EOF
az network vnet subnet create --vnet-name $vnet_name --name $hub_linuxnva_subnet_name -g $rg --address-prefixes $hub_linuxnva_subnet_prefix
az network route-table create -n hublinuxnva -g $rg -l $location --disable-bgp-route-propagation
az network vnet subnet update -g $rg --vnet-name $vnet_name -n $hub_linuxnva_subnet_name --route-table hublinuxnva
az vm create -n $hub_linuxnva_name -g $rg -l $location --image ubuntuLTS --generate-ssh-keys \
    --public-ip-address $hub_linuxnva_pip_name --vnet-name $vnet_name --size Standard_B1s --subnet $hub_linuxnva_subnet_name \
    --custom-data $hub_linuxnva_cloudinit_file 
hub_linuxnva_nic_id=$(az vm show -n $hub_linuxnva_name -g "$rg" --query 'networkProfile.networkInterfaces[0].id' -o tsv)
az network nic update --ids $hub_linuxnva_nic_id --ip-forwarding
hub_linuxnva_pip_ip=$(az network public-ip show -n $hub_linuxnva_pip_name -g $rg --query ipAddress -o tsv) && echo $hub_linuxnva_pip_ip
hub_linuxnva_private_ip=$(az network nic show --ids $hub_linuxnva_nic_id --query 'ipConfigurations[0].privateIpAddress' -o tsv) && echo $hub_linuxnva_private_ip
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "ip a"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "systemctl status bird"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "sudo birdc show route"
hub_linuxnva_default_gw=$(ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip netstat -rnv | awk '$1 == "0.0.0.0" {print $2}') && echo $hub_linuxnva_default_gw

# RS adjacency
az network routeserver peering create --routeserver $rs_name -g $rg --peer-ip $hub_linuxnva_private_ip --peer-asn $hub_linuxnva_asn -n $hub_linuxnva_name

# Bird config file
bird_config_file=/tmp/bird.conf
cat <<EOF > $bird_config_file
log syslog all;
router id $hub_linuxnva_private_ip;
protocol device {
        scan time 10;           # Scan interfaces every 10 seconds
}
# Disable automatically generating direct routes to all network interfaces.
protocol direct {
      #disabled;               # Enabled by default
}
# Forbid synchronizing BIRD routing tables with the OS kernel.
protocol kernel {
      import all;       # Import to table, default is import all
      export none;      # Export to protocol. default is export none
}
# Static IPv4 routes.
protocol static {
      route 5.5.5.5/32 via $hub_linuxnva_default_gw;
      route 0.0.0.0/1 via $hub_linuxnva_default_gw;
      route 128.0.0.0/1 via $hub_linuxnva_default_gw;
      # route $vnet_prefix via $hub_linuxnva_default_gw;
}
# BGP peers
protocol bgp rs0 {
      description "RouteServer instance 0";
      multihop;
      local $hub_linuxnva_private_ip as $hub_linuxnva_asn;
      neighbor $rs_ip1 as $rs_asn;
          import filter {accept;};
          export filter {accept;};
}
protocol bgp rs1 {
      description "RouteServer instance 1";
      multihop;
      local $hub_linuxnva_private_ip as $hub_linuxnva_asn;
      neighbor $rs_ip2 as $rs_asn;
          import filter {accept;};
          export filter {accept;};
}
EOF
hub_linuxnva_user=$(ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "whoami") && echo $hub_linuxnva_user
scp $bird_config_file "${hub_linuxnva_pip_ip}:/home/${hub_linuxnva_user}/bird.conf"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "sudo mv /home/${hub_linuxnva_user}/bird.conf /etc/bird/bird.conf"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "sudo systemctl restart bird"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "systemctl status bird"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "sudo birdc show status"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "sudo birdc show route"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "sudo birdc show protocols all"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "sudo birdc show protocols rs0"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "sudo birdc show protocols rs1"

# Effective routes in a spoke (5.5.5.5/32 should be there)
az network nic show-effective-route-table --ids $spoke1_nic_id -o table

# Routes advertised to ARS
az network routeserver peering list-learned-routes -n $hub_linuxnva_name --routeserver $rs_name -g $rg --query 'RouteServiceRole_IN_0' -o table


#################################
# Onprem simulation with VPN/ER #
#################################

# Initialization
az network vnet create -n $onprem_vnet_name -g $rg --address-prefixes $onprem_vnet_prefix --subnet-name $onprem_nva_subnet_name --subnet-prefixes $onprem_nva_subnet_prefix
az network vnet subnet create -g $rg --vnet-name $onprem_vnet_name -n GatewaySubnet --address-prefix $onprem_gw_subnet_prefix
az network route-table create -n onpremnva -g $rg -l $location --disable-bgp-route-propagation
az network vnet subnet update -g $rg --vnet-name $onprem_vnet_name -n $onprem_nva_subnet_name --route-table onpremnva

# OPTIONAL: Create ER and ER Gateway to simulate onprem
ergw_name=onpremergw
ergw_pip="${ergw_name}-pip"
er_location=germanywestcentral
er_pop=Frankfurt
er_provider=Megaport
er_circuit_name=onpremer
az network public-ip create -g $rg -n $ergw_pip --allocation-method Dynamic --sku Basic
az network vnet-gateway create -g $rg -n $ergw_name --gateway-type ExpressRoute --sku Standard -l $location \
    --vnet $onprem_vnet_name --public-ip-addresses $ergw_pip --no-wait
az network express-route create -n $er_circuit_name --peering-location $er_pop -g $rg \
    --bandwidth 50 --provider $er_provider -l $er_location --sku-family MeteredData --sku-tier Standard
# After provisioning circuit in Megaport portal continue
circuit_id=$(az network express-route show -n $er_circuit_name -g $rg -o tsv --query id) && echo $circuit_id
az network vpn-connection create -n erconnection -g $rg --vnet-gateway1 $ergw_name --express-route-circuit2 $circuit_id

# Create onprem Linux NVA
onprem_linuxnva_cloudinit_file=/tmp/linuxnva_cloudinit.txt
cat <<EOF > $onprem_linuxnva_cloudinit_file
#cloud-config
runcmd:
  - apt update && apt install -y bird strongswan
  - sysctl -w net.ipv4.ip_forward=1
  - sysctl -w net.ipv4.conf.all.accept_redirects = 0 
  - sysctl -w net.ipv4.conf.all.send_redirects = 0
EOF
az vm create -n $onprem_linuxnva_name -g $rg -l $location --image ubuntuLTS --generate-ssh-keys \
    --public-ip-address $onprem_linuxnva_pip --vnet-name $onprem_vnet_name --size Standard_B1s --subnet $onprem_nva_subnet_name \
    --custom-data $onprem_linuxnva_cloudinit_file --private-ip-address "$onprem_linuxnva_ip"
onprem_linuxnva_nic_id=$(az vm show -n $onprem_linuxnva_name -g "$rg" --query 'networkProfile.networkInterfaces[0].id' -o tsv)
az network nic update --ids $onprem_linuxnva_nic_id --ip-forwarding
onprem_linuxnva_pip_ip=$(az network public-ip show -n $onprem_linuxnva_pip -g $rg --query ipAddress -o tsv) && echo $onprem_linuxnva_pip_ip
onprem_linuxnva_private_ip=$(az network nic show --ids $onprem_linuxnva_nic_id --query 'ipConfigurations[0].privateIpAddress' -o tsv) && echo $onprem_linuxnva_private_ip
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "ip a"
onprem_linuxnva_default_gw=$(ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip netstat -rnv | awk '$1 == "0.0.0.0" {print $2}') && echo $onprem_linuxnva_default_gw
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "systemctl status bird"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "sudo birdc show route"

# Configure StrongSwan VPN to the hub Linux NVA
# See https://blog.sys4.de/routing-based-vpn-with-strongswan-de.html
# See https://wiki.strongswan.org/projects/strongswan/wiki/RouteBasedVPN
echo "Configuring VPN between A:${linuxnva_pip_ip}/${linuxnva_private_ip} and B:${onprem_linuxnva_pip_ip}/${onprem_linuxnva_private_ip}"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "sudo ip tunnel add vti0 local $hub_linuxnva_private_ip remote $onprem_linuxnva_pip_ip mode vti key 12"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "sudo sysctl -w net.ipv4.conf.vti0.disable_policy=1"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "sudo ip link set up dev vti0"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "sudo ip route add $onprem_linuxnva_private_ip/32 dev vti0"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "sudo sed -i 's/# install_routes = yes/install_routes = no/' /etc/strongswan.d/charon.conf"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "sudo ip tunnel add vti0 local $onprem_linuxnva_private_ip remote $hub_linuxnva_pip_ip mode vti key 12"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "sudo sysctl -w net.ipv4.conf.vti0.disable_policy=1"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "sudo ip link set up dev vti0"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "sudo ip route add $hub_linuxnva_private_ip/32 dev vti0"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "sudo sed -i 's/# install_routes = yes/install_routes = no/' /etc/strongswan.d/charon.conf"
vpn_psk=$(openssl rand -base64 64)
vpn_psk=${vpn_psk//$'\n'/}  # Remove line breaks
psk_file_a=/tmp/ipsec.secrets.a
psk_file_b=/tmp/ipsec.secrets.b
cat <<EOF > $psk_file_a
$hub_linuxnva_pip_ip $onprem_linuxnva_pip_ip : PSK "$vpn_psk"
EOF
cat <<EOF > $psk_file_b
$onprem_linuxnva_pip_ip $hub_linuxnva_pip_ip : PSK "$vpn_psk"
EOF
ipsec_file_a=/tmp/ipsec.conf.a
ipsec_file_b=/tmp/ipsec.conf.b
cat <<EOF > $ipsec_file_a
config setup
        charondebug="all"
        uniqueids=yes
        strictcrlpolicy=no
conn to-onprem
  authby=secret
  leftid=$hub_linuxnva_pip_ip
  leftsubnet=0.0.0.0/0
  right=$onprem_linuxnva_pip_ip
  rightsubnet=0.0.0.0/0
  ike=aes256-sha2_256-modp1024!
  esp=aes256-sha2_256!
  keyingtries=0
  ikelifetime=1h
  lifetime=8h
  dpddelay=30
  dpdtimeout=120
  dpdaction=restart
  auto=start
  mark=12
EOF
cat <<EOF > $ipsec_file_b
config setup
        charondebug="all"
        uniqueids=yes
        strictcrlpolicy=no
conn to-azure
  authby=secret
  leftid=$onprem_linuxnva_pip_ip
  leftsubnet=0.0.0.0/0
  right=$hub_linuxnva_pip_ip
  rightsubnet=0.0.0.0/0
  ike=aes256-sha2_256-modp1024!
  esp=aes256-sha2_256!
  keyingtries=0
  ikelifetime=1h
  lifetime=8h
  dpddelay=30
  dpdtimeout=120
  dpdaction=restart
  auto=start
  mark=12
EOF
username=$(whoami)
scp $psk_file_a $hub_linuxnva_pip_ip:/home/$username/ipsec.secrets
scp $psk_file_b $onprem_linuxnva_pip_ip:/home/$username/ipsec.secrets
scp $ipsec_file_a $hub_linuxnva_pip_ip:/home/$username/ipsec.conf
scp $ipsec_file_b $onprem_linuxnva_pip_ip:/home/$username/ipsec.conf
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "sudo mv ./ipsec.* /etc/"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "sudo systemctl restart ipsec"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "sudo mv ./ipsec.* /etc/"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "sudo systemctl restart ipsec"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "systemctl status ipsec"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "systemctl status ipsec"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "sudo ipsec status"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "sudo ipsec status"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "ping $onprem_linuxnva_private_ip -c 5"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "ping $hub_linuxnva_private_ip -c 5"

# Configure BGP onprem with Bird
bird_config_file=/tmp/bird.conf
cat <<EOF > $bird_config_file
log syslog all;
router id $onprem_linuxnva_private_ip;
protocol device {
        scan time 10;
}
protocol direct {
      #disabled;
}
protocol kernel {
      import all;
      export none;
}
protocol static {
      route 6.6.6.6/32 via $onprem_linuxnva_default_gw;
      route $onprem_vnet_prefix via $onprem_linuxnva_default_gw;
}
protocol bgp azurenva {
      description "BGP to Azure";
      multihop;
      local $onprem_linuxnva_private_ip as $onprem_linuxnva_asn;
      neighbor $hub_linuxnva_private_ip as $hub_linuxnva_asn;
          import filter {accept;};
          export filter {accept;};
}
EOF
scp $bird_config_file "${onprem_linuxnva_pip_ip}:/home/${hub_linuxnva_user}/bird.conf"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "sudo mv /home/${hub_linuxnva_user}/bird.conf /etc/bird/bird.conf"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "sudo systemctl restart bird"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "systemctl status bird"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "sudo birdc show status"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "sudo birdc show protocols"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "sudo birdc show route"

# Configure BGP on Azure
bird_config_file=/tmp/bird.conf
cat <<EOF > $bird_config_file
protocol bgp onprem {
      description "BGP to Onprem";
      multihop;
      local $hub_linuxnva_private_ip as $hub_linuxnva_asn;
      neighbor $onprem_linuxnva_private_ip as $onprem_linuxnva_asn;
          import filter {accept;};
          export filter {accept;};
}
EOF
scp $bird_config_file "${hub_linuxnva_pip_ip}:/home/${username}/bird.conf.new"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "cat /home/${hub_linuxnva_user}/bird.conf.new"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "sudo cat /etc/bird/bird.conf /home/${hub_linuxnva_user}/bird.conf.new >/home/${hub_linuxnva_user}/bird.conf"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "sudo mv /home/${hub_linuxnva_user}/bird.conf /etc/bird/bird.conf"
# ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "sudo more /etc/bird/bird.conf"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "sudo systemctl restart bird"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "systemctl status bird"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "sudo birdc show status"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "sudo birdc show protocols"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "sudo birdc show route"

# Check effective routes (6.6.6.6 should be there)
az network nic show-effective-route-table --ids $hub_vm_nic_id -o table
az network nic show-effective-route-table --ids $spoke1_vm_nic_id -o table
$spoke1_

# Ping onprem's IP
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $spoke1_ip "ping $onprem_linuxnva_private_ip -c 5"

# Create onprem CSR (WORK IN PROGRESS)
nva_size=Standard_B2ms
onprem_csr_ip=192.168.0.10
publisher=cisco
offer=cisco-csr-1000v
sku=16_12-byol
version=$(az vm image list -p $publisher -f $offer -s $sku --all --query '[0].version' -o tsv 2>/dev/null)
az vm create -n onprem-csr -g "$rg" -l "$location" --image "${publisher}:${offer}:${sku}:${version}" --size "$nva_size" \
    --generate-ssh-keys --public-ip-address "onprem-csr-pip" --public-ip-address-allocation static --public-ip-sku Standard \
    --vnet-name "$onprem_vnet_name" --subnet "$onprem_nva_subnet_name" \
    --private-ip-address "$onprem_csr_ip"

# Configure onprem CSR for VPN
onprem_default_gw=192.168.0.1
vpngw_pip_ip=$(az network public-ip show -n $vpngw_pip -g $rg --query ipAddress -o tsv) && echo $vpngw_pip_ip
vpngw_bgp_ip=$(az network vnet-gateway show -n $vpngw_name -g $rg --query bgpSettings.bgpPeeringAddresses[0].tunnelIpAddresses[0]) && echo $vpngw_bgp_ip
vpngw_bgp_asn=$(az network vnet-gateway show -n $vpngw_name -g $rg --query bgpSettings.asn) && echo $vpngw_bgp_asn
onprem_asn=65200
ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o KexAlgorithms=+diffie-hellman-group14-sha1 "$onprem_csr_ip"
    config t
      crypto ikev2 proposal azure-proposal
        encryption aes-cbc-256 aes-cbc-128 3des
        integrity sha1
        group 2
      crypto ikev2 policy azure-policy
        proposal azure-proposal
      crypto ikev2 profile azure-profile
        match address local interface GigabitEthernet1
        authentication remote pre-share
        authentication local pre-share
        keyring local azure-keyring
      crypto ipsec transform-set azure-ipsec-proposal-set esp-aes 256 esp-sha-hmac
        mode tunnel
      crypto ipsec profile azure-vti
        set security-association lifetime kilobytes 102400000
        set transform-set azure-ipsec-proposal-set
        set ikev2-profile azure-profile
      crypto isakmp policy 1
        encr aes
        authentication pre-share
        group 14
      crypto ipsec transform-set csr-ts esp-aes esp-sha-hmac
        mode tunnel
      crypto ikev2 keyring azure-keyring
        peer ${vpngw_pip_ip}
          address ${vpngw_pip_ip}
          pre-shared-key ${ipsec_psk}
      crypto ikev2 profile azure-profile
        match identity remote address ${vpngw_pip_ip} 255.255.255.255
      crypto isakmp key ${psk} address ${vpngw_pip_ip}
      interface Tunnel0
        ip unnumbered GigabitEthernet1
        ip tcp adjust-mss 1350
        tunnel source GigabitEthernet1
        tunnel mode ipsec ipv4
        tunnel destination ${vpngw_pip_ip}
        tunnel protection ipsec profile azure-vti
      router bgp ${onprem_asn}
        bgp router-id interface GigabitEthernet1
        bgp log-neighbor-changes
        redistribute ospf 100 route-map O2B
        redistribute static route-map S2B
        maximum-paths eibgp 4
        neighbor ${vpngw_bgp_ip} remote-as ${vpngw_bgp_asn}
        neighbor ${vpngw_bgp_ip} update-source GigabitEthernet1
        neighbor ${vpngw_bgp_ip} ebgp-multihop 5
      ip route ${vpngw_bgp_ip} 255.255.255.255 Tunnel0
      ip route ${vpngw_pip_ip} 255.255.255.255 ${onprem_default_gw}
    end
    wr mem
EOF
# Create LNG, connection
az network local-gateway create -g $rg -n onprem --gateway-ip-address $onprem_csr_ip --local-address-prefixes "$onprem_csr_ip\32" \
  --bgp-peering-address $onprem_csr_ip --asn ${onprem_asn}
az network vpn-connection create -g $rg --shared-key $psk -n onprem --vnet-gateway1 $vpngw_name --local-gateway2 onprem 

################
# ExpressRoute #
################

# Create ER and ER Gateway
az network public-ip create -g $rg -n $ergw_pip --allocation-method Dynamic --sku Basic
az network vnet subnet create -g $rg --vnet-name $vnet_name -n GatewaySubnet --address-prefix $gw_subnet_prefix
az network vnet-gateway create -g $rg -n $ergw_name --gateway-type ExpressRoute --sku Standard -l $location --vnet $vnet_name --public-ip-addresses $ergw_pip --no-wait
az network express-route create -n $er_circuit_name --peering-location $er_pop -g $rg \
    --bandwidth 50 --provider $er_provider -l $er_location --sku-family MeteredData --sku-tier Standard
# After provisioning circuit in Megaport portal continue
circuit_id=$(az network express-route show -n $er_circuit_name -g $rg -o tsv --query id) && echo $circuit_id
az network vpn-connection create -n erconnection2 -g $rg --vnet-gateway1 $ergw_name --express-route-circuit2 $circuit_id

# Configure Route Server to exchange routes with Vnet Gateways
az network routeserver update -n $rs_name -g $rg --set allowBranchToBranchTraffic=true

# BGP neighbors of ER gateway should show the two Route Server instances
az network vnet-gateway list-bgp-peer-status -n $ergw_name -g $rg -o table

#######
# VPN #
#######

# Azure VPN Gateway gets created in Failed state if deployed AFTER the ARS (this might be fixed now)

# Create VPN Gateway
az network vnet subnet create -g $rg --vnet-name $vnet_name -n GatewaySubnet --address-prefix $gw_subnet_prefix
az network public-ip create -g $rg -n "${vpngw_pip}-a" --allocation-method Dynamic --sku Basic
az network public-ip create -g $rg -n "${vpngw_pip}-b" --allocation-method Dynamic --sku Basic
az network vnet-gateway create -g "$rg" --sku VpnGw1 --gateway-type Vpn --vpn-type RouteBased \
            --vnet "$vnet_name" -n $vpngw_name --asn "$vpngw_asn" --public-ip-addresses "${vpngw_pip}-a" "${vpngw_pip}-b" --no-wait
az network vnet-gateway list -g $rg -o table

# Inspect learnt routes
az network vnet-gateway list-learned-routes -n $vpngw_name -g $rg --query 'value[].{LocalAddress:localAddress, Peer:sourcePeer, Network:network, NextHop:nextHop, ASPath: asPath, Origin:origin, Weight:weight}' -o tsv

###############
# Diagnostics #
###############

az network routeserver list -g $rg -o table
az network routeserver show -n $rs_name -g $rg
az network routeserver peering list --routeserver $rs_name -g $rg -o table
az network routeserver peering show -n csr --routeserver $rs_name -g $rg

##########################
# REST: WORK IN PROGRESS #
##########################

# REST: routeserver
subscription=$(az account show --query id -o tsv)
version=2020-07-01
url="https://management.azure.com/subscriptions/${subscription}/resourceGroups/${rg}/providers/Microsoft.Network/virtualRouters/${rs_name}?api-version=${version}"
url="https://management.azure.com/subscriptions/${subscription}/resourceGroups/${rg}/providers/Microsoft.Network/virtualHubs/${rs_name}?api-version=${version}"
az rest --method get --url $url

# REST: peering
subscription=$(az account show --query id -o tsv)
version=2020-07-01
url="https://management.azure.com/subscriptions/${subscription}/resourceGroups/${rg}/providers/Microsoft.Network/virtualRouters/${rs_name}/peerings/csr?api-version=${version}"
az rest --method get --url $rl