-
Notifications
You must be signed in to change notification settings - Fork 34
/
Copy pathagicv2.azcli
267 lines (233 loc) · 11.2 KB
/
agicv2.azcli
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
# AGICv2 (aka kubic, aka traffic controller, aka AppGW for containers, aka AGC)
# There are 2 steps to create AGC:
# 1. Create ALB Controller: https://learn.microsoft.com/en-us/azure/application-gateway/for-containers/quickstart-deploy-application-gateway-for-containers-alb-controller?tabs=install-helm-windows
# 2. Create AGC
# Option a - Managed: https://learn.microsoft.com/en-us/azure/application-gateway/for-containers/quickstart-create-application-gateway-for-containers-managed-by-alb-controller?tabs=new-subnet-aks-vnet
# Option b - BYOD: https://learn.microsoft.com/en-us/azure/application-gateway/for-containers/quickstart-create-application-gateway-for-containers-byo-deployment?tabs=existing-vnet-subnet
# Variables
rg=agc
location=eastus2
aks_name=agicv2
aks_node_size=Standard_B2ms
vnet_name=aksVnet
vnet_prefix=10.13.0.0/16
aks_subnet_name=aks1stpool
aks_subnet_prefix=10.13.76.0/24 # Min /25 with Azure CNI!
pod_subnet_name=pods
pod_subnet_prefix=10.13.80.0/24
aks_service_cidr=10.0.0.0/16
aks_id_name=aksid
alb_id_name=albid
alb_ns_name=azure-alb-test
agfc_name=appgw4c
agfc_frontend_name=test-frontend
agfc_subnet_name=subnet-alb
agfc_subnet_prefix=10.13.100.0/24
# Register required resource providers on Azure.
az provider register --namespace Microsoft.ContainerService
az provider register --namespace Microsoft.Network
az provider register --namespace Microsoft.NetworkFunction
az provider register --namespace Microsoft.ServiceNetworking
# Install Azure CLI extensions.
az extension add --name alb
# Create RG and VNets
az group create -n $rg -l $location -o none --only-show-errors
az network vnet create -g $rg -n $vnet_name --address-prefix $vnet_prefix -l $location -o none
az network vnet subnet create -g $rg -n $aks_subnet_name --vnet-name $vnet_name --address-prefix $aks_subnet_prefix -o none --only-show-errors
az network vnet subnet create -g $rg -n $pod_subnet_name --vnet-name $vnet_name --address-prefix $pod_subnet_prefix -o none --only-show-errors
aks_subnet_id=$(az network vnet subnet show -n $aks_subnet_name --vnet-name $vnet_name -g $rg --query id -o tsv)
pod_subnet_id=$(az network vnet subnet show -n $pod_subnet_name --vnet-name $vnet_name -g $rg --query id -o tsv)
# Create identity for AKS
id_id=$(az identity show -n $aks_id_name -g $rg --query id -o tsv 2>/dev/null)
if [[ -z "$id_id" ]]
then
echo "Identity $aks_id_name not found, creating a new one..."
az identity create -n $aks_id_name -g $rg -o none
id_id=$(az identity show -n $aks_id_name -g $rg --query id -o tsv)
else
echo "Identity $aks_id_name found with ID $id_id"
fi
id_principal_id=$(az identity show -n $aks_id_name -g $rg --query principalId -o tsv)
vnet_id=$(az network vnet show -n $vnet_name -g $rg --query id -o tsv)
sleep 15 # Time for creation to propagate
az role assignment create --scope $vnet_id --assignee $id_principal_id --role Contributor -o none
# Create AKS cluster with workload identity enabled.
az aks create -g $rg -n $aks_name -l $location -o none \
--enable-oidc-issuer --enable-workload-identity \
-c 1 -s $aks_node_size --generate-ssh-keys -u $(whoami) \
--enable-managed-identity --assign-identity $id_id \
--network-plugin azure --vnet-subnet-id $aks_subnet_id --pod-subnet-id $pod_subnet_id --service-cidr $aks_service_cidr \
--network-policy azure \
--load-balancer-sku Standard \
--node-resource-group "$aks_name"-iaas-"$RANDOM"
# For existing clusters
# az aks update -g $rg -n $aks_name --enable-oidc-issuer --enable-workload-identity -o none
# Create identity for AGC
node_rg=$(az aks show -n $aks_name -g $rg --query "nodeResourceGroup" -o tsv)
node_rg_id=$(az group show -n $node_rg --query id -o tsv)
echo "Creating identity $alb_id_name in resource group $rg"
az identity create -n $alb_id_name -g $rg -o none --only-show-errors
alb_id_principal_id="$(az identity show -n $alb_id_name -g $rg --query principalId -o tsv)"
echo "Waiting 15 seconds to allow for replication of the identity..."
sleep 15
echo "Apply Reader role to the AKS managed cluster resource group for the newly provisioned identity"
reader_role=acdd72a7-3385-48ef-bd42-f606fba81ae7
az role assignment create --assignee-object-id $alb_id_principal_id --role $reader_role --scope $node_rg_id -o none --only-show-errors
echo "Set up federation with AKS OIDC issuer"
aks_oidc_issuer="$(az aks show -n "$aks_name" -g "$rg" --query "oidcIssuerProfile.issuerUrl" -o tsv)"
az identity federated-credential create -n $alb_id_name -g $rg -o none --only-show-errors \
--identity-name $alb_id_name --issuer "$aks_oidc_issuer" --subject "system:serviceaccount:azure-alb-system:alb-controller-sa"
# Deploy Helm chart
az aks get-credentials -n $aks_name -g $rg --overwrite-existing
helm install alb-controller oci://mcr.microsoft.com/application-lb/charts/alb-controller \
--version 1.0.2 \
--set albController.podIdentity.clientID=$(az identity show -g $rg -n $alb_id_name --query clientId -o tsv)
# Verify
kubectl get pods -n azure-alb-system
kubectl get gatewayclass azure-alb-external -o yaml
#############################
# Managed #
#############################
# Create AppGW for containers
az network vnet subnet create -g $rg --vnet-name $vnet_name -n $agfc_subnet_name --address-prefix $agfc_subnet_prefix --delegations 'Microsoft.ServiceNetworking/trafficControllers' -o none --only-show-errors
agfc_subnet_id=$(az network vnet subnet show -g $rg --vnet-name $vnet_name -n $agfc_subnet_name --query id --output tsv)
# Delegate roles to the ALB identity
echo "Adding roles for ALB identity..."
node_rg=$(az aks show -n $aks_name -g $rg --query "nodeResourceGroup" -o tsv)
node_rg_id=$(az group show -n $node_rg --query id -o tsv)
az role assignment create --assignee-object-id $alb_id_principal_id --scope $node_rg_id --role "fbc52c3f-28ad-4303-a892-8a056630b8f1" -o none # AppGw for Containers Configuration Manager role
az role assignment create --assignee-object-id $alb_id_principal_id --scope $agfc_subnet_id --role "4d97b98b-1d4f-4787-a291-c67834d212e7" -o none # Network Contributor
# Configure ALB (NOTE: this is only for managed ALB, not for BYO ALB): this will create the ALB resource in Azure
kubectl create ns $alb_ns_name
kubectl apply -f - <<EOF
apiVersion: alb.networking.azure.io/v1
kind: ApplicationLoadBalancer
metadata:
name: $agfc_name
namespace: $alb_ns_name
spec:
associations:
- $agfc_subnet_id
EOF
kubectl get ApplicationLoadBalancer -n $alb_ns_name
kubectl describe ApplicationLoadBalancer -n $alb_ns_name
# Create Gateway in k8s: this will create the association and frontends in Azure
alb_id=$(az network alb list -g $node_rg --query '[0].id' -o tsv)
alb_name=$(az network alb list -g $node_rg --query '[0].name' -o tsv)
echo "Creating Gateway in k8s associated to ALB ID $alb_id..."
kubectl apply -f - <<EOF
apiVersion: gateway.networking.k8s.io/v1beta1
kind: Gateway
metadata:
name: $alb_name
annotations:
alb.networking.azure.io/alb-namespace: $alb_ns_name
alb.networking.azure.io/alb-id: $alb_id
spec:
gatewayClassName: azure-alb-external
listeners:
- name: http
port: 80
protocol: HTTP
allowedRoutes:
namespaces:
from: Same
EOF
k get gateway
#############################
# Bring your own deployment #
#############################
# Create AppGW for containers
az network alb create -g $rg -n $agfc_name -o none --only-show-errors
az network alb frontend create -g $rg --alb-name $agfc_name -n $agfc_frontend_name -o none --only-show-errors
az network vnet subnet create -g $rg --vnet-name $vnet_name -n $agfc_subnet_name --address-prefix $agfc_subnet_prefix --delegations 'Microsoft.ServiceNetworking/trafficControllers' -o none --only-show-errors
agfc_subnet_id=$(az network vnet subnet show -g $rg --vnet-name $vnet_name -n $agfc_subnet_name --query id --output tsv)
# Delegate roles to the ALB identity
rg_id=$(az group show -n $rg --query id -o tsv)
az role assignment create --assignee-object-id $alb_id_principal_id --scope $rg_id --role "fbc52c3f-28ad-4303-a892-8a056630b8f1" -o none # AppGw for Containers Configuration Manager role
az role assignment create --assignee-object-id $alb_id_principal_id --scope $agfc_subnet_id --role "4d97b98b-1d4f-4787-a291-c67834d212e7" -o none # Network Contributor
# Create Gateway and frontend
az network alb create -g $rg -n $agfc_name -o none --only-show-errors
az network alb frontend create -g $rg --alb-name $agfc_name -n $agfc_frontend_name -o none --only-show-errors
# Associate the AppGW4C with the subnet
association_name='AppGW4Cassociation'
az network alb association create -g $rg -n $association_name --alb-name $agfc_name --subnet $agfc_subnet_id -o none --only-show-errors
# Create Gateway in k8s
agfc_id=$(az network alb list -g $rg --query '[0].id' -o tsv)
# agfc_name=$(az network alb list -g $rg --query '[0].name' -o tsv)
# agfc_frontend_name=$(az network alb frontend list --alb-name $agfc_name -g $rg --query '[0].name' -o tsv)
echo "Creating Gateway in k8s associated to ALB ID $alb_id and frontend $agfc_frontend_name..."
kubectl apply -f - <<EOF
apiVersion: gateway.networking.k8s.io/v1beta1
kind: Gateway
metadata:
name: $agfc_name
annotations:
alb.networking.azure.io/alb-namespace: $alb_ns_name
alb.networking.azure.io/alb-id: $agfc_id
spec:
gatewayClassName: azure-alb-external
listeners:
- name: http
port: 80
protocol: HTTP
allowedRoutes:
namespaces:
from: Same
addresses:
- type: alb.networking.azure.io/alb-frontend
value: $agfc_frontend_name
EOF
k get gateway
##################################
# Sample workload in the cluster #
##################################
kubectl create deployment yadaapi --image=erjosito/yadaapi:1.0 --port=8080 --replicas=1
kubectl expose deploy yadaapi --port=8080 --target-port=8080
# HTTP route
fqdn=$(kubectl get gateway $agfc_name -o jsonpath='{.status.addresses[0].value}') && echo $fqdn
kubectl apply -f - <<EOF
apiVersion: gateway.networking.k8s.io/v1beta1
kind: HTTPRoute
metadata:
name: yadaapi
spec:
parentRefs:
- name: $agfc_name
hostnames:
- "$fqdn"
rules:
- backendRefs:
- name: yadaapi
port: 8080
EOF
# Check
k get httproute
k describe httproute yadaapi
# Test
curl "$fqdn/api/healthcheck"
#####################
# Azure Diagnostics #
#####################
az network alb list -g $node_rg -o table
agfc_name=$(az network alb list -g $node_rg --query '[0].name' -o tsv)
az network alb association list --alb-name $agfc_name -g $node_rg -o table
az network alb frontend list --alb-name $agfc_name -g $node_rg -o table
##########################
# Kubernetes Diagnostics #
##########################
# Check the agic pods
kubectl get pods -n azure-alb-system
# Verify the ALB in Azure
az network alb list -g $rg -o table
alb_name=$(az network alb list -g $rg --query '[0].name' -o tsv)
az network alb frontend list -g $rg --alb-name $alb_name -o table
# Verify the ALB in k8s
k get applicationLoadBalancer
# Verify the gatewayclass
kubectl get gatewayclass azure-alb-external -o yaml
# Gateways
k get gateway
k describe gateway
# HTTP routes
k get httproute