From b7100eeda6501728357189100c6457ea8507657c Mon Sep 17 00:00:00 2001 From: l1b0k Date: Wed, 18 Dec 2024 17:07:34 +0800 Subject: [PATCH] add the datapath configuration checker Signed-off-by: l1b0k --- Makefile | 5 +- tests/kind/Makefile | 13 ++ tests/kind/cluster.yml | 7 + tests/kind/conf/eniip_datapathv2_cmdline | 1 + tests/kind/conf/eniip_default_cmdline | 1 + .../kind/conf/eniip_legacy_ciliumargs_cmdline | 1 + tests/kind/run.sh | 201 ++++++++++++++++++ 7 files changed, 228 insertions(+), 1 deletion(-) create mode 100644 tests/kind/Makefile create mode 100644 tests/kind/cluster.yml create mode 100644 tests/kind/conf/eniip_datapathv2_cmdline create mode 100644 tests/kind/conf/eniip_default_cmdline create mode 100644 tests/kind/conf/eniip_legacy_ciliumargs_cmdline create mode 100644 tests/kind/run.sh diff --git a/Makefile b/Makefile index b2fa7edf..766eaa00 100644 --- a/Makefile +++ b/Makefile @@ -50,7 +50,7 @@ vet: ## Run go vet against code. GOOS=linux go vet --tags "$(GO_BUILD_TAGS)" ./... .PHONY: test -test: manifests generate fmt vet envtest ## Run tests. +test: manifests generate fmt vet envtest datapath-test## Run tests. KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test -race --tags "$(GO_BUILD_TAGS)" $$(go list ./... | grep -Ev '/e2e|/mocks|/generated|/apis|/examples|/tests|/rpc') -coverprofile coverage.txt .PHONY: lint @@ -61,6 +61,9 @@ lint: golangci-lint ## Run golangci-lint linter & yamllint lint-fix: golangci-lint ## Run golangci-lint linter and perform fixes $(GOLANGCI_LINT) run +.PHONY: datapath-test +datapath-test: ## Run datapath tests using the Makefile in tests/kind directory. + make -C tests/kind datapath-test ##@ Build .PHONY: build diff --git a/tests/kind/Makefile b/tests/kind/Makefile new file mode 100644 index 00000000..f16a9520 --- /dev/null +++ b/tests/kind/Makefile @@ -0,0 +1,13 @@ +# tests/kind/Makefile + +##@ General + +.PHONY: help +help: ## Display this help. + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) + +##@ Tests + +.PHONY: datapath-test +datapath-test: ## Run datapath tests using run.sh script. + chmod +x run.sh && ./run.sh diff --git a/tests/kind/cluster.yml b/tests/kind/cluster.yml new file mode 100644 index 00000000..ccd9356d --- /dev/null +++ b/tests/kind/cluster.yml @@ -0,0 +1,7 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +networking: + disableDefaultCNI: true +nodes: + - role: control-plane + image: kindest/node:v1.30.8@sha256:17cd608b3971338d9180b00776cb766c50d0a0b6b904ab4ff52fd3fc5c6369bf diff --git a/tests/kind/conf/eniip_datapathv2_cmdline b/tests/kind/conf/eniip_datapathv2_cmdline new file mode 100644 index 00000000..6915c9b4 --- /dev/null +++ b/tests/kind/conf/eniip_datapathv2_cmdline @@ -0,0 +1 @@ +cilium-agent--cni-chaining-mode=terway-chainer--tunnel=disabled--enable-ipv4-masquerade=false--enable-ipv6-masquerade=false--disable-envoy-version-check=true--ipv4-range=169.254.10.0/30--ipv6-range=fe80:2400:3200:baba::/30--enable-local-node-route=false--enable-endpoint-health-checking=false--enable-health-checking=false--enable-service-topology=true--disable-cnp-status-updates=true--k8s-heartbeat-timeout=0--enable-session-affinity=true--install-iptables-rules=false--enable-l7-proxy=false--ipam=cluster-pool--enable-runtime-device-detection=true--enable-bandwidth-manager=true--agent-health-port=9099--enable-policy=never--labels=k8s:io\.kubernetes\.pod\.namespace--datapath-mode=veth--enable-in-cluster-loadbalance=true \ No newline at end of file diff --git a/tests/kind/conf/eniip_default_cmdline b/tests/kind/conf/eniip_default_cmdline new file mode 100644 index 00000000..cb9790bf --- /dev/null +++ b/tests/kind/conf/eniip_default_cmdline @@ -0,0 +1 @@ +cilium-agent --cni-chaining-mode=terway-chainer --tunnel=disabled --enable-ipv4-masquerade=false --enable-ipv6-masquerade=false --disable-envoy-version-check=true --ipv4-range=169.254.10.0/30 --ipv6-range=fe80:2400:3200:baba::/30 --enable-local-node-route=false --enable-endpoint-health-checking=false --enable-health-checking=false --enable-service-topology=true --disable-cnp-status-updates=true --k8s-heartbeat-timeout=0 --enable-session-affinity=true --install-iptables-rules=false --enable-l7-proxy=false --ipam=cluster-pool --enable-runtime-device-detection=true --enable-bandwidth-manager=true --agent-health-port=9099 --enable-policy=default --datapath-mode=veth --enable-in-cluster-loadbalance=true \ No newline at end of file diff --git a/tests/kind/conf/eniip_legacy_ciliumargs_cmdline b/tests/kind/conf/eniip_legacy_ciliumargs_cmdline new file mode 100644 index 00000000..cb9790bf --- /dev/null +++ b/tests/kind/conf/eniip_legacy_ciliumargs_cmdline @@ -0,0 +1 @@ +cilium-agent --cni-chaining-mode=terway-chainer --tunnel=disabled --enable-ipv4-masquerade=false --enable-ipv6-masquerade=false --disable-envoy-version-check=true --ipv4-range=169.254.10.0/30 --ipv6-range=fe80:2400:3200:baba::/30 --enable-local-node-route=false --enable-endpoint-health-checking=false --enable-health-checking=false --enable-service-topology=true --disable-cnp-status-updates=true --k8s-heartbeat-timeout=0 --enable-session-affinity=true --install-iptables-rules=false --enable-l7-proxy=false --ipam=cluster-pool --enable-runtime-device-detection=true --enable-bandwidth-manager=true --agent-health-port=9099 --enable-policy=default --datapath-mode=veth --enable-in-cluster-loadbalance=true \ No newline at end of file diff --git a/tests/kind/run.sh b/tests/kind/run.sh new file mode 100644 index 00000000..82a8e7d1 --- /dev/null +++ b/tests/kind/run.sh @@ -0,0 +1,201 @@ +#!/bin/bash + +set -e + +install_kind(){ + # For AMD64 / x86_64 + [ $(uname -m) = x86_64 ] && curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.26.0/kind-linux-amd64 && curl -LO "https://dl.k8s.io/release/v1.30.8/bin/linux/amd64/kubectl" + # For ARM64 + [ $(uname -m) = aarch64 ] && curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.26.0/kind-linux-arm64 && curl -LO "https://dl.k8s.io/release/v1.30.8/bin/linux/arm64/kubectl" + chmod +x ./kind ./kubectl + sudo mv ./kind /usr/local/bin/kind + sudo mv ./kubectl /usr/local/bin/kubectl +} + +install_helm(){ + curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash +} + +build_terway_images(){ + docker build -t local/terway -f ../../deploy/images/terway/Dockerfile ../../ + docker build -t local/terway-controlplane -f ../../deploy/images/terway-controlplane/Dockerfile ../../ + docker tag local/terway local/terway:1 + docker tag local/terway-controlplane local/terway-controlplane:1 +} + +prepare_kind(){ + kind delete cluster || true + kind create cluster --config cluster.yml + kind load docker-image local/terway:1 + kind load docker-image local/terway-controlplane:1 + kubectl cluster-info --context kind-kind +} + +get_cilium_cmdline() { + ctrlID=$(docker ps --filter "name=kind-control-plane" --format "{{.ID}}") + echo "pid=\$(pidof cilium-agent);if [ -z \"\$pid\" ];then exit 1;fi; cat /proc/\${pid}/cmdline" > cmd + docker cp cmd "${ctrlID}:/" + docker exec "${ctrlID}" bash /cmd +} + +tear_down_callback(){ + helm uninstall -n kube-system terway-eniip +} + +eniip_default_setup(){ + prepare_kind + helm install -n kube-system terway-eniip ../../charts/terway \ + --replace --force \ + --set terway.image.repository=local/terway \ + --set terway.image.tag=1 \ + --set terway.accessKey=foo \ + --set terwayControlplane.accessSecret=bar \ + --set terwayControlplane.image.repository=local/terway-controlplane \ + --set terwayControlplane.image.tag=1 \ + --set terwayControlplane.accessKey=foo \ + --set terwayControlplane.accessSecret=bar \ + --set terway.enableNetworkPolicy=true +} + +eniip_default_check() { + echo "Checking eniip default setup..." >&2 + local current="" + for ((i=1; i<=10; i++)); do + set +e + current=$(get_cilium_cmdline) + exit_code=$? + set -e + if [ $exit_code -eq 0 ]; then + echo "Success on attempt $i" >&2 + break + else + echo "Attempt $i failed. Retrying in 10 seconds..." >&2 + sleep 10 + fi + done + + if ! diff -w <(echo "$current") conf/eniip_default_cmdline; then + echo "Files are not equal." + exit 1 + fi +} + +eniip_datapathv2_setup(){ + prepare_kind + helm install -n kube-system terway-eniip ../../charts/terway \ + --replace --force \ + --set terway.image.repository=local/terway \ + --set terway.image.tag=1 \ + --set terway.accessKey=foo \ + --set terwayControlplane.accessSecret=bar \ + --set terwayControlplane.image.repository=local/terway-controlplane \ + --set terwayControlplane.image.tag=1 \ + --set terwayControlplane.accessKey=foo \ + --set terwayControlplane.accessSecret=bar \ + --set terway.enableDatapathV2=true +} + +eniip_datapathv2_check() { + echo "Checking eniip default setup..." >&2 + local current="" + for ((i=1; i<=10; i++)); do + set +e + current=$(get_cilium_cmdline) + exit_code=$? + set -e + if [ $exit_code -eq 0 ]; then + echo "Success on attempt $i" >&2 + break + else + echo "Attempt $i failed. Retrying in 10 seconds..." >&2 + sleep 10 + fi + done + + if ! diff -w <(echo "$current") conf/eniip_datapathv2_cmdline; then + echo "Files are not equal." + exit 1 + fi +} + +eniip_legacy_ciliumargs_setup(){ + prepare_kind + helm install -n kube-system terway-eniip ../../charts/terway \ + --replace --force \ + --set terway.image.repository=local/terway \ + --set terway.image.tag=1 \ + --set terway.accessKey=foo \ + --set terwayControlplane.accessSecret=bar \ + --set terwayControlplane.image.repository=local/terway-controlplane \ + --set terwayControlplane.image.tag=1 \ + --set terwayControlplane.accessKey=foo \ + --set terwayControlplane.accessSecret=bar \ + --set terway.enableNetworkPolicy=true \ + --set terway.ciliumArgs="--disable-per-package-lb=true" +} + + +eniip_legacy_ciliumargs_check() { + echo "Checking eniip default setup..." >&2 + local current="" + for ((i=1; i<=10; i++)); do + set +e + current=$(get_cilium_cmdline) + exit_code=$? + set -e + if [ $exit_code -eq 0 ]; then + echo "Success on attempt $i" >&2 + break + else + echo "Attempt $i failed. Retrying in 10 seconds..." >&2 + sleep 10 + fi + done + + if ! diff -w <(echo "$current") conf/eniip_legacy_ciliumargs_cmdline; then + echo "Files are not equal." + exit 1 + fi +} + + +run_test_function() { + local test_name="$1" + echo "Running test $test_name" + +} + +run_test() { + local setup_callback="$1" + local run_test_callback="$2" + local tear_down_callback="$3" + + if [ -n "$setup_callback" ]; then + $setup_callback + fi + + if [ -n "$run_test_callback" ]; then + $run_test_callback + fi + + if [ -n "$tear_down_callback" ]; then + $tear_down_callback + fi +} + +set -e + +install_kind +install_helm +build_terway_images + +tests=( + "eniip_default_setup eniip_default_check tear_down_callback" + "eniip_datapathv2_setup eniip_datapathv2_check tear_down_callback" + "eniip_legacy_ciliumargs_setup eniip_legacy_ciliumargs_check tear_down_callback" +) + +for test in "${tests[@]}"; do + IFS=' ' read -r setup_callback run_test_callback tear_down_callback <<< "$test" + run_test "$setup_callback" "$run_test_callback" "$tear_down_callback" +done \ No newline at end of file