Skip to content

Commit 1bde234

Browse files
danwinshipdanwinship
committed
KEP-4974: Add ci-kubernetes-kind-network-deprecate-endpoints
1 parent 77a8b6f commit 1bde234

File tree

2 files changed

+360
-0
lines changed

2 files changed

+360
-0
lines changed

‎config/jobs/kubernetes/sig-network/sig-network-kind.yaml‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -965,3 +965,47 @@ periodics:
965965
# this is mostly for building kubernetes
966966
memory: 9Gi
967967
cpu: 7
968+
- interval: 24h
969+
cluster: k8s-infra-prow-build
970+
name: ci-kubernetes-kind-network-deprecate-endpoints
971+
annotations:
972+
testgrid-dashboards: sig-network-kind
973+
testgrid-tab-name: sig-network-kind, deprecate-endpoints
974+
description: Uses kind to run e2e sig-network tests against latest kubernetes master with the Endpoints and EndpointSlice mirroring controllers disabled
975+
testgrid-alert-email: [email protected], [email protected]
976+
testgrid-num-columns-recent: '6'
977+
labels:
978+
preset-dind-enabled: "true"
979+
decorate: true
980+
decoration_config:
981+
timeout: 60m
982+
extra_refs:
983+
- org: kubernetes
984+
repo: kubernetes
985+
base_ref: master
986+
path_alias: k8s.io/kubernetes
987+
spec:
988+
containers:
989+
- image: gcr.io/k8s-staging-test-infra/krte:v20250925-95b5a2c7a5-master
990+
command:
991+
- wrapper.sh
992+
- bash
993+
- -c
994+
- curl -sSL https://kind.sigs.k8s.io/dl/latest/linux-amd64.tgz | tar xvfz - -C "${PATH%%:*}/" && $GOPATH/src/k8s.io/test-infra/experiment/kind-noendpoints-e2e.sh
995+
env:
996+
- name: LABEL_FILTER
997+
value: "(Conformance || sig-network ) && !EndpointsController && !EndpointSliceMirroring && !Disruptive && !Flaky"
998+
- name: SKIP
999+
value: SCTPConnectivity
1000+
- name: PARALLEL
1001+
value: "true"
1002+
# we need privileged mode in order to do docker in docker
1003+
securityContext:
1004+
privileged: true
1005+
resources:
1006+
limits:
1007+
cpu: 4
1008+
memory: 9Gi
1009+
requests:
1010+
cpu: 4
1011+
memory: 9Gi

‎experiment/kind-noendpoints-e2e.sh‎

Lines changed: 316 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,316 @@
1+
#!/bin/sh
2+
# Copyright 2018 The Kubernetes Authors.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
# hack script for running a kind e2e
17+
# must be run with a kubernetes checkout in $PWD (IE from the checkout)
18+
# Usage: SKIP="ginkgo skip regex" FOCUS="ginkgo focus regex" kind-e2e.sh
19+
20+
set -o errexit -o nounset -o xtrace
21+
22+
# Settings:
23+
# SKIP: ginkgo skip regex
24+
# FOCUS: ginkgo focus regex
25+
# LABEL_FILTER: ginkgo label query for selecting tests (see "Spec Labels" in https://onsi.github.io/ginkgo/#filtering-specs)
26+
#
27+
# The default is to focus on conformance tests. Serial tests get skipped when
28+
# parallel testing is enabled. Using LABEL_FILTER instead of combining SKIP and
29+
# FOCUS is recommended (more expressive, easier to read than regexp).
30+
#
31+
# GA_ONLY: true - limit to GA APIs/features as much as possible
32+
# false - (default) APIs and features left at defaults
33+
# FEATURE_GATES:
34+
# JSON or YAML encoding of a string/bool map: {"FeatureGateA": true, "FeatureGateB": false}
35+
# Enables or disables feature gates in the entire cluster.
36+
# Cannot be used when GA_ONLY=true.
37+
# RUNTIME_CONFIG:
38+
# JSON or YAML encoding of a string/string (!) map: {"apia.example.com/v1alpha1": "true", "apib.example.com/v1beta1": "false"}
39+
# Enables API groups in the apiserver via --runtime-config.
40+
# Cannot be used when GA_ONLY=true.
41+
42+
# cleanup logic for cleanup on exit
43+
CLEANED_UP=false
44+
cleanup() {
45+
if [ "$CLEANED_UP" = "true" ]; then
46+
return
47+
fi
48+
# KIND_CREATE_ATTEMPTED is true once we: kind create
49+
if [ "${KIND_CREATE_ATTEMPTED:-}" = true ]; then
50+
kind "export" logs "${ARTIFACTS}" || true
51+
kind delete cluster || true
52+
fi
53+
rm -f _output/bin/e2e.test || true
54+
# remove our tempdir, this needs to be last, or it will prevent kind delete
55+
if [ -n "${TMP_DIR:-}" ]; then
56+
rm -rf "${TMP_DIR:?}"
57+
fi
58+
CLEANED_UP=true
59+
}
60+
61+
# setup signal handlers
62+
# shellcheck disable=SC2317 # this is not unreachable code
63+
signal_handler() {
64+
if [ -n "${GINKGO_PID:-}" ]; then
65+
kill -TERM "$GINKGO_PID" || true
66+
fi
67+
cleanup
68+
}
69+
trap signal_handler INT TERM
70+
71+
# build kubernetes / node image, e2e binaries
72+
build() {
73+
# build the node image w/ kubernetes
74+
kind build node-image -v 1
75+
# Ginkgo v1 is used by Kubernetes 1.24 and earlier, fallback if v2 is not available.
76+
GINKGO_SRC_DIR="vendor/github.com/onsi/ginkgo/v2/ginkgo"
77+
if [ ! -d "$GINKGO_SRC_DIR" ]; then
78+
GINKGO_SRC_DIR="vendor/github.com/onsi/ginkgo/ginkgo"
79+
fi
80+
# make sure we have e2e requirements
81+
make all WHAT="cmd/kubectl test/e2e/e2e.test ${GINKGO_SRC_DIR}"
82+
83+
# Ensure the built kubectl is used instead of system
84+
export PATH="${PWD}/_output/bin:$PATH"
85+
}
86+
87+
check_structured_log_support() {
88+
case "${KUBE_VERSION}" in
89+
v1.1[0-8].*)
90+
echo "$1 is only supported on versions >= v1.19, got ${KUBE_VERSION}"
91+
exit 1
92+
;;
93+
esac
94+
}
95+
96+
# up a cluster with kind
97+
create_cluster() {
98+
# Grab the version of the cluster we're about to start
99+
KUBE_VERSION="$(docker run --rm --entrypoint=cat "kindest/node:latest" /kind/version)"
100+
101+
# Default Log level for all components in test clusters
102+
KIND_CLUSTER_LOG_LEVEL=${KIND_CLUSTER_LOG_LEVEL:-4}
103+
104+
# potentially enable --logging-format
105+
CLUSTER_LOG_FORMAT=${CLUSTER_LOG_FORMAT:-}
106+
scheduler_extra_args=" \"v\": \"${KIND_CLUSTER_LOG_LEVEL}\""
107+
controllerManager_extra_args=" \"v\": \"${KIND_CLUSTER_LOG_LEVEL}\""
108+
apiServer_extra_args=" \"v\": \"${KIND_CLUSTER_LOG_LEVEL}\""
109+
if [ -n "$CLUSTER_LOG_FORMAT" ]; then
110+
check_structured_log_support "CLUSTER_LOG_FORMAT"
111+
scheduler_extra_args="${scheduler_extra_args}
112+
\"logging-format\": \"${CLUSTER_LOG_FORMAT}\""
113+
controllerManager_extra_args="${controllerManager_extra_args}
114+
\"logging-format\": \"${CLUSTER_LOG_FORMAT}\""
115+
apiServer_extra_args="${apiServer_extra_args}
116+
\"logging-format\": \"${CLUSTER_LOG_FORMAT}\""
117+
fi
118+
kubelet_extra_args=" \"v\": \"${KIND_CLUSTER_LOG_LEVEL}\"
119+
\"container-log-max-files\": \"10\"
120+
\"container-log-max-size\": \"100Mi\""
121+
KUBELET_LOG_FORMAT=${KUBELET_LOG_FORMAT:-$CLUSTER_LOG_FORMAT}
122+
if [ -n "$KUBELET_LOG_FORMAT" ]; then
123+
check_structured_log_support "KUBECTL_LOG_FORMAT"
124+
kubelet_extra_args="${kubelet_extra_args}
125+
\"logging-format\": \"${KUBELET_LOG_FORMAT}\""
126+
fi
127+
128+
# JSON or YAML map injected into featureGates config
129+
feature_gates="${FEATURE_GATES:-{\}}"
130+
# --runtime-config argument value passed to the API server, again as a map
131+
runtime_config="${RUNTIME_CONFIG:-{\}}"
132+
133+
case "${GA_ONLY:-false}" in
134+
false)
135+
:
136+
;;
137+
true)
138+
if [ "${feature_gates}" != "{}" ]; then
139+
echo "GA_ONLY=true and FEATURE_GATES=${feature_gates} are mutually exclusive."
140+
exit 1
141+
fi
142+
if [ "${runtime_config}" != "{}" ]; then
143+
echo "GA_ONLY=true and RUNTIME_CONFIG=${runtime_config} are mutually exclusive."
144+
exit 1
145+
fi
146+
147+
echo "Limiting to GA APIs and features for ${KUBE_VERSION}"
148+
feature_gates='{"AllAlpha":false,"AllBeta":false}'
149+
runtime_config='{"api/alpha":"false", "api/beta":"false"}'
150+
;;
151+
*)
152+
echo "\$GA_ONLY set to '${GA_ONLY}'; supported values are true and false (default)"
153+
exit 1
154+
;;
155+
esac
156+
157+
# create the config file
158+
cat <<EOF > "${ARTIFACTS}/kind-config.yaml"
159+
# config for 1 control plane node and 2 workers (necessary for conformance)
160+
kind: Cluster
161+
apiVersion: kind.x-k8s.io/v1alpha4
162+
networking:
163+
ipFamily: ${IP_FAMILY:-ipv4}
164+
kubeProxyMode: ${KUBE_PROXY_MODE:-iptables}
165+
# don't pass through host search paths
166+
# TODO: possibly a reasonable default in the future for kind ...
167+
dnsSearch: []
168+
nodes:
169+
- role: control-plane
170+
- role: worker
171+
- role: worker
172+
featureGates: ${feature_gates}
173+
runtimeConfig: ${runtime_config}
174+
kubeadmConfigPatches:
175+
- |
176+
kind: ClusterConfiguration
177+
metadata:
178+
name: config
179+
apiServer:
180+
extraArgs:
181+
${apiServer_extra_args}
182+
controllerManager:
183+
extraArgs:
184+
controllers: "-endpoints-controller,-endpointslice-mirroring-controller,*"
185+
${controllerManager_extra_args}
186+
scheduler:
187+
extraArgs:
188+
${scheduler_extra_args}
189+
---
190+
kind: InitConfiguration
191+
nodeRegistration:
192+
kubeletExtraArgs:
193+
${kubelet_extra_args}
194+
---
195+
kind: JoinConfiguration
196+
nodeRegistration:
197+
kubeletExtraArgs:
198+
${kubelet_extra_args}
199+
EOF
200+
# NOTE: must match the number of workers above
201+
NUM_NODES=2
202+
# actually create the cluster
203+
# TODO(BenTheElder): settle on verbosity for this script
204+
KIND_CREATE_ATTEMPTED=true
205+
kind create cluster \
206+
--image=kindest/node:latest \
207+
--retain \
208+
--wait=1m \
209+
-v=3 \
210+
"--config=${ARTIFACTS}/kind-config.yaml"
211+
212+
# debug cluster version
213+
kubectl version
214+
215+
# Patch kube-proxy to set the verbosity level
216+
kubectl patch -n kube-system daemonset/kube-proxy \
217+
--type='json' -p='[{"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "--v='"${KIND_CLUSTER_LOG_LEVEL}"'" }]'
218+
}
219+
220+
# run e2es with ginkgo-e2e.sh
221+
run_tests() {
222+
# IPv6 clusters need some CoreDNS changes in order to work in k8s CI:
223+
# 1. k8s CI doesn´t offer IPv6 connectivity, so CoreDNS should be configured
224+
# to work in an offline environment:
225+
# https://github.com/coredns/coredns/issues/2494#issuecomment-457215452
226+
# 2. k8s CI adds following domains to resolv.conf search field:
227+
# c.k8s-prow-builds.internal google.internal.
228+
# CoreDNS should handle those domains and answer with NXDOMAIN instead of SERVFAIL
229+
# otherwise pods stops trying to resolve the domain.
230+
if [ "${IP_FAMILY:-ipv4}" = "ipv6" ]; then
231+
# Get the current config
232+
original_coredns=$(kubectl get -oyaml -n=kube-system configmap/coredns)
233+
echo "Original CoreDNS config:"
234+
echo "${original_coredns}"
235+
# Patch it
236+
fixed_coredns=$(
237+
printf '%s' "${original_coredns}" | sed \
238+
-e 's/^.*kubernetes cluster\.local/& internal/' \
239+
-e '/^.*upstream$/d' \
240+
-e '/^.*fallthrough.*$/d' \
241+
-e '/^.*forward . \/etc\/resolv.conf$/d' \
242+
-e '/^.*loop$/d' \
243+
)
244+
echo "Patched CoreDNS config:"
245+
echo "${fixed_coredns}"
246+
printf '%s' "${fixed_coredns}" | kubectl apply -f -
247+
fi
248+
249+
# ginkgo regexes and label filter
250+
SKIP="${SKIP:-}"
251+
FOCUS="${FOCUS:-}"
252+
LABEL_FILTER="${LABEL_FILTER:-}"
253+
if [ -z "${FOCUS}" ] && [ -z "${LABEL_FILTER}" ]; then
254+
FOCUS="\\[Conformance\\]"
255+
fi
256+
# if we set PARALLEL=true, skip serial tests set --ginkgo-parallel
257+
if [ "${PARALLEL:-false}" = "true" ]; then
258+
export GINKGO_PARALLEL=y
259+
if [ -z "${SKIP}" ]; then
260+
SKIP="\\[Serial\\]"
261+
else
262+
SKIP="\\[Serial\\]|${SKIP}"
263+
fi
264+
fi
265+
266+
# setting this env prevents ginkgo e2e from trying to run provider setup
267+
export KUBERNETES_CONFORMANCE_TEST='y'
268+
# setting these is required to make RuntimeClass tests work ... :/
269+
export KUBE_CONTAINER_RUNTIME=remote
270+
export KUBE_CONTAINER_RUNTIME_ENDPOINT=unix:///run/containerd/containerd.sock
271+
export KUBE_CONTAINER_RUNTIME_NAME=containerd
272+
# ginkgo can take forever to exit, so we run it in the background and save the
273+
# PID, bash will not run traps while waiting on a process, but it will while
274+
# running a builtin like `wait`, saving the PID also allows us to forward the
275+
# interrupt
276+
./hack/ginkgo-e2e.sh \
277+
'--provider=skeleton' "--num-nodes=${NUM_NODES}" \
278+
"--ginkgo.focus=${FOCUS}" "--ginkgo.skip=${SKIP}" "--ginkgo.label-filter=${LABEL_FILTER}" \
279+
"--report-dir=${ARTIFACTS}" '--disable-log-dump=true' &
280+
GINKGO_PID=$!
281+
wait "$GINKGO_PID"
282+
}
283+
284+
main() {
285+
# create temp dir and setup cleanup
286+
TMP_DIR=$(mktemp -d)
287+
288+
# ensure artifacts (results) directory exists when not in CI
289+
export ARTIFACTS="${ARTIFACTS:-${PWD}/_artifacts}"
290+
mkdir -p "${ARTIFACTS}"
291+
292+
# export the KUBECONFIG to a unique path for testing
293+
KUBECONFIG="${HOME}/.kube/kind-test-config"
294+
export KUBECONFIG
295+
echo "exported KUBECONFIG=${KUBECONFIG}"
296+
297+
# debug kind version
298+
kind version
299+
300+
# build kubernetes
301+
build
302+
# in CI attempt to release some memory after building
303+
if [ -n "${KUBETEST_IN_DOCKER:-}" ]; then
304+
sync || true
305+
echo 1 > /proc/sys/vm/drop_caches || true
306+
fi
307+
308+
# create the cluster and run tests
309+
res=0
310+
create_cluster || res=$?
311+
run_tests || res=$?
312+
cleanup || res=$?
313+
exit $res
314+
}
315+
316+
main

0 commit comments

Comments
 (0)