From 11a17e24e13107f9ad0113421c2da705e13fa546 Mon Sep 17 00:00:00 2001 From: l1b0k Date: Mon, 23 Dec 2024 14:45:14 +0800 Subject: [PATCH] tests: update e2e Signed-off-by: l1b0k --- Makefile | 2 +- tests/connecctive_test.go | 55 ++-- tests/main_test.go | 45 ++- tests/stress/setup_test.go | 3 +- tests/trunk/perf_test.go | 87 ------ tests/trunk/setup_test.go | 87 ------ tests/{trunk => }/trunk_test.go | 518 +++++++++----------------------- tests/utils_test.go | 18 ++ 8 files changed, 202 insertions(+), 613 deletions(-) delete mode 100644 tests/trunk/perf_test.go delete mode 100644 tests/trunk/setup_test.go rename tests/{trunk => }/trunk_test.go (50%) diff --git a/Makefile b/Makefile index 766eaa00..7064149f 100644 --- a/Makefile +++ b/Makefile @@ -51,7 +51,7 @@ vet: ## Run go vet against code. .PHONY: test test: manifests generate fmt vet envtest datapath-test## Run tests. - KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test -race --tags "$(GO_BUILD_TAGS)" $$(go list ./... | grep -Ev '/e2e|/mocks|/generated|/apis|/examples|/tests|/rpc') -coverprofile coverage.txt + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test -race --tags "$(GO_BUILD_TAGS)" $$(go list ./... | grep -Ev '/e2e|/mocks|/generated|/apis|/examples|/tests|/rpc|/windows') -coverprofile coverage.txt .PHONY: lint lint: golangci-lint ## Run golangci-lint linter & yamllint diff --git a/tests/connecctive_test.go b/tests/connecctive_test.go index 1d32bf33..88577277 100644 --- a/tests/connecctive_test.go +++ b/tests/connecctive_test.go @@ -14,16 +14,14 @@ import ( networkingv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilerrors "k8s.io/apimachinery/pkg/util/errors" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/e2e-framework/klient" - "sigs.k8s.io/e2e-framework/klient/k8s" "sigs.k8s.io/e2e-framework/klient/wait" "sigs.k8s.io/e2e-framework/klient/wait/conditions" "sigs.k8s.io/e2e-framework/pkg/envconf" "sigs.k8s.io/e2e-framework/pkg/features" ) -var resourceKey struct{} - func getStack() []string { var r []string if testIPv4 { @@ -63,19 +61,19 @@ func TestConnective(t *testing.T) { { name: "trunk pod", podFunc: func(pod *Pod) *Pod { - return pod.WithLabels(map[string]string{"trunk": "enable"}) + return pod.WithLabels(map[string]string{"netplan": "default"}) }, }, { name: "trunk pod alinux2", podFunc: func(pod *Pod) *Pod { - return pod.WithLabels(map[string]string{"trunk": "enable"}).WithNodeAffinity(map[string]string{"e2e-os": "alinux2"}) + return pod.WithLabels(map[string]string{"netplan": "default"}).WithNodeAffinity(map[string]string{"e2e-os": "alinux2"}) }, }, { name: "trunk pod alinux3", podFunc: func(pod *Pod) *Pod { - return pod.WithLabels(map[string]string{"trunk": "enable"}).WithNodeAffinity(map[string]string{"e2e-os": "alinux3"}) + return pod.WithLabels(map[string]string{"netplan": "default"}).WithNodeAffinity(map[string]string{"e2e-os": "alinux3"}) }, }, } @@ -86,7 +84,7 @@ func TestConnective(t *testing.T) { hairpin := features.New(fmt.Sprintf("PodConnective/hairpin-%s", name)). Setup(func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { - var objs []k8s.Object + var objs []client.Object server := fn(NewPod("server", config.Namespace()). WithLabels(map[string]string{"app": "server"}). @@ -110,7 +108,7 @@ func TestConnective(t *testing.T) { } objs = append(objs, svc.Service) } - ctx = context.WithValue(ctx, resourceKey, objs) + ctx = SaveResources(ctx, objs...) return ctx }). Assess("Pod can access own service", func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { @@ -145,7 +143,7 @@ func TestConnective(t *testing.T) { podSameNode := features.New(fmt.Sprintf("PodConnective/podSameNode-%s", name)). Setup(func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { - var objs []k8s.Object + var objs []client.Object server := fn(NewPod("server", config.Namespace()). WithLabels(map[string]string{"app": "server"}). @@ -182,8 +180,7 @@ func TestConnective(t *testing.T) { } objs = append(objs, svc.Service) } - ctx = context.WithValue(ctx, resourceKey, objs) - + ctx = SaveResources(ctx, objs...) return ctx }). Assess("Pod can access server", func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { @@ -225,7 +222,7 @@ func TestConnective(t *testing.T) { podDifferentNode := features.New(fmt.Sprintf("PodConnective/podDifferentNode-%s", name)). Setup(func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { - var objs []k8s.Object + var objs []client.Object server := fn(NewPod("server", config.Namespace()). WithLabels(map[string]string{"app": "server"}). @@ -262,7 +259,7 @@ func TestConnective(t *testing.T) { } objs = append(objs, svc.Service) } - ctx = context.WithValue(ctx, resourceKey, objs) + ctx = SaveResources(ctx, objs...) return ctx }). @@ -305,7 +302,7 @@ func TestConnective(t *testing.T) { hostToPodSameNode := features.New(fmt.Sprintf("PodConnective/hostToSameNode-%s", name)). Setup(func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { - var objs []k8s.Object + var objs []client.Object server := fn(NewPod("server", config.Namespace()). WithLabels(map[string]string{"app": "server"}). @@ -344,7 +341,7 @@ func TestConnective(t *testing.T) { } objs = append(objs, svc.Service) } - ctx = context.WithValue(ctx, resourceKey, objs) + ctx = SaveResources(ctx, objs...) return ctx }). @@ -427,19 +424,19 @@ func TestNetworkPolicy(t *testing.T) { { name: "trunk pod", podFunc: func(pod *Pod) *Pod { - return pod.WithLabels(map[string]string{"trunk": "enable"}) + return pod.WithLabels(map[string]string{"netplan": "default"}) }, }, { name: "trunk pod alinux2", podFunc: func(pod *Pod) *Pod { - return pod.WithLabels(map[string]string{"trunk": "enable"}).WithNodeAffinity(map[string]string{"e2e-os": "alinux2"}) + return pod.WithLabels(map[string]string{"netplan": "default"}).WithNodeAffinity(map[string]string{"e2e-os": "alinux2"}) }, }, { name: "trunk pod alinux3", podFunc: func(pod *Pod) *Pod { - return pod.WithLabels(map[string]string{"trunk": "enable"}).WithNodeAffinity(map[string]string{"e2e-os": "alinux3"}) + return pod.WithLabels(map[string]string{"netplan": "default"}).WithNodeAffinity(map[string]string{"e2e-os": "alinux3"}) }, }, } @@ -449,7 +446,7 @@ func TestNetworkPolicy(t *testing.T) { healthCheck := features.New(fmt.Sprintf("NetworkPolicy/PodHealthCheck-%s", name)). Setup(func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { - var objs []k8s.Object + var objs []client.Object policy := NewNetworkPolicy("default-deny-ingress", config.Namespace()). WithPolicyType(networkingv1.PolicyTypeIngress) @@ -469,7 +466,7 @@ func TestNetworkPolicy(t *testing.T) { } } - ctx = context.WithValue(ctx, resourceKey, objs) + ctx = SaveResources(ctx, objs...) return ctx }). @@ -486,7 +483,7 @@ func TestNetworkPolicy(t *testing.T) { denyIngressSameNode := features.New(fmt.Sprintf("NetworkPolicy/DenyIngressSameNode-%s", name)). Setup(func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { - var objs []k8s.Object + var objs []client.Object policy := NewNetworkPolicy("deny-ingress", config.Namespace()). WithPolicyType(networkingv1.PolicyTypeIngress). @@ -525,7 +522,7 @@ func TestNetworkPolicy(t *testing.T) { } } - ctx = context.WithValue(ctx, resourceKey, objs) + ctx = SaveResources(ctx, objs...) return ctx }). @@ -557,7 +554,7 @@ func TestNetworkPolicy(t *testing.T) { denyIngressotherNode := features.New(fmt.Sprintf("NetworkPolicy/denyIngressotherNode-%s", name)). Setup(func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { - var objs []k8s.Object + var objs []client.Object policy := NewNetworkPolicy("deny-ingress", config.Namespace()). WithPolicyType(networkingv1.PolicyTypeIngress). @@ -596,7 +593,7 @@ func TestNetworkPolicy(t *testing.T) { } } - ctx = context.WithValue(ctx, resourceKey, objs) + ctx = SaveResources(ctx, objs...) return ctx }). @@ -628,7 +625,7 @@ func TestNetworkPolicy(t *testing.T) { denyEgressSameNode := features.New(fmt.Sprintf("NetworkPolicy/denyEgressSameNode-%s", name)). Setup(func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { - var objs []k8s.Object + var objs []client.Object policy := NewNetworkPolicy("deny-ingress", config.Namespace()). WithPolicyType(networkingv1.PolicyTypeEgress). @@ -660,8 +657,7 @@ func TestNetworkPolicy(t *testing.T) { } } - ctx = context.WithValue(ctx, resourceKey, objs) - + ctx = SaveResources(ctx, objs...) return ctx }). Assess("Check ingress policy", func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { @@ -685,7 +681,7 @@ func TestNetworkPolicy(t *testing.T) { denyEgressOtherNode := features.New(fmt.Sprintf("NetworkPolicy/denyEgressOtherNode-%s", name)). Setup(func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { - var objs []k8s.Object + var objs []client.Object policy := NewNetworkPolicy("deny-ingress", config.Namespace()). WithPolicyType(networkingv1.PolicyTypeEgress). @@ -717,8 +713,7 @@ func TestNetworkPolicy(t *testing.T) { } } - ctx = context.WithValue(ctx, resourceKey, objs) - + ctx = SaveResources(ctx, objs...) return ctx }). Assess("Check ingress policy", func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { diff --git a/tests/main_test.go b/tests/main_test.go index 00bc74bf..546d92c0 100644 --- a/tests/main_test.go +++ b/tests/main_test.go @@ -13,7 +13,9 @@ import ( "testing" "time" + "github.com/samber/lo" k8sErr "k8s.io/apimachinery/pkg/api/errors" + "sigs.k8s.io/controller-runtime/pkg/client" "go.uber.org/atomic" appsv1 "k8s.io/api/apps/v1" @@ -61,6 +63,8 @@ var ( isFailed atomic.Bool nginxImage string + + defaultPodNetworkingName string ) func init() { @@ -68,6 +72,8 @@ func init() { flag.StringVar(&timeout, "timeout", "2m", "2m") flag.StringVar(&vSwitchIDs, "vswitch-ids", "", "extra vSwitchIDs") flag.StringVar(&securityGroupIDs, "security-group-ids", "", "extra securityGroupIDs") + + defaultPodNetworkingName = "default-pn" } func TestMain(m *testing.M) { @@ -99,21 +105,15 @@ func TestMain(m *testing.M) { setPodNetworking, ) testenv.AfterEachFeature(func(ctx context.Context, config *envconf.Config, t *testing.T, feature features.Feature) (context.Context, error) { - objs, ok := ctx.Value(resourceKey).([]k8s.Object) - if !ok { - return ctx, nil - } - for _, obj := range objs { - _ = config.Client().Resources().Delete(ctx, obj) + lo.ForEach(ResourcesFromCtx(ctx), func(item client.Object, index int) { + _ = config.Client().Resources().Delete(ctx, item) - err = wait.For(conditions.New(config.Client().Resources()).ResourceDeleted(obj), wait.WithContext(ctx), - wait.WithTimeout(parsedTimeout), - wait.WithInterval(1*time.Second)) + err := wait.For(conditions.New(config.Client().Resources()).ResourceDeleted(item), + wait.WithInterval(1*time.Second), wait.WithImmediate(), wait.WithTimeout(1*time.Minute)) if err != nil { - t.Error(err) - t.FailNow() + t.Fatal("failed waiting for pods to be deleted", err) } - } + }) return ctx, nil }) @@ -124,6 +124,7 @@ func TestMain(m *testing.M) { pn := &networkv1beta1.PodNetworking{} pn.Name = "trunk" + pn.Spec.ENIOptions = networkv1beta1.ENIOptions{ENIAttachType: networkv1beta1.ENIOptionTypeTrunk} _ = config.Client().Resources().Delete(ctx, pn) return ctx, nil @@ -223,9 +224,10 @@ func setPodNetworking(ctx context.Context, config *envconf.Config) (context.Cont } pn := &networkv1beta1.PodNetworking{} - pn.Name = "trunk" + pn.Name = defaultPodNetworkingName + pn.Spec.ENIOptions = networkv1beta1.ENIOptions{ENIAttachType: networkv1beta1.ENIOptionTypeTrunk} pn.Spec.Selector = networkv1beta1.Selector{PodSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"trunk": "enable"}, + MatchLabels: map[string]string{"netplan": "default"}, }} if securityGroupIDs != "" { pn.Spec.SecurityGroupIDs = strings.Split(securityGroupIDs, ",") @@ -241,19 +243,7 @@ func setPodNetworking(ctx context.Context, config *envconf.Config) (context.Cont } } - err = wait.For(func(ctx context.Context) (bool, error) { - pn := &networkv1beta1.PodNetworking{} - err := config.Client().Resources().Get(ctx, "trunk", "", pn) - if err != nil { - return false, err - } - if pn.Status.Status != networkv1beta1.NetworkingStatusReady { - return false, nil - } - return true, nil - }, - wait.WithTimeout(10*time.Second), - wait.WithInterval(1*time.Second)) + err = WaitPodNetworkingReady(pn.Name, config.Client()) return ctx, err } @@ -269,6 +259,7 @@ func patchNamespace(ctx context.Context, config *envconf.Config) (context.Contex "labels": map[string]interface{}{ "ns": config.Namespace(), "node-local-dns-injection": "enabled", + "ns-trunking": "true", }, }, }) diff --git a/tests/stress/setup_test.go b/tests/stress/setup_test.go index 6f2d4220..90cafd64 100644 --- a/tests/stress/setup_test.go +++ b/tests/stress/setup_test.go @@ -8,12 +8,13 @@ import ( "path/filepath" "testing" - "github.com/AliyunContainerService/terway/tests/utils" "k8s.io/client-go/kubernetes/scheme" clientgoscheme "k8s.io/client-go/kubernetes/scheme" "sigs.k8s.io/e2e-framework/pkg/env" "sigs.k8s.io/e2e-framework/pkg/envconf" "sigs.k8s.io/e2e-framework/pkg/envfuncs" + + "github.com/AliyunContainerService/terway/tests/utils" ) var ( diff --git a/tests/trunk/perf_test.go b/tests/trunk/perf_test.go deleted file mode 100644 index 7c4de27b..00000000 --- a/tests/trunk/perf_test.go +++ /dev/null @@ -1,87 +0,0 @@ -//go:build e2e - -package trunk - -import ( - "context" - "testing" - "time" - - "github.com/AliyunContainerService/terway/pkg/apis/network.alibabacloud.com/v1beta1" - "github.com/AliyunContainerService/terway/pkg/utils/k8sclient" - terwayTypes "github.com/AliyunContainerService/terway/types" - - "github.com/google/uuid" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - ctrl "sigs.k8s.io/controller-runtime" -) - -func Test_10KPod(t *testing.T) { - restConf := ctrl.GetConfigOrDie() - ns := "perf" - k8sclient.RegisterClients(restConf) - ctx := context.Background() - - _, _ = k8sclient.K8sClient.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{Name: ns, Labels: map[string]string{"trunk": "trunk", "perf": "perf"}}, - }, metav1.CreateOptions{}) - - pn := newPodNetworking("pn", nil, nil, nil, - &metav1.LabelSelector{MatchLabels: map[string]string{"trunk": "trunk"}}) - _, _ = k8sclient.NetworkClient.NetworkV1beta1().PodNetworkings().Create(ctx, pn, metav1.CreateOptions{}) - - for i := 0; i < 10000; i++ { - name := uuid.NewString() - peni, err := k8sclient.NetworkClient.NetworkV1beta1().PodENIs(ns).Create(ctx, &v1beta1.PodENI{ - ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, Labels: map[string]string{ - terwayTypes.PodNetworking: "pn", - terwayTypes.PodENI: "true", - }}, - Spec: v1beta1.PodENISpec{ - Allocations: []v1beta1.Allocation{ - { - AllocationType: v1beta1.AllocationType{}, - ENI: v1beta1.ENI{ - ID: uuid.NewString(), - MAC: uuid.NewString(), - Zone: "", - VSwitchID: uuid.NewString(), - SecurityGroupIDs: []string{uuid.NewString()}, - }, - IPv4: "127.0.0.1", - IPv6: "::1", - IPv4CIDR: "127.0.0.1/8", - IPv6CIDR: "::1/8", - Interface: "eth0", - DefaultRoute: false, - }, - }, - Zone: "cn-hangzhou-x", - }, - }, metav1.CreateOptions{}) - - if err != nil { - t.Error(err) - continue - } - update := peni.DeepCopy() - update.Status.Phase = v1beta1.ENIPhaseBind - update.Status.PodLastSeen = metav1.NewTime(time.Now().Add(time.Hour)) - _, _ = k8sclient.NetworkClient.NetworkV1beta1().PodENIs(update.Namespace).UpdateStatus(ctx, update, metav1.UpdateOptions{}) - - _, _ = k8sclient.K8sClient.CoreV1().Pods(ns).Create(ctx, &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns}, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "foo", - Image: "registry.cn-hangzhou.aliyuncs.com/acs/pause:3.2", - ImagePullPolicy: corev1.PullIfNotPresent, - Command: []string{"/pause"}, - }, - }, - }, - }, metav1.CreateOptions{}) - } -} diff --git a/tests/trunk/setup_test.go b/tests/trunk/setup_test.go deleted file mode 100644 index 881de291..00000000 --- a/tests/trunk/setup_test.go +++ /dev/null @@ -1,87 +0,0 @@ -//go:build e2e - -package trunk - -import ( - "context" - "encoding/json" - "flag" - "os" - "path/filepath" - "testing" - - networkv1beta1 "github.com/AliyunContainerService/terway/pkg/apis/network.alibabacloud.com/v1beta1" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/kubernetes/scheme" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" - "sigs.k8s.io/e2e-framework/klient/k8s" - "sigs.k8s.io/e2e-framework/pkg/env" - "sigs.k8s.io/e2e-framework/pkg/envconf" - "sigs.k8s.io/e2e-framework/pkg/envfuncs" -) - -var ( - testenv env.Environment -) - -const ( - defaultPodNetworkingName = "default-config" - podName = "pod-use-trunking" -) - -var ( - vSwitchIDs string - securityGroupIDs string -) - -func init() { - flag.StringVar(&vSwitchIDs, "vswitch-ids", "", "extra vSwitchIDs") - flag.StringVar(&securityGroupIDs, "security-group-ids", "", "extra securityGroupIDs") -} - -func TestMain(m *testing.M) { - home, err := os.UserHomeDir() - if err != nil { - panic("error get home path") - } - - envCfg := envconf.NewWithKubeConfig(filepath.Join(home, ".kube", "config")). - WithRandomNamespace() - - testenv = env.NewWithConfig(envCfg) - - _ = clientgoscheme.AddToScheme(scheme.Scheme) - _ = networkv1beta1.AddToScheme(scheme.Scheme) - - testenv.Setup( - envfuncs.CreateNamespace(envCfg.Namespace()), - func(ctx context.Context, config *envconf.Config) (context.Context, error) { - ns := &corev1.Namespace{} - err := config.Client().Resources().Get(ctx, config.Namespace(), "", ns) - if err != nil { - return nil, err - } - mergePatch, _ := json.Marshal(map[string]interface{}{ - "metadata": map[string]interface{}{ - "labels": map[string]interface{}{ - "trunking-pod": "true", - }, - }, - }) - err = config.Client().Resources().Patch(ctx, ns, k8s.Patch{PatchType: types.StrategicMergePatchType, Data: mergePatch}) - return ctx, err - }, - func(ctx context.Context, config *envconf.Config) (context.Context, error) { - _ = config.Client().Resources().Delete(ctx, newPodNetworking(defaultPodNetworkingName, nil, nil, nil, nil)) - return ctx, nil - }, - ) - - testenv.Finish( - envfuncs.DeleteNamespace(envCfg.Namespace()), - ) - - os.Exit(testenv.Run(m)) -} diff --git a/tests/trunk/trunk_test.go b/tests/trunk_test.go similarity index 50% rename from tests/trunk/trunk_test.go rename to tests/trunk_test.go index 5addede2..892ca7a5 100644 --- a/tests/trunk/trunk_test.go +++ b/tests/trunk_test.go @@ -1,23 +1,13 @@ //go:build e2e -package trunk +package tests import ( "context" "fmt" - "strings" "testing" "time" - appsv1 "k8s.io/api/apps/v1" - "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/util/intstr" - - "github.com/AliyunContainerService/terway/pkg/apis/network.alibabacloud.com/v1beta1" - "github.com/AliyunContainerService/terway/tests/utils" - terwayTypes "github.com/AliyunContainerService/terway/types" - "github.com/AliyunContainerService/terway/types/controlplane" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/e2e-framework/klient" @@ -26,55 +16,64 @@ import ( "sigs.k8s.io/e2e-framework/klient/wait/conditions" "sigs.k8s.io/e2e-framework/pkg/envconf" "sigs.k8s.io/e2e-framework/pkg/features" + + "github.com/AliyunContainerService/terway/pkg/apis/network.alibabacloud.com/v1beta1" + "github.com/AliyunContainerService/terway/tests/utils" + terwayTypes "github.com/AliyunContainerService/terway/types" + "github.com/AliyunContainerService/terway/types/controlplane" ) -func TestDefaultConfigPodNetworking(t *testing.T) { +func TestCreatePodNetworking(t *testing.T) { + pnName := "pn" + podName := "test-pod" defaultConfig := features.New("PodNetworking/DefaultConfig").WithLabel("env", "trunking"). Setup(func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { - pn := newPodNetworking(defaultPodNetworkingName, nil, nil, &metav1.LabelSelector{ + err := WaitPodNetworkingDeleted(pnName, config.Client()) + if err != nil { + t.Fatal(err) + } + pn := newPodNetworking(pnName, nil, nil, &metav1.LabelSelector{ MatchLabels: map[string]string{"trunking-pod": "true"}, }, nil) - if err := config.Client().Resources().Create(ctx, pn); err != nil { + if err = config.Client().Resources().Create(ctx, pn); err != nil { t.Fatal(err) } + ctx = SaveResources(ctx, pn) return ctx }). Assess("with default config", func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { var pn v1beta1.PodNetworking - err := config.Client().Resources().Get(ctx, defaultPodNetworkingName, config.Namespace(), &pn) + err := config.Client().Resources().Get(ctx, pnName, config.Namespace(), &pn) if err != nil { t.Fatal(err) } if len(pn.Spec.VSwitchOptions) == 0 { - t.Errorf("vSwitchOptions not set") + t.Fatal("vSwitchOptions not set") } if len(pn.Spec.SecurityGroupIDs) == 0 { - t.Errorf("securityGroupIDs not set") + t.Fatal("securityGroupIDs not set") } return ctx - }). - Teardown(func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { - pn := &v1beta1.PodNetworking{ - ObjectMeta: metav1.ObjectMeta{Name: defaultPodNetworkingName, Namespace: config.Namespace()}, - } - _ = config.Client().Resources().Delete(ctx, pn) - return ctx - }). - Feature() + }).Feature() defaultVSwitch := features.New("PodNetworking/DefaultVSwitch").WithLabel("env", "trunking"). Setup(func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { - pn := newPodNetworking(defaultPodNetworkingName, []string{"foo"}, nil, &metav1.LabelSelector{ + err := WaitPodNetworkingDeleted(pnName, config.Client()) + if err != nil { + t.Fatal(err) + } + pn := newPodNetworking(pnName, []string{"foo"}, nil, &metav1.LabelSelector{ MatchLabels: map[string]string{"trunking-pod": "true"}, }, nil) - if err := config.Client().Resources().Create(ctx, pn); err != nil { + if err = config.Client().Resources().Create(ctx, pn); err != nil { t.Fatal(err) } + ctx = SaveResources(ctx, pn) return ctx }). Assess("vSwitchOptions is foo", func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { var pn v1beta1.PodNetworking - err := config.Client().Resources().Get(ctx, defaultPodNetworkingName, config.Namespace(), &pn) + err := config.Client().Resources().Get(ctx, pnName, config.Namespace(), &pn) if err != nil { t.Fatal(err) } @@ -89,28 +88,25 @@ func TestDefaultConfigPodNetworking(t *testing.T) { } return ctx }). - Teardown(func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { - pn := &v1beta1.PodNetworking{ - ObjectMeta: metav1.ObjectMeta{Name: defaultPodNetworkingName, Namespace: config.Namespace()}, - } - _ = config.Client().Resources().Delete(ctx, pn) - return ctx - }). Feature() defaultSecurityGroup := features.New("PodNetworking/DefaultSecurityGroup").WithLabel("env", "trunking"). Setup(func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { - pn := newPodNetworking(defaultPodNetworkingName, nil, []string{"foo"}, &metav1.LabelSelector{ + err := WaitPodNetworkingDeleted(pnName, config.Client()) + if err != nil { + t.Fatal(err) + } + pn := newPodNetworking(pnName, nil, []string{"foo"}, &metav1.LabelSelector{ MatchLabels: map[string]string{"trunking-pod": "true"}, }, nil) - if err := config.Client().Resources().Create(ctx, pn); err != nil { + if err = config.Client().Resources().Create(ctx, pn); err != nil { t.Fatal(err) } return ctx }). Assess("SecurityGroup is foo", func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { var pn v1beta1.PodNetworking - err := config.Client().Resources().Get(ctx, defaultPodNetworkingName, config.Namespace(), &pn) + err := config.Client().Resources().Get(ctx, pnName, config.Namespace(), &pn) if err != nil { t.Fatal(err) } @@ -124,23 +120,16 @@ func TestDefaultConfigPodNetworking(t *testing.T) { t.Errorf("securityGroupIDs not equal foo") } return ctx - }). - Teardown(func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { - pn := &v1beta1.PodNetworking{ - ObjectMeta: metav1.ObjectMeta{Name: defaultPodNetworkingName, Namespace: config.Namespace()}, - } - _ = config.Client().Resources().Delete(ctx, pn) - return ctx - }). - Feature() + }).Feature() defaultAnnotationConfig := features.New("Annotation/DefaultAnnoConfig").WithLabel("env", "trunking"). Setup(func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { - p := newPod(config.Namespace(), podName, nil, map[string]string{terwayTypes.PodENI: "true"}) - err := config.Client().Resources().Create(ctx, p) + pod := newPod(config.Namespace(), podName, nil, map[string]string{"k8s.aliyun.com/pod-eni": "true"}) + err := config.Client().Resources().Create(ctx, pod) if err != nil { t.Error(err) } + ctx = SaveResources(ctx, pod) return ctx }). Assess("with default config", func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { @@ -158,22 +147,16 @@ func TestDefaultConfigPodNetworking(t *testing.T) { } return ctx }). - Teardown(func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { - pod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Name: podName, Namespace: config.Namespace()}, - } - _ = config.Client().Resources().Delete(ctx, pod) - return ctx - }). Feature() defaultVSwitchAnnotationConfig := features.New("Annotation/DefaultVSwitchAnnoConfig").WithLabel("env", "trunking"). Setup(func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { - p := newPod(config.Namespace(), podName, nil, map[string]string{terwayTypes.PodENI: "true", terwayTypes.PodNetworks: "{\"podNetworks\":[{\"vSwitchOptions\":[\"foo\"],\"interface\":\"eth0\"}]}"}) - err := config.Client().Resources().Create(ctx, p) + pod := newPod(config.Namespace(), podName, nil, map[string]string{"k8s.aliyun.com/pod-eni": "true", "k8s.aliyun.com/pod-networks": "{\"podNetworks\":[{\"vSwitchOptions\":[\"foo\"],\"interface\":\"eth0\"}]}"}) + err := config.Client().Resources().Create(ctx, pod) if err != nil { t.Error(err) } + ctx = SaveResources(ctx, pod) return ctx }). Assess("vSwitchOptions is foo", func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { @@ -197,142 +180,110 @@ func TestDefaultConfigPodNetworking(t *testing.T) { } return ctx }). - Teardown(func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { - pod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Name: podName, Namespace: config.Namespace()}, - } - _ = config.Client().Resources().Delete(ctx, pod) - return ctx - }). Feature() testenv.Test(t, defaultConfig, defaultVSwitch, defaultSecurityGroup, defaultAnnotationConfig, defaultVSwitchAnnotationConfig) } func TestSelector(t *testing.T) { + pnName := "pn" podSelector := features.New("PodNetworking/PodSelector").WithLabel("env", "trunking"). Setup(func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { - pn := newPodNetworking(defaultPodNetworkingName, nil, nil, &metav1.LabelSelector{ - MatchLabels: map[string]string{"trunking-pod": "true"}, + pn := newPodNetworking(pnName, nil, nil, &metav1.LabelSelector{ + MatchLabels: map[string]string{"netplan": pnName}, }, nil) if err := config.Client().Resources().Create(ctx, pn); err != nil { t.Fatal(err) } - return ctx - }). - Assess("podNetworking status ready", func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { - err := WaitPodNetworkingReady(defaultPodNetworkingName, config.Client()) + t.Logf("podNetworking %s %s", pnName, pn.UID) + + err := WaitPodNetworkingReady(pnName, config.Client()) if err != nil { t.Fatal(err) } + t.Logf("podNetworking %s status ready", pnName) + + ctx = SaveResources(ctx, pn) return ctx }). Setup(func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { - p := newPod("default", "pod-use-trunking", map[string]string{"trunking-pod": "true"}, nil) - err := config.Client().Resources().Create(ctx, p) + pod := newPod(config.Namespace(), "pod-default-pn", map[string]string{"netplan": pnName}, nil) + err := config.Client().Resources().Create(ctx, pod) if err != nil { t.Error(err) } + t.Logf("pod %s %s created", pod.Name, pod.UID) + + ctx = SaveResources(ctx, pod) return ctx }). Assess("pod have trunking config", func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { - pod := corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Name: "pod-use-trunking", Namespace: "default"}, - } - err := wait.For(conditions.New(config.Client().Resources()).ResourceMatch(&pod, func(object k8s.Object) bool { - p := object.(*corev1.Pod) - if !terwayTypes.PodUseENI(p) { - return false - } - if p.Annotations[terwayTypes.PodNetworking] != defaultPodNetworkingName { - return false - } - - return true - }), wait.WithTimeout(time.Second*5)) + err := WaitPodHaveValidateConfig(config.Namespace(), "pod-default-pn", config.Client(), pnName) if err != nil { t.Fatal(err) } return ctx }). - Teardown(func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { - p := newPod("default", podName, map[string]string{"trunking-pod": "true"}, nil) - _ = config.Client().Resources().Delete(ctx, p) - pn := &v1beta1.PodNetworking{ - ObjectMeta: metav1.ObjectMeta{Name: defaultPodNetworkingName, Namespace: config.Namespace()}, - } - _ = config.Client().Resources().Delete(ctx, pn) - return ctx - }). Feature() nsSelector := features.New("PodNetworking/NamespaceSelector").WithLabel("env", "trunking"). Setup(func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { - pn := newPodNetworking(defaultPodNetworkingName, nil, nil, nil, &metav1.LabelSelector{ - MatchLabels: map[string]string{"trunking-pod": "true"}, + pn := newPodNetworking(pnName, nil, nil, nil, &metav1.LabelSelector{ + MatchLabels: map[string]string{"ns-trunking": "true"}, }) if err := config.Client().Resources().Create(ctx, pn); err != nil { t.Fatal(err) } - return ctx - }). - Assess("podNetworking status ready", func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { - err := WaitPodNetworkingReady(defaultPodNetworkingName, config.Client()) + err := WaitPodNetworkingReady(pnName, config.Client()) if err != nil { t.Fatal(err) } + ctx = SaveResources(ctx, pn) return ctx }). Setup(func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { - p := newPod(config.Namespace(), "any-pod", nil, nil) - err := config.Client().Resources().Create(ctx, p) + pod := newPod(config.Namespace(), "any-pod", nil, nil) + err := config.Client().Resources().Create(ctx, pod) if err != nil { t.Error(err) } + ctx = SaveResources(ctx, pod) return ctx }). Assess("pod have trunking config", func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { - err := WaitPodHaveValidateConfig(config.Namespace(), "any-pod", config.Client(), defaultPodNetworkingName) + err := WaitPodHaveValidateConfig(config.Namespace(), "any-pod", config.Client(), pnName) if err != nil { - t.Error(err) + t.Fatal(err) } return ctx }). Setup(func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { - p := newPod("default", "different-ns", nil, nil) - err := config.Client().Resources().Create(ctx, p) + pod := newPod("default", "different-ns", nil, nil) + err := config.Client().Resources().Create(ctx, pod) if err != nil { - t.Error(err) + t.Fatal(err) } + ctx = SaveResources(ctx, pod) return ctx }). Assess("default ns pod should not using trunking", func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { var pod corev1.Pod err := config.Client().Resources().Get(ctx, "different-ns", "default", &pod) if err != nil { - t.Error(err) + t.Fatal(err) } if terwayTypes.PodUseENI(&pod) { - t.Error(fmt.Errorf("pod in namespace default should not use trunking")) + t.Fatal(fmt.Errorf("pod in namespace default should not use trunking")) } return ctx }). - Teardown(func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { - p1 := newPod(config.Namespace(), "any-pod", nil, nil) - p2 := newPod("default", "different-ns", nil, nil) - _ = config.Client().Resources().Delete(ctx, p1) - _ = config.Client().Resources().Delete(ctx, p2) - pn := &v1beta1.PodNetworking{ - ObjectMeta: metav1.ObjectMeta{Name: defaultPodNetworkingName, Namespace: config.Namespace()}, - } - _ = config.Client().Resources().Delete(ctx, pn) - return ctx - }). Feature() testenv.Test(t, podSelector, nsSelector) } func TestZoneLimit(t *testing.T) { + pnName := "pn" + podName := "test-pod" zoneLimit := features.New("PodNetworking/ZoneLimit").WithLabel("env", "trunking"). Setup(func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { podENI := &v1beta1.PodENI{ @@ -356,7 +307,7 @@ func TestZoneLimit(t *testing.T) { } t.Logf("podENI created %#v", podENI) - pn := newPodNetworking(defaultPodNetworkingName, nil, nil, &metav1.LabelSelector{ + pn := newPodNetworking(pnName, nil, nil, &metav1.LabelSelector{ MatchLabels: map[string]string{"trunking-pod": "true"}, }, nil) pn.Spec.AllocationType = v1beta1.AllocationType{ @@ -367,25 +318,25 @@ func TestZoneLimit(t *testing.T) { t.Fatal(err) } t.Logf("podNetworking created %#v", pn) - return ctx - }). - Assess("podNetworking status ready", func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { - err := WaitPodNetworkingReady(defaultPodNetworkingName, config.Client()) + + err := WaitPodNetworkingReady(pnName, config.Client()) if err != nil { t.Fatal(err) } - t.Logf("podNetworking %s status is ready", defaultPodNetworkingName) + t.Logf("podNetworking %s status is ready", pnName) + + ctx = SaveResources(ctx, podENI, pn) return ctx }). Setup(func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { - p := newPod(config.Namespace(), podName, map[string]string{"trunking-pod": "true"}, nil) - p.OwnerReferences = append(p.OwnerReferences, metav1.OwnerReference{ + pod := newPod(config.Namespace(), podName, map[string]string{"trunking-pod": "true"}, nil) + pod.OwnerReferences = append(pod.OwnerReferences, metav1.OwnerReference{ Kind: "StatefulSet", Name: "foo", UID: "foo", APIVersion: "foo", }) - p.Spec.Affinity = &corev1.Affinity{ + pod.Spec.Affinity = &corev1.Affinity{ NodeAffinity: &corev1.NodeAffinity{ RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ NodeSelectorTerms: []corev1.NodeSelectorTerm{ @@ -418,11 +369,13 @@ func TestZoneLimit(t *testing.T) { }, } - err := config.Client().Resources().Create(ctx, p) + err := config.Client().Resources().Create(ctx, pod) if err != nil { t.Fatal(err) } - t.Logf("pod created %#v", p) + t.Logf("pod created %#v", pod) + + ctx = SaveResources(ctx, pod) return ctx }). Assess("pod have NodeSelectorTerms", func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { @@ -434,12 +387,12 @@ func TestZoneLimit(t *testing.T) { if !terwayTypes.PodUseENI(p) { return false } - if p.Annotations[terwayTypes.PodNetworking] != defaultPodNetworkingName { + if p.Annotations[terwayTypes.PodNetworking] != pnName { return false } return true - }), wait.WithTimeout(time.Second*5)) + }), wait.WithTimeout(time.Second*50)) if err != nil { t.Fatal(err) } @@ -458,29 +411,20 @@ func TestZoneLimit(t *testing.T) { } return ctx }). - Teardown(func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { - _ = config.Client().Resources().Delete(ctx, &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Name: podName, Namespace: config.Namespace()}, - }) - _ = config.Client().Resources().Delete(ctx, &v1beta1.PodNetworking{ - ObjectMeta: metav1.ObjectMeta{Name: defaultPodNetworkingName, Namespace: config.Namespace()}, - }) - _ = config.Client().Resources().Delete(ctx, &v1beta1.PodENI{ - ObjectMeta: metav1.ObjectMeta{Name: podName, Namespace: config.Namespace()}, - }) - - return ctx - }). Feature() testenv.Test(t, zoneLimit) } func TestFixedIP(t *testing.T) { + pnName := "fixed-ip" fixedIP := features.New("FixedIP").WithLabel("env", "trunking"). Setup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - _ = cfg.Client().Resources().Delete(ctx, &v1beta1.PodNetworking{ObjectMeta: metav1.ObjectMeta{Name: "fixed-ip"}}) - pn := newPodNetworking("fixed-ip", nil, nil, &metav1.LabelSelector{ - MatchLabels: map[string]string{"fixed-ip": "true"}, + err := WaitPodNetworkingDeleted(pnName, cfg.Client()) + if err != nil { + t.Fatal(err) + } + pn := newPodNetworking(pnName, nil, nil, &metav1.LabelSelector{ + MatchLabels: map[string]string{"netplan": "fixedip"}, }, nil) pn.Spec.AllocationType = v1beta1.AllocationType{ Type: v1beta1.IPAllocTypeFixed, @@ -490,45 +434,34 @@ func TestFixedIP(t *testing.T) { if err := cfg.Client().Resources().Create(ctx, pn); err != nil { t.Error(err) } - err := WaitPodNetworkingReady("fixed-ip", cfg.Client()) + err = WaitPodNetworkingReady(pnName, cfg.Client()) if err != nil { t.Fatal(err) } + ctx = SaveResources(ctx, pn) return ctx }). Setup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - ports := []corev1.ServicePort{ - { - Name: "http", - Protocol: corev1.ProtocolTCP, - Port: int32(80), - TargetPort: intstr.FromInt(80), - }, - } + for _, args := range [][]interface{}{ { "sts-1", "connective-test", - "l1b0k/echo:v0.0.1", + nginxImage, }, { "sts-2", "connective-test", - "l1b0k/echo:v0.0.1", + nginxImage, }, } { sts := utils.NewSts(args[0].(string), cfg.Namespace(), args[1].(string), args[2].(string), 1) - sts.Sts.Spec.Template.Labels["fixed-ip"] = "true" + sts.Sts.Spec.Template.Labels["netplan"] = "fixedip" err := cfg.Client().Resources().Create(ctx, sts.Sts) if err != nil { t.Fatal(err) } - svc := sts.Expose("") - svc.Spec.Ports = ports - err = cfg.Client().Resources().Create(ctx, svc) - if err != nil { - t.Fatal(err) - } + ctx = SaveResources(ctx, sts.Sts) } return ctx @@ -543,7 +476,7 @@ func TestFixedIP(t *testing.T) { if !terwayTypes.PodUseENI(p) { return false } - if p.Annotations[terwayTypes.PodNetworking] != "fixed-ip" { + if p.Annotations[terwayTypes.PodNetworking] != pnName { return false } if p.Status.Phase != corev1.PodRunning { @@ -558,34 +491,6 @@ func TestFixedIP(t *testing.T) { } return ctx }). - Assess("test connective", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - pod := utils.NewPod("client", cfg.Namespace(), "client", "l1b0k/echo:v0.0.1") - pod.Pod.Spec.Containers[0].Command = []string{"/usr/bin/echo", "-mode", "client", "-cases", "dns://aliyun.com,http://sts-1,http://sts-2,tcp://100.100.100.200:80"} - pod.Pod.Labels["fixed-ip"] = "true" - pod.Pod.Spec.RestartPolicy = corev1.RestartPolicyNever - err := cfg.Client().Resources().Create(ctx, pod.Pod) - if err != nil { - t.Fatal(err) - } - p := corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Name: "client", Namespace: cfg.Namespace()}, - } - err = wait.For(conditions.New(cfg.Client().Resources()).ResourceMatch(&p, func(object k8s.Object) bool { - p := object.(*corev1.Pod) - switch p.Status.Phase { - case corev1.PodSucceeded: - return true - case corev1.PodFailed: - t.Fatal("pod status failed") - } - return false - }), wait.WithInterval(time.Second), wait.WithTimeout(time.Minute*2)) - if err != nil { - t.Fatal(err) - } - err = cfg.Client().Resources().Delete(ctx, pod.Pod) - return ctx - }). Assess("recreate pod", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { for _, name := range []string{"sts-1-0", "sts-2-0"} { pod := &corev1.Pod{ @@ -611,7 +516,7 @@ func TestFixedIP(t *testing.T) { if !p.DeletionTimestamp.IsZero() { return false } - if p.Annotations[terwayTypes.PodNetworking] != "fixed-ip" { + if p.Annotations[terwayTypes.PodNetworking] != pnName { return false } if p.Status.Phase != corev1.PodRunning { @@ -629,186 +534,21 @@ func TestFixedIP(t *testing.T) { } return ctx }). - Assess("re-test connective", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - pod := utils.NewPod("client", cfg.Namespace(), "client", "l1b0k/echo:v0.0.1") - pod.Pod.Spec.Containers[0].Command = []string{"/usr/bin/echo", "-mode", "client", "-cases", "dns://aliyun.com,http://sts-1,http://sts-2,tcp://100.100.100.200:80"} - pod.Pod.Labels["fixed-ip"] = "true" - pod.Pod.Spec.RestartPolicy = corev1.RestartPolicyNever - err := cfg.Client().Resources().Create(ctx, pod.Pod) - if err != nil { - t.Fatal(err) - } - p := corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Name: "client", Namespace: cfg.Namespace()}, - } - err = wait.For(conditions.New(cfg.Client().Resources()).ResourceMatch(&p, func(object k8s.Object) bool { - p := object.(*corev1.Pod) - switch p.Status.Phase { - case corev1.PodSucceeded: - return true - case corev1.PodFailed: - t.Fatal("pod status failed") - } - return false - }), wait.WithInterval(time.Second), wait.WithTimeout(time.Minute*2)) - if err != nil { - t.Fatal(err) - } - err = cfg.Client().Resources().Delete(ctx, pod.Pod) - if err != nil { - t.Fatal(err) - } - return ctx - }). - Teardown(func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { - _ = config.Client().Resources().Delete(ctx, &v1beta1.PodNetworking{ - ObjectMeta: metav1.ObjectMeta{Name: "fixed-ip"}, - }) - for _, arg := range []string{"sts-1", "sts-2"} { - _ = config.Client().Resources().Delete(ctx, &appsv1.StatefulSet{ - ObjectMeta: metav1.ObjectMeta{Name: arg, Namespace: config.Namespace()}, - }) - _ = config.Client().Resources().Delete(ctx, &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{Name: arg, Namespace: config.Namespace()}, - }) - } - return ctx - }).Feature() + Feature() testenv.Test(t, fixedIP) } -// TestConnective run test cover several cases. -// pod to pod ,pod to service, dns resolve -func TestConnective(t *testing.T) { - // this test requires at least 2 vSwitches 2 nodes - vsw := strings.Split(vSwitchIDs, ",") - if len(vsw) < 2 { - return +func WaitPodNetworkingDeleted(name string, client klient.Client) error { + pn := v1beta1.PodNetworking{ + ObjectMeta: metav1.ObjectMeta{Name: name}, } - crossVSwitch := features.New("Connective/MultiVSwitch").WithLabel("env", "trunking"). - Setup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - for _, args := range [][]interface{}{ - { - "use-vsw-1", - []string{vsw[0]}, - }, - { - "use-vsw-2", - []string{vsw[1]}, - }, - } { - pn := newPodNetworking(args[0].(string), args[1].([]string), nil, &metav1.LabelSelector{ - MatchLabels: map[string]string{args[0].(string): ""}, - }, nil) - err := cfg.Client().Resources().Create(ctx, pn) - if err != nil && !errors.IsAlreadyExists(err) { - t.Fatal(err) - } - } - return ctx - }). - Assess("podNetworking status ready", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - for _, arg := range []string{"use-vsw-1", "use-vsw-2"} { - err := WaitPodNetworkingReady(arg, cfg.Client()) - if err != nil { - t.Fatal(err) - } - } - return ctx - }). - Setup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - ports := []corev1.ServicePort{ - { - Name: "http", - Protocol: corev1.ProtocolTCP, - Port: int32(80), - TargetPort: intstr.FromInt(80), - }, - } - for _, args := range [][]interface{}{ - { - "pod-1", - "connective-test", - "l1b0k/echo:v0.0.1", - "use-vsw-1", - }, - { - "pod-2", - "connective-test", - "l1b0k/echo:v0.0.1", - "use-vsw-2", - }, - } { - pod := utils.NewPod(args[0].(string), cfg.Namespace(), args[1].(string), args[2].(string)) - pod.Pod.Labels[args[3].(string)] = "" - err := cfg.Client().Resources().Create(ctx, pod.Pod) - if err != nil { - t.Error(err) - } - svc := pod.Expose("") - svc.Spec.Ports = ports - err = cfg.Client().Resources().Create(ctx, svc) - if err != nil { - t.Error(err) - } - } - return ctx - }). - Assess("wait for pod ready", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - for _, name := range []string{"pod-1", "pod-2"} { - pod := corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: cfg.Namespace()}, - } - err := wait.For(conditions.New(cfg.Client().Resources()).ResourceMatch(&pod, func(object k8s.Object) bool { - p := object.(*corev1.Pod) - if !terwayTypes.PodUseENI(p) { - return false - } - return p.Status.Phase == corev1.PodRunning - }), wait.WithInterval(time.Second), wait.WithTimeout(time.Minute*2)) - if err != nil { - t.Fatal(err) - } - } - return ctx - }). - Assess("test connective", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - pod := utils.NewPod("client", cfg.Namespace(), "client", "l1b0k/echo:v0.0.1") - pod.Pod.Spec.Containers[0].Command = []string{"/usr/bin/echo", "-mode", "client", "-cases", "dns://aliyun.com,http://pod-1,http://pod-2,tcp://100.100.100.200:80"} - pod.Pod.Labels["use-vsw-1"] = "" - pod.Pod.Spec.RestartPolicy = corev1.RestartPolicyNever - err := cfg.Client().Resources().Create(ctx, pod.Pod) - if err != nil { - t.Fatal(err) - } - p := corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Name: "client", Namespace: cfg.Namespace()}, - } - err = wait.For(conditions.New(cfg.Client().Resources()).ResourceMatch(&p, func(object k8s.Object) bool { - p := object.(*corev1.Pod) - return p.Status.Phase == corev1.PodSucceeded - }), wait.WithInterval(time.Second), wait.WithTimeout(time.Minute*2)) - if err != nil { - t.Fatal(err) - } - err = cfg.Client().Resources().Delete(ctx, pod.Pod) - if err != nil { - t.Fatal(err) - } - return ctx - }). - Teardown(func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { - for _, arg := range []string{"use-vsw-1", "use-vsw-2"} { - pn1 := &v1beta1.PodNetworking{ - ObjectMeta: metav1.ObjectMeta{Name: arg}, - } - _ = config.Client().Resources().Delete(ctx, pn1) - } - return ctx - }).Feature() - - testenv.Test(t, crossVSwitch) + err := wait.For(conditions.New(client.Resources()).ResourceDeleted(&pn), + wait.WithImmediate(), + wait.WithInterval(1*time.Second), + wait.WithTimeout(30*time.Second), + ) + return err } func WaitPodNetworkingReady(name string, client klient.Client) error { @@ -826,16 +566,19 @@ func WaitPodNetworkingReady(name string, client klient.Client) error { } } return p.Status.Status == v1beta1.NetworkingStatusReady - }), wait.WithTimeout(time.Minute*1)) - time.Sleep(5 * time.Second) + }), + wait.WithImmediate(), + wait.WithInterval(1*time.Second), + wait.WithTimeout(30*time.Second), + ) return err } func WaitPodHaveValidateConfig(namespace, name string, client klient.Client, podNetworkingName string) error { - pod := corev1.Pod{ + pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, } - return wait.For(conditions.New(client.Resources()).ResourceMatch(&pod, func(object k8s.Object) bool { + err := wait.For(conditions.New(client.Resources()).ResourceMatch(pod, func(object k8s.Object) bool { p := object.(*corev1.Pod) if !terwayTypes.PodUseENI(p) { return false @@ -844,7 +587,19 @@ func WaitPodHaveValidateConfig(namespace, name string, client klient.Client, pod return false } return true - }), wait.WithTimeout(time.Second*5)) + }), + wait.WithImmediate(), + wait.WithInterval(1*time.Second), + wait.WithTimeout(10*time.Second), + ) + + if err != nil { + if pod != nil { + return fmt.Errorf("%w, Anno %#v, Labels %#v", err, pod.Annotations, pod.Labels) + } + return err + } + return nil } func newPodNetworking(name string, vSwitchOptions, securityGroupIDs []string, podSelector, namespaceSelector *metav1.LabelSelector) *v1beta1.PodNetworking { @@ -857,6 +612,9 @@ func newPodNetworking(name string, vSwitchOptions, securityGroupIDs []string, po }, VSwitchOptions: vSwitchOptions, SecurityGroupIDs: securityGroupIDs, + ENIOptions: v1beta1.ENIOptions{ + ENIAttachType: v1beta1.ENIOptionTypeTrunk, + }, }, } } diff --git a/tests/utils_test.go b/tests/utils_test.go index c7192d98..c56cced9 100644 --- a/tests/utils_test.go +++ b/tests/utils_test.go @@ -3,6 +3,7 @@ package tests import ( + "context" "net/netip" corev1 "k8s.io/api/core/v1" @@ -10,6 +11,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/sets" + "sigs.k8s.io/controller-runtime/pkg/client" ) type Pod struct { @@ -250,3 +252,19 @@ func getIP(pod *corev1.Pod) (v4 netip.Addr, v6 netip.Addr) { } return } + +type resourceKey struct{} + +func ResourcesFromCtx(ctx context.Context) []client.Object { + v, ok := ctx.Value(resourceKey{}).([]client.Object) + if !ok { + return nil + } + return v +} + +func SaveResources(ctx context.Context, resources ...client.Object) context.Context { + prev := ResourcesFromCtx(ctx) + prev = append(prev, resources...) + return context.WithValue(ctx, resourceKey{}, prev) +}