Sonobuoy Conformance Results

.gitignore

@@ -11,3 +11,5 @@ workloads/*/dynamic-ingressroute.yaml
 /config/*.env.*
 /config/*.tfvars
 tmp/
+entrypoint/conformance/my-sonobuoy-plugins
+entrypoint/conformance/*.tar.gz

Makefile

@@ -71,6 +71,9 @@ test-upgrade-manual:
 test-upgrade-node-replacement:
 	@go test -timeout=120m -v -tags=upgradereplacement -count=1 ./entrypoint/upgradecluster/... -installVersionOrCommit ${INSTALL_VERSION_OR_COMMIT} -channel ${CHANNEL}
 
+test-run-sonobuoy:
+	@go test -timeout=170m -v -count=1 ./entrypoint/conformance/... $(if ${SONOBUOY_VERSION},-sonobuoyVersion ${SONOBUOY_VERSION}) --ginkgo.timeout=170m
+
 test-create-mixedos:
 	@go test -timeout=45m -v -count=1 ./entrypoint/mixedoscluster/... $(if ${SONOBUOY_VERSION},-sonobuoyVersion ${SONOBUOY_VERSION})
 
@@ -125,3 +128,4 @@ go-check:
 shell-check:
 	@shellcheck modules/airgap/setup/*.sh
 	@shellcheck modules/ipv6only/scripts/*.sh
+	@shellcheck scripts/*.sh

docs/testing.md

@@ -284,4 +284,21 @@ S3_FOLDER=snapshots
 ### Not supported/implemented currently for cluster restore:
 - Hardened Cluster Setup
 - ExternalDB Setup
-- Selinux Setup
+- Selinux Setup
+
+## Validating Conformance Tests with Sonobuoy
+ - Please note that the sonobuoy version has not been updated for a year and the functionality of sonobuoy is degrading with minor versions of k8s.
+ - Full conformance tests done for patch validations should be 3 servers 1 agent minimum.
+ - You can use the make file command `make test-run-sonobuoy` to run the conformance tests.
+ - Additionally you can use `go test -timeout=140m -v -count=1 ./entrypoint/conformance/... --ginkgo.timeout=140m` you must extend the ginkgo timeout in addition to the go test timeout 
+ - Required vars in `*.tfvars` file minimum conformance configuration: 
+```
+no_of_server_nodes = 1
+no_of_worker_nodes = 1
+```
+- sonobuoy's output is becoming unreliable for status checks observe remaining count incorrect at 404.
+ 	sono status
+          PLUGIN     STATUS   RESULT   COUNT                                PROGRESS
+             e2e   complete   passed       1   Passed:  0, Failed:  0, Remaining:404
+    systemd-logs   complete   passed       2
+ Sonobuoy has completed. Use `sonobuoy retrieve` to get results

entrypoint/conformance/conformance_suite_test.go

@@ -0,0 +1,99 @@
+package sonobuoyconformance
+
+import (
+	"flag"
+	"os"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/rancher/distros-test-framework/config"
+	"github.com/rancher/distros-test-framework/pkg/customflag"
+	"github.com/rancher/distros-test-framework/pkg/qase"
+	"github.com/rancher/distros-test-framework/shared"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var (
+	qaseReport = os.Getenv("REPORT_TO_QASE")
+	kubeconfig string
+	cluster    *shared.Cluster
+	cfg        *config.Env
+	err        error
+)
+
+func TestMain(m *testing.M) {
+	flag.StringVar(&customflag.ServiceFlag.External.SonobuoyVersion, "sonobuoyVersion", "0.57.2", "Sonobuoy binary version")
+	flag.Var(&customflag.ServiceFlag.Destroy, "destroy", "Destroy cluster after test")
+	flag.Parse()
+
+	verifyClusterNodes()
+
+	cfg, err = config.AddEnv()
+	if err != nil {
+		shared.LogLevel("error", "error adding env vars: %w\n", err)
+		os.Exit(1)
+	}
+
+	kubeconfig = os.Getenv("KUBE_CONFIG")
+	if kubeconfig == "" {
+		// gets a cluster from terraform.
+		cluster = shared.ClusterConfig(cfg)
+	} else {
+		// gets a cluster from kubeconfig.
+		cluster = shared.KubeConfigCluster(kubeconfig)
+	}
+
+	os.Exit(m.Run())
+	os.Exit(1)
+}
+
+func TestConformance(t *testing.T) {
+	RegisterFailHandler(Fail)
+
+	RunSpecs(t, "Run Conformance Suite")
+}
+
+var _ = ReportAfterSuite("Conformance Suite", func(report Report) {
+
+	if strings.ToLower(qaseReport) == "true" {
+		qaseClient, err := qase.AddQase()
+		Expect(err).ToNot(HaveOccurred(), "error adding qase")
+
+		qaseClient.SpecReportTestResults(qaseClient.Ctx, &report, cfg.InstallVersion)
+	} else {
+		shared.LogLevel("info", "Qase reporting is not enabled")
+	}
+})
+
+var _ = AfterSuite(func() {
+	if customflag.ServiceFlag.Destroy {
+		status, err := shared.DestroyCluster(cfg)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(status).To(Equal("cluster destroyed"))
+	}
+})
+
+func verifyClusterNodes() {
+	// if re-running locally the env variables are not set after cleanup
+	shared.LogLevel("info", "verying cluster configuration matches minimum requirements for conformance tests")
+	serverNum, err := strconv.Atoi(os.Getenv("no_of_server_nodes"))
+	if err != nil {
+		shared.LogLevel("error", "error converting no_of_server_nodes to int: %w", err)
+		os.Exit(1)
+	}
+
+	agentNum, _ := strconv.Atoi(os.Getenv("no_of_agent_nodes"))
+	if err != nil {
+		shared.LogLevel("error", "error converting no_of_agent_nodes to int: %w", err)
+		os.Exit(1)
+	}
+
+	if serverNum < 1 && agentNum < 1 {
+		shared.LogLevel("error", "%s", "cluster must at least consist of 1 server and 1 agent")
+		os.Exit(1)
+	}
+
+}

entrypoint/conformance/sonobuoy_test.go

@@ -0,0 +1,44 @@
+package sonobuoyconformance
+
+import (
+	"fmt"
+
+	"github.com/rancher/distros-test-framework/pkg/assert"
+	"github.com/rancher/distros-test-framework/pkg/testcase"
+
+	. "github.com/onsi/ginkgo/v2"
+)
+
+var _ = Describe("Sonobuoy Conformance Tests...", func() {
+
+	It("Starts Up with no issues", func() {
+		testcase.TestBuildCluster(cluster)
+	})
+
+	It("Validates Node", func() {
+		testcase.TestNodeStatus(
+			cluster,
+			assert.NodeAssertReadyStatus(),
+			nil,
+		)
+	})
+
+	It("Validate Pods", func() {
+		testcase.TestPodStatus(
+			cluster,
+			assert.PodAssertRestart(),
+			assert.PodAssertReady())
+	})
+
+	It("Validates the releases conformance with upstream requirements", func() {
+		testcase.ConformanceTest("certified-conformance")
+	})
+})
+
+var _ = AfterEach(func() {
+	if CurrentSpecReport().Failed() {
+		fmt.Printf("\nFAILED! %s\n\n", CurrentSpecReport().FullText())
+	} else {
+		fmt.Printf("\nPASSED! %s\n\n", CurrentSpecReport().FullText())
+	}
+})

pkg/testcase/cluster.go

@@ -4,11 +4,9 @@
 	"fmt"
 	"strings"
 
-	"github.com/rancher/distros-test-framework/pkg/customflag"
 	"github.com/rancher/distros-test-framework/pkg/testcase/support"
 	"github.com/rancher/distros-test-framework/shared"
 
-	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 )
 
@@ -50,35 +48,17 @@
 	}
 }
 
-// TestSonobuoyMixedOS runs sonobuoy tests for mixed os cluster (linux + windows) node.
-func TestSonobuoyMixedOS(deleteWorkload bool) {
-	sonobuoyVersion := customflag.ServiceFlag.External.SonobuoyVersion
-	err := shared.SonobuoyMixedOS("install", sonobuoyVersion)
-	Expect(err).NotTo(HaveOccurred())
+func checkAndPrintAgentNodeIPs(agentNum int, agentIPs []string, isWindows bool) {
+	info := "Agent Node IPs:"
 
-	cmd := "sonobuoy run --kubeconfig=" + shared.KubeConfigFile +
-		" --plugin my-sonobuoy-plugins/mixed-workload-e2e/mixed-workload-e2e.yaml" +
-		" --aggregator-node-selector kubernetes.io/os:linux --wait"
-	res, err := shared.RunCommandHost(cmd)
-	Expect(err).NotTo(HaveOccurred(), "failed output: "+res)
-
-	cmd = "sonobuoy retrieve --kubeconfig=" + shared.KubeConfigFile
-	testResultTar, err := shared.RunCommandHost(cmd)
-	Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd)
-
-	cmd = "sonobuoy results  " + testResultTar
-	res, err = shared.RunCommandHost(cmd)
-	Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd)
-	Expect(res).Should(ContainSubstring("Plugin: mixed-workload-e2e\nStatus: passed\n"))
+	if isWindows {
+		info = "Windows " + info
+	}
 
-	if deleteWorkload {
-		cmd = "sonobuoy delete --all --wait --kubeconfig=" + shared.KubeConfigFile
-		_, err = shared.RunCommandHost(cmd)
-		Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd)
-		err = shared.SonobuoyMixedOS("delete", sonobuoyVersion)
-		if err != nil {
-			GinkgoT().Errorf("error: %v", err)
-			return
-		}
+	if agentNum > 0 {
+		Expect(agentIPs).ShouldNot(BeEmpty())
+		shared.LogLevel("info", info+"  %v", agentIPs)
+	} else {
+		Expect(agentIPs).Should(BeEmpty())
 	}
 }

pkg/testcase/conformance.go

@@ -0,0 +1,138 @@
+package testcase
+
+import (
+	"os"
+	"strings"
+
+	"github.com/rancher/distros-test-framework/shared"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+// TestSonobuoyMixedOS runs sonobuoy tests for mixed os cluster (linux + windows) node.
+func TestSonobuoyMixedOS(deleteWorkload bool) {
+	installConformanceBinary()
+
+	cmd := "sonobuoy run --kubeconfig=" + shared.KubeConfigFile +
+		" --plugin my-sonobuoy-plugins/mixed-workload-e2e/mixed-workload-e2e.yaml" +
+		" --aggregator-node-selector kubernetes.io/os:linux --wait"
+	res, err := shared.RunCommandHost(cmd)
+	Expect(err).NotTo(HaveOccurred(), "failed output: "+res)
+
+	cmd = "sonobuoy retrieve --kubeconfig=" + shared.KubeConfigFile
+	testResultTar, err := shared.RunCommandHost(cmd)
+	Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd)
+
+	cmd = "sonobuoy results  " + testResultTar
+	res, err = shared.RunCommandHost(cmd)
+	Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd)
+	Expect(res).Should(ContainSubstring("Plugin: mixed-workload-e2e\nStatus: passed\n"))
+
+	if deleteWorkload {
+		cmd = "sonobuoy delete --all --wait --kubeconfig=" + shared.KubeConfigFile
+		_, err = shared.RunCommandHost(cmd)
+		Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd)
+		err = shared.InstallSonobuoy("delete")
+		if err != nil {
+			GinkgoT().Errorf("error: %v", err)
+			return
+		}
+	}
+}
+
+func ConformanceTest(testName string) {
+	installConformanceBinary()
+	launchSonobuoyTests(testName)
+	checkStatus()
+	testResultTar := getResults()
+	shared.LogLevel("info", "%s", "testResultTar: "+testResultTar)
+	rerunFailedTests(testResultTar)
+	testResultTar = getResults()
+	shared.LogLevel("info", "%s", "testResultTar: "+testResultTar)
+	parseResults(testResultTar)
+	cleanupTests()
+}
+
+func installConformanceBinary() {
+	shared.LogLevel("info", "installing sonobuoy binary")
+	err := shared.InstallSonobuoy("install")
+	Expect(err).NotTo(HaveOccurred())
+}
+
+func launchSonobuoyTests(testMode string) {
+	shared.LogLevel("info", "checking namespace existence")
+	cmds := "kubectl get namespace sonobuoy --kubeconfig=" + shared.KubeConfigFile
+	res, _ := shared.RunCommandHost(cmds)
+
+	if strings.Contains(res, "Active") {
+		shared.LogLevel("info", "%s", "sonobuoy namespace is active, waiting for it to complete")
+		return
+	}
+
+	if strings.Contains(res, "Error from server (NotFound): namespaces \"sonobuoy\" not found") {
+		cmd := "sonobuoy run --kubeconfig=" + shared.KubeConfigFile +
+			" --mode=" + testMode + " --kubernetes-version=" + shared.ExtractKubeImageVersion()
+		_, err := shared.RunCommandHost(cmd)
+		Expect(err).NotTo(HaveOccurred())
+	}
+}
+
+func checkStatus() {
+	shared.LogLevel("info", "checking status of running tests")
+	cmd := "sonobuoy status --kubeconfig=" + shared.KubeConfigFile
+	Eventually(func() string {
+		res, err := shared.RunCommandHost(cmd)
+		Expect(err).NotTo(HaveOccurred())
+		return res
+	}, "170m", "10m").Should(ContainSubstring("Sonobuoy has completed"), "timed out waiting for sonobuoy")
+}
+
+func getResults() string {
+	shared.LogLevel("info", "getting sonobuoy results")
+	cmd := "sonobuoy retrieve --kubeconfig=" + shared.KubeConfigFile
+	res, err := shared.RunCommandHost(cmd)
+	Expect(err).NotTo(HaveOccurred())
+
+	return res
+}
+
+func rerunFailedTests(testResultTar string) {
+	ciliumExpectedFailures := `
+		[sig-network] Services should serve endpoints on same port and different protocols
+	 	Services should be able to switch session affinity for service with type clusterIP
+		Services should have session affinity work for service with type clusterIP`
+
+	if strings.Contains(os.Getenv("cni"), "cilium") {
+		shared.LogLevel("info", "Cilium has known issues with conformance tests, skipping re-run")
+		shared.LogLevel("info", "ciliumExpectedFailures: %s", ciliumExpectedFailures)
+
+		return
+	}
+
+	shared.LogLevel("info", "re-running tests that failed from previous run")
+
+	cmd := "sonobuoy run --rerun-failed=" + testResultTar + " --kubeconfig=" + shared.KubeConfigFile +
+		"--kubernetes-version=" + shared.ExtractKubeImageVersion()
+
+	res, err := shared.RunCommandHost(cmd)
+	Expect(err).To(HaveOccurred(), "failed cmd: "+cmd)
+	Expect(res).Should(ContainSubstring("no tests failed for plugin"))
+}
+
+func parseResults(testResultTar string) {
+	shared.LogLevel("info", "parsing sonobuoy results")
+	cmd := "sonobuoy results  " + testResultTar
+	res, err := shared.RunCommandHost(cmd)
+	Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd)
+	Expect(res).Should(ContainSubstring("Status: passed"))
+	shared.LogLevel("info", "%s", "sonobuoy results: "+res)
+}
+
+func cleanupTests() {
+	shared.LogLevel("info", "cleaning up cluster conformance tests and deleting sonobuoy namespace")
+	cmd := "sonobuoy delete --all --wait --kubeconfig=" + shared.KubeConfigFile
+	res, err := shared.RunCommandHost(cmd)
+	Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd)
+	Expect(res).Should(ContainSubstring("deleted"))
+}