Update validation logic

actions/provisioning/verify.go

@@ -141,7 +141,7 @@ func VerifyClusterReady(t *testing.T, client *rancher.Client, cluster *steveV1.S
 		checkFunc := shepherdclusters.IsProvisioningClusterReady
 		err = wait.WatchWait(watchInterface, checkFunc)
 		if err != nil {
-			logrus.Warningf("Unable to get cluster status (%s): %v. Retrying", cluster.Name, err)
+			logrus.Warningf("Unable to get cluster status (%s) Retrying", cluster.Name)
 			return false, nil
 		}
 

actions/workloads/deployment/verify.go

@@ -4,12 +4,17 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"slices"
 	"strconv"
 	"time"
 
 	"github.com/rancher/shepherd/clients/rancher"
+	steveV1 "github.com/rancher/shepherd/clients/rancher/v1"
+	v1 "github.com/rancher/shepherd/clients/rancher/v1"
 	"github.com/rancher/shepherd/extensions/charts"
+	"github.com/rancher/shepherd/extensions/clusters"
 	"github.com/rancher/shepherd/extensions/defaults"
+	"github.com/rancher/shepherd/extensions/defaults/stevetypes"
 	"github.com/rancher/shepherd/extensions/workloads"
 	namegen "github.com/rancher/shepherd/pkg/namegenerator"
 	"github.com/rancher/shepherd/pkg/wrangler"
@@ -22,6 +27,10 @@ import (
 )
 
 const (
+	Webhook            = "rancher-webhook"
+	SUC                = "system-upgrade-controller"
+	Fleet              = "fleet-agent"
+	ClusterAgent       = "cattle-cluster-agent"
 	revisionAnnotation = "deployment.kubernetes.io/revision"
 )
 
@@ -490,3 +499,88 @@ func VerifyDeploymentOrchestration(client *rancher.Client, clusterID, namespace,
 
 	return err
 }
+
+// VerifyClusterDeployments verifies that all required deployments are present and available in the cluster
+func VerifyClusterDeployments(client *rancher.Client, cluster *v1.SteveAPIObject) error {
+	clusterID, err := clusters.GetClusterIDByName(client, cluster.Name)
+	if err != nil {
+		return err
+	}
+
+	downstreamClient, err := client.Steve.ProxyDownstream(clusterID)
+	if err != nil {
+		return err
+	}
+	if downstreamClient == nil {
+		return errors.New("downstream client is nil")
+	}
+
+	deploymentClient := downstreamClient.SteveType(stevetypes.Deployment)
+	requiredDeployments := []string{ClusterAgent, Webhook, Fleet, SUC}
+
+	logrus.Debugf("Verifying all required deployments exist: %v", requiredDeployments)
+	err = kwait.PollUntilContextTimeout(context.TODO(), 5*time.Second, defaults.TenMinuteTimeout, true, func(ctx context.Context) (done bool, err error) {
+		clusterDeployments, err := deploymentClient.List(nil)
+		if err != nil {
+			return false, nil
+		}
+
+		for _, deployment := range clusterDeployments.Data {
+			k8sDeployment := &appv1.Deployment{}
+			err := steveV1.ConvertToK8sType(deployment.JSONResp, k8sDeployment)
+			if err != nil {
+				return false, nil
+			}
+
+			if slices.Contains(requiredDeployments, k8sDeployment.Name) {
+				requiredDeployments = slices.Delete(requiredDeployments, slices.Index(requiredDeployments, k8sDeployment.Name), slices.Index(requiredDeployments, k8sDeployment.Name)+1)
+			}
+		}
+		if len(requiredDeployments) != 0 {
+			return false, nil
+		}
+
+		return true, nil
+	})
+
+	if err != nil {
+		return fmt.Errorf("Not all required deployments exist: %v", requiredDeployments)
+	}
+
+	logrus.Debug("Verifying all deployments")
+	var failedDeployments []appv1.Deployment
+	err = kwait.PollUntilContextTimeout(context.TODO(), 5*time.Second, defaults.TenMinuteTimeout, true, func(ctx context.Context) (done bool, err error) {
+		clusterDeployments, err := deploymentClient.List(nil)
+		if err != nil {
+			return false, nil
+		}
+
+		failedDeployments = []appv1.Deployment{}
+		for _, deploymentObj := range clusterDeployments.Data {
+			k8sDeployment := &appv1.Deployment{}
+			err := steveV1.ConvertToK8sType(deploymentObj.JSONResp, k8sDeployment)
+			if err != nil {
+				return false, nil
+			}
+
+			if k8sDeployment.Status.AvailableReplicas != *k8sDeployment.Spec.Replicas {
+				failedDeployments = append(failedDeployments, *k8sDeployment)
+			}
+		}
+
+		return true, nil
+	})
+
+	if len(failedDeployments) > 0 {
+		for _, deploymentObj := range failedDeployments {
+
+			for _, condition := range deploymentObj.Status.Conditions {
+				logrus.Error("Deployment:", deploymentObj.Name, "Condition: ", condition.Message)
+			}
+		}
+
+		return nil
+	}
+
+	return err
+}

actions/workloads/pods/verify.go

@@ -3,7 +3,6 @@ package pods
 import (
 	"context"
 	"errors"
-	"slices"
 	"strings"
 	"testing"
 	"time"
@@ -84,23 +83,6 @@ func VerifyClusterPods(client *rancher.Client, cluster *steveV1.SteveAPIObject)
 		}
 
 		steveClient := downstreamClient.SteveType(stevetypes.Pod)
-		deploymentClient := downstreamClient.SteveType(stevetypes.Deployment)
-		clusterDeployments, err := deploymentClient.List(nil)
-		if err != nil {
-			return false, nil
-		}
-
-		requiredDeployments := []string{ClusterAgent, Webhook, Fleet, SUC}
-		requiredDeploymentCount := 0
-		for _, deployment := range clusterDeployments.Data {
-			if slices.Contains(requiredDeployments, deployment.Name) {
-				logrus.Tracef("Deployment: %s exists", deployment.Name)
-				requiredDeploymentCount += 1
-			}
-		}
-		if requiredDeploymentCount != len(requiredDeployments) {
-			return false, nil
-		}
 
 		podErrors = []error{}
 

actions/workloads/verify.go

@@ -97,7 +97,15 @@ func VerifyWorkloads(client *rancher.Client, clusterName string, workloads Workl
 	}
 
 	err = pods.VerifyClusterPods(client, cluster)
+	if err != nil {
+		return nil, err
+	}
+
+	err = deployment.VerifyClusterDeployments(client, cluster)
+	if err != nil {
+		return nil, err
+	}
 
-	return &workloads, err
+	return &workloads, nil
 
 }

validation/auth/kubeconfigs/kubeconfigs_test.go

@@ -17,6 +17,7 @@ import (
 	"github.com/rancher/tests/actions/provisioning"
 	"github.com/rancher/tests/actions/rbac"
 	"github.com/rancher/tests/actions/settings"
+	"github.com/rancher/tests/actions/workloads/deployment"
 	"github.com/rancher/tests/actions/workloads/pods"
 	log "github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/require"
@@ -73,6 +74,10 @@ func (kc *ExtKubeconfigTestSuite) SetupSuite() {
 	require.NoError(kc.T(), err)
 
 	provisioning.VerifyClusterReady(kc.T(), client, aceClusterObject1)
+
+	err = deployment.VerifyClusterDeployments(client, aceClusterObject1)
+	require.NoError(kc.T(), err)
+
 	err = pods.VerifyClusterPods(client, aceClusterObject1)
 	require.NoError(kc.T(), err)
 	provisioning.VerifyDynamicCluster(kc.T(), client, aceClusterObject1)
@@ -81,6 +86,10 @@ func (kc *ExtKubeconfigTestSuite) SetupSuite() {
 	log.Infof("ACE-enabled cluster created: %s (%s)", kc.aceCluster1.Name, aceCluster1ID)
 
 	provisioning.VerifyClusterReady(kc.T(), client, aceClusterObject2)
+
+	err = deployment.VerifyClusterDeployments(client, aceClusterObject2)
+	require.NoError(kc.T(), err)
+
 	err = pods.VerifyClusterPods(client, aceClusterObject2)
 	require.NoError(kc.T(), err)
 	provisioning.VerifyDynamicCluster(kc.T(), client, aceClusterObject2)
@@ -89,6 +98,10 @@ func (kc *ExtKubeconfigTestSuite) SetupSuite() {
 	log.Infof("ACE-enabled cluster created: %s (%s)", kc.aceCluster2.Name, aceCluster2ID)
 
 	provisioning.VerifyClusterReady(kc.T(), client, clusterObject2)
+
+	err = deployment.VerifyClusterDeployments(client, clusterObject2)
+	require.NoError(kc.T(), err)
+
 	err = pods.VerifyClusterPods(client, clusterObject2)
 	require.NoError(kc.T(), err)
 	provisioning.VerifyDynamicCluster(kc.T(), client, clusterObject2)

validation/certificates/dualstack/cert_rotation_test.go

@@ -20,6 +20,7 @@ import (
 	"github.com/rancher/tests/actions/provisioning"
 	"github.com/rancher/tests/actions/provisioninginput"
 	"github.com/rancher/tests/actions/qase"
+	"github.com/rancher/tests/actions/workloads/deployment"
 	"github.com/rancher/tests/actions/workloads/pods"
 	"github.com/rancher/tests/validation/certificates"
 	resources "github.com/rancher/tests/validation/provisioning/resources/provisioncluster"
@@ -112,6 +113,10 @@ func (c *CertRotationDualstackTestSuite) TestCertRotationDualstack() {
 			logrus.Infof("Verifying the cluster is ready (%s)", cluster.Name)
 			provisioning.VerifyClusterReady(c.T(), c.client, cluster)
 
+			logrus.Infof("Verifying cluster deployments (%s)", cluster.Name)
+			err = deployment.VerifyClusterDeployments(c.client, cluster)
+			require.NoError(c.T(), err)
+
 			logrus.Infof("Verifying cluster pods (%s)", cluster.Name)
 			err = pods.VerifyClusterPods(c.client, cluster)
 			require.NoError(c.T(), err)

validation/certificates/ipv6/cert_rotation_test.go

@@ -20,6 +20,7 @@ import (
 	"github.com/rancher/tests/actions/provisioning"
 	"github.com/rancher/tests/actions/provisioninginput"
 	"github.com/rancher/tests/actions/qase"
+	"github.com/rancher/tests/actions/workloads/deployment"
 	"github.com/rancher/tests/actions/workloads/pods"
 	"github.com/rancher/tests/validation/certificates"
 	resources "github.com/rancher/tests/validation/provisioning/resources/provisioncluster"
@@ -112,6 +113,10 @@ func (c *CertRotationIPv6TestSuite) TestCertRotationIPv6() {
 			logrus.Infof("Verifying the cluster is ready (%s)", cluster.Name)
 			provisioning.VerifyClusterReady(c.T(), c.client, cluster)
 
+			logrus.Infof("Verifying cluster deployments (%s)", cluster.Name)
+			err = deployment.VerifyClusterDeployments(c.client, cluster)
+			require.NoError(c.T(), err)
+
 			logrus.Infof("Verifying cluster pods (%s)", cluster.Name)
 			err = pods.VerifyClusterPods(c.client, cluster)
 			require.NoError(c.T(), err)

validation/certificates/rke2k3s/cert_rotation_existing_cluster_test.go

@@ -15,6 +15,7 @@ import (
 	"github.com/rancher/tests/actions/config/defaults"
 	"github.com/rancher/tests/actions/logging"
 	"github.com/rancher/tests/actions/provisioning"
+	"github.com/rancher/tests/actions/workloads/deployment"
 	"github.com/rancher/tests/actions/workloads/pods"
 	"github.com/rancher/tests/validation/certificates"
 	"github.com/sirupsen/logrus"
@@ -78,6 +79,10 @@ func (c *CertRotationExistingClusterTestSuite) TestCertRotationExistingCluster()
 			logrus.Infof("Verifying the cluster is ready (%s)", cluster.Name)
 			provisioning.VerifyClusterReady(c.T(), c.client, cluster)
 
+			logrus.Infof("Verifying cluster deployments (%s)", cluster.Name)
+			err = deployment.VerifyClusterDeployments(c.client, cluster)
+			require.NoError(c.T(), err)
+
 			logrus.Infof("Verifying cluster pods (%s)", cluster.Name)
 			err = pods.VerifyClusterPods(c.client, cluster)
 			require.NoError(c.T(), err)

validation/certificates/rke2k3s/cert_rotation_test.go

@@ -18,6 +18,7 @@ import (
 	"github.com/rancher/tests/actions/logging"
 	"github.com/rancher/tests/actions/provisioning"
 	"github.com/rancher/tests/actions/qase"
+	"github.com/rancher/tests/actions/workloads/deployment"
 	"github.com/rancher/tests/actions/workloads/pods"
 	"github.com/rancher/tests/validation/certificates"
 	resources "github.com/rancher/tests/validation/provisioning/resources/provisioncluster"
@@ -99,6 +100,10 @@ func (c *CertRotationTestSuite) TestCertRotation() {
 			logrus.Infof("Verifying the cluster is ready (%s)", cluster.Name)
 			provisioning.VerifyClusterReady(c.T(), c.client, cluster)
 
+			logrus.Infof("Verifying cluster deployments (%s)", cluster.Name)
+			err = deployment.VerifyClusterDeployments(c.client, cluster)
+			require.NoError(c.T(), err)
+
 			logrus.Infof("Verifying cluster pods (%s)", cluster.Name)
 			err = pods.VerifyClusterPods(c.client, cluster)
 			require.NoError(c.T(), err)

validation/certificates/rke2k3s/cert_rotation_wins_test.go

@@ -21,6 +21,7 @@ import (
 	"github.com/rancher/tests/actions/provisioning"
 	"github.com/rancher/tests/actions/provisioninginput"
 	"github.com/rancher/tests/actions/qase"
+	"github.com/rancher/tests/actions/workloads/deployment"
 	"github.com/rancher/tests/actions/workloads/pods"
 	"github.com/rancher/tests/validation/certificates"
 	resources "github.com/rancher/tests/validation/provisioning/resources/provisioncluster"
@@ -116,6 +117,10 @@ func (c *CertRotationWindowsTestSuite) TestCertRotationWindows() {
 			logrus.Infof("Verifying the cluster is ready (%s)", cluster.Name)
 			provisioning.VerifyClusterReady(c.T(), c.client, cluster)
 
+			logrus.Infof("Verifying cluster deployments (%s)", cluster.Name)
+			err = deployment.VerifyClusterDeployments(c.client, cluster)
+			require.NoError(c.T(), err)
+
 			logrus.Infof("Verifying cluster pods (%s)", cluster.Name)
 			err = pods.VerifyClusterPods(c.client, cluster)
 			require.NoError(c.T(), err)

validation/charts/backup_restore/backup_restore.go

@@ -14,6 +14,7 @@ import (
 	actionscharts "github.com/rancher/tests/actions/charts"
 	"github.com/rancher/tests/actions/projects"
 	"github.com/rancher/tests/actions/secrets"
+	"github.com/rancher/tests/actions/workloads/deployment"
 	"github.com/rancher/tests/actions/workloads/pods"
 	"github.com/rancher/tests/interoperability/charts"
 	"github.com/stretchr/testify/require"
@@ -319,6 +320,10 @@ func createRKE2dsCluster(t *testing.T, client *rancher.Client) (*v1.SteveAPIObje
 	logrus.Infof("Verifying the cluster is ready (%s)", steveObject.Name)
 	provisioning.VerifyClusterReady(t, client, steveObject)
 
+	logrus.Infof("Verifying cluster deployments (%s)", steveObject.Name)
+	err = deployment.VerifyClusterDeployments(client, steveObject)
+	require.NoError(t, err)
+
 	logrus.Infof("Verifying cluster pods (%s)", steveObject.Name)
 	err = pods.VerifyClusterPods(client, steveObject)
 	require.NoError(t, err)

validation/charts/backup_restore/backup_restore_test.go

@@ -15,6 +15,7 @@ import (
 	"github.com/rancher/shepherd/pkg/session"
 	"github.com/rancher/tests/actions/projects"
 	"github.com/rancher/tests/actions/provisioning"
+	"github.com/rancher/tests/actions/workloads/deployment"
 	"github.com/rancher/tests/actions/workloads/pods"
 	"github.com/rancher/tests/interoperability/charts"
 	"github.com/stretchr/testify/assert"
@@ -118,6 +119,10 @@ func (b *BackupTestSuite) TestS3InPlaceRestore() {
 	logrus.Infof("Verifying the cluster is ready (%s)", rke2SteveObj.Name)
 	provisioning.VerifyClusterReady(b.T(), b.client, rke2SteveObj)
 
+	logrus.Infof("Verifying cluster deployments (%s)", rke2SteveObj.Name)
+	err = deployment.VerifyClusterDeployments(b.client, rke2SteveObj)
+	require.NoError(b.T(), err)
+
 	logrus.Infof("Verifying cluster pods (%s)", rke2SteveObj.Name)
 	err = pods.VerifyClusterPods(b.client, rke2SteveObj)
 	require.NoError(b.T(), err)

validation/deleting/dualstack/delete_init_machine_test.go

@@ -19,6 +19,7 @@ import (
 	"github.com/rancher/tests/actions/provisioning"
 	"github.com/rancher/tests/actions/provisioninginput"
 	"github.com/rancher/tests/actions/qase"
+	"github.com/rancher/tests/actions/workloads/deployment"
 	"github.com/rancher/tests/actions/workloads/pods"
 	"github.com/rancher/tests/validation/deleting/rke2k3s"
 	resources "github.com/rancher/tests/validation/provisioning/resources/provisioncluster"
@@ -107,6 +108,10 @@ func (d *DeleteInitMachineDualstackTestSuite) TestDeleteInitMachineDualstack() {
 			logrus.Infof("Verifying the cluster is ready (%s)", cluster.Name)
 			provisioning.VerifyClusterReady(d.T(), d.client, cluster)
 
+			logrus.Infof("Verifying cluster deployments (%s)", cluster.Name)
+			err = deployment.VerifyClusterDeployments(d.client, cluster)
+			require.NoError(d.T(), err)
+
 			logrus.Infof("Verifying cluster pods (%s)", cluster.Name)
 			pods.VerifyClusterPods(d.client, cluster)
 		})

validation/deleting/ipv6/delete_init_machine_test.go

@@ -20,6 +20,7 @@ import (
 	"github.com/rancher/tests/actions/provisioning"
 	"github.com/rancher/tests/actions/provisioninginput"
 	"github.com/rancher/tests/actions/qase"
+	"github.com/rancher/tests/actions/workloads/deployment"
 	"github.com/rancher/tests/actions/workloads/pods"
 	"github.com/rancher/tests/validation/deleting/rke2k3s"
 	resources "github.com/rancher/tests/validation/provisioning/resources/provisioncluster"
@@ -120,6 +121,10 @@ func (d *DeleteInitMachineIPv6TestSuite) TestDeleteInitMachineIPv6() {
 			logrus.Infof("Verifying the cluster is ready (%s)", cluster.Name)
 			provisioning.VerifyClusterReady(d.T(), d.client, cluster)
 
+			logrus.Infof("Verifying cluster deployments (%s)", cluster.Name)
+			err = deployment.VerifyClusterDeployments(d.client, cluster)
+			require.NoError(d.T(), err)
+
 			logrus.Infof("Verifying cluster pods (%s)", cluster.Name)
 			err = pods.VerifyClusterPods(d.client, cluster)
 			require.NoError(d.T(), err)