resolve conficts

pkg/k8s/node.go

@@ -45,16 +45,6 @@ func (k *Client) WaitForNodesReady(minReadyNodes int) error {
 	return nil
 }
 
-func (k *Client) WaitForNodeReady(ip string) error {
-	// nodeList, err := k.ListResources(ResourceTypeNode, "", "")
-	// if err != nil {
-	// 	return fmt.Errorf("failed to list nodes: %w", err)
-	// }
-
-	return nil
-
-}
-
 // checkInitialNodesReady receive the amount of minReadyNodes.
 // checks the initial state of the nodes and returns:
 // - nodeMap: a map of node names to their readiness status.

pkg/testcase/clusterrestore.go

@@ -98,7 +98,7 @@ func stopInstances(cluster *shared.Cluster, ec2 *aws.Client) {
 func newInstance(awsClient *aws.Client) (newServerName, newExternalIP string) {
 	resourceName := os.Getenv("resource_name")
 	var serverName []string
-	serverName = append(serverName, fmt.Sprintf("%s-server-fresh", resourceName))
+	serverName = append(serverName, resourceName+"-server-fresh")
 
 	externalServerIP, _, _, createErr := awsClient.CreateInstances(serverName...)
 	Expect(createErr).NotTo(HaveOccurred(), createErr)
@@ -241,7 +241,7 @@ func postValidationRestore(cluster *shared.Cluster, k8sClient *k8s.Client, newSe
 		kubectlCmd = exportKubeConfigCmd + " && " + pathCmd + " && " + kubectlCmd
 	}
 
-	getNodesPodsCmd := kubectlCmd + fmt.Sprintf(" get nodes,pods -A -o wide %s", kubeconfigFlagRemote)
+	getNodesPodsCmd := kubectlCmd + " get nodes,pods -A -o wide " + kubeconfigFlagRemote
 	_, nodesPodsErr := shared.RunCommandOnNode(getNodesPodsCmd, newServerIP)
 	Expect(nodesPodsErr).NotTo(HaveOccurred())
 

pkg/testcase/pod.go

@@ -88,10 +88,11 @@ func CheckPodStatus(cluster *shared.Cluster) {
 		if res != "" {
 			shared.LogLevel("info", "Waiting for pods: \n%s", res)
 			return false
-		} 
+		}
 		for i := range pods {
 			processPodStatus(cluster, g, &pods[i], podAssertRestarts, podAssertReady)
 		}
+
 		return true
 	}, "600s", "10s").Should(BeTrue(), "failed to process pods status")
 }

pkg/testcase/upgrademanual.go

@@ -3,12 +3,12 @@ package testcase
 import (
 	"errors"
 	"fmt"
-	"strings"
 	"time"
 
-	"github.com/rancher/distros-test-framework/pkg/customflag"
 	"github.com/rancher/distros-test-framework/pkg/k8s"
 	"github.com/rancher/distros-test-framework/shared"
+
+	. "github.com/onsi/gomega"
 )
 
 const (
@@ -30,20 +30,20 @@ func TestUpgradeClusterManual(cluster *shared.Cluster, k8sClient *k8s.Client, ve
 
 	if cluster.NumServers > 0 {
 		for _, ip := range cluster.ServerIPs {
-			if err := upgradeProduct(k8sClient, cluster.Config.Product, server, version, ip); err != nil {
+			if err := upgradeProduct(cluster.Config.Product, server, version, ip); err != nil {
+				shared.LogLevel("warn", "upgrading %s %s: %v", server, ip, err)
 				return err
 			}
-			shared.LogLevel("info", "Checking pod status after restarting %v node: %v", server, ip)
 			CheckPodStatus(cluster)
 		}
 	}
 
 	if cluster.NumAgents > 0 {
 		for _, ip := range cluster.AgentIPs {
-			if err := upgradeProduct(k8sClient, cluster.Config.Product, agent, version, ip); err != nil {
+			if err := upgradeProduct(cluster.Config.Product, agent, version, ip); err != nil {
+				shared.LogLevel("warn", "upgrading %s %s: %v", agent, ip, err)
 				return err
 			}
-			shared.LogLevel("info", "Checking pod status after restarting %v node: %v", agent, ip)
 			CheckPodStatus(cluster)
 		}
 	}
@@ -60,57 +60,53 @@ func TestUpgradeClusterManual(cluster *shared.Cluster, k8sClient *k8s.Client, ve
 }
 
 // upgradeProduct upgrades a node server or agent type to the specified version.
-func upgradeProduct(k8sClient *k8s.Client, product, nodeType, installType, ip string) error {
-	upgradeCommand := getInstallCmd(product, installType, nodeType)
+func upgradeProduct(product, nodeType, installType, ip string) error {
+	upgradeCommand := shared.GetInstallCmd(product, installType, nodeType)
 	shared.LogLevel("info", "Upgrading %s %s: %s", ip, nodeType, upgradeCommand)
 	if _, err := shared.RunCommandOnNode(upgradeCommand, ip); err != nil {
 		shared.LogLevel("warn", "upgrading %s %s: %v", nodeType, ip, err)
 		return err
 	}
-	shared.LogLevel("info", "Waiting 30s after installing upgrade...")
-	time.Sleep(30 * time.Second)
+
 	if product == "rke2" {
+		shared.LogLevel("info", "Waiting 1min after installing upgrade...")
+		time.Sleep(1 * time.Minute)
 		shared.LogLevel("info", "Restarting %s service on %s node: %s", product, nodeType, ip)
 		_, err := shared.ManageService(product, "restart", nodeType, []string{ip})
 		if err != nil {
 			return err
 		}
-		shared.LogLevel("info", "Waiting for 180s after restarting service")
-		time.Sleep(180 * time.Second)
-		shared.LogLevel("info", "Waiting for %v node to be ready: %v", nodeType, ip)
-		err = k8sClient.WaitForNodeReady(ip)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
+		shared.LogLevel("info", "Waiting for 3min after restarting service")
+		time.Sleep(3 * time.Minute)
 
-func getInstallCmd(product, installType, nodeType string) string {
-	var installFlag string
-	var installCmd string
-
-	var channel = getChannel(product)
-
-	if strings.HasPrefix(installType, "v") {
-		installFlag = fmt.Sprintf("INSTALL_%s_VERSION=%s", strings.ToUpper(product), installType)
-	} else {
-		installFlag = fmt.Sprintf("INSTALL_%s_COMMIT=%s", strings.ToUpper(product), installType)
+		Eventually(func(g Gomega) bool {
+			res, err := shared.ManageService(product, "status", nodeType, []string{ip})
+			if err != nil {
+				shared.LogLevel("info", "\n%s", err)
+			}
+			shared.LogLevel("info", "Checking service status after upgrade on %s node: %s", nodeType, ip)
+			shared.LogLevel("info", "\n%s", res)
+			
+			return true
+		}, "300s", "30s").Should(BeTrue(), "failed to validate service status")
 	}
 
-	installCmd = fmt.Sprintf("curl -sfL https://get.%s.io | sudo %%s %%s sh -s - %s", product, nodeType)
-
-	return fmt.Sprintf(installCmd, installFlag, channel)
-}
+	if product == "k3s" {
+		shared.LogLevel("info", "Waiting for 1min after upgrading service...")
+		time.Sleep(60 * time.Second)
 
-func getChannel(product string) string {
-	var defaultChannel = fmt.Sprintf("INSTALL_%s_CHANNEL=%s", strings.ToUpper(product), "stable")
-
-	if customflag.ServiceFlag.Channel.String() != "" {
-		return fmt.Sprintf("INSTALL_%s_CHANNEL=%s", strings.ToUpper(product),
-			customflag.ServiceFlag.Channel.String())
+		Eventually(func(g Gomega) bool {
+			res, err := shared.ManageService(product, "status", nodeType, []string{ip})
+			if err != nil {
+				shared.LogLevel("info", "\n%s", err)
+			}
+			shared.LogLevel("info", "Checking service status after upgrade on %s node: %s", nodeType, ip)
+			shared.LogLevel("info", "\n%s", res)
+
+			return true
+		}, "120s", "10s").Should(BeTrue(), "failed to validate service status")
 	}
 
-	return defaultChannel
+	return nil
 }
+

shared/aux.go

@@ -617,7 +617,7 @@ func ManageService(service, action, nodeType string, ips []string) (string, erro
 			retry.Delay(5*time.Second),
 			retry.OnRetry(func(n uint, err error) {
 				if n == 0 || n == 19 {
-					LogLevel("warn", "Failed to run command: %s on node %s: Attempt-%v\nError: %v", cmd, ip, n+1, runErr)
+					LogLevel("warn", "Failed to run command: %s on node %s: Attempt-%v\nError: %v", cmd, ip, n+1, err)
 				}
 			}),
 		)

shared/kubeconfig.go

@@ -137,7 +137,7 @@ func NewLocalKubeconfigFile(newServerIP, resourceName, product, localPath string
 	}
 
 	serverIPRgx := regexp.MustCompile(`server: https://\d+\.\d+\.\d+\.\d+`)
-	replace := fmt.Sprintf("server: https://%s", newServerIP)
+	replace := "server: https://" + newServerIP
 	updated := serverIPRgx.ReplaceAllString(kubeconfigContent, replace)
 
 	writeErr := os.WriteFile(localPath, []byte(updated), 0o644)