fix cluster restore and cert rotate (#209)

Signed-off-by: Francisco <[email protected]>

Author: fmoral2

docs/examples/.env.example

@@ -1,34 +1,56 @@
 # Framework config variables
+
+### Variables that would need to change for different products,test cases,pem key ####
+### Required variables to be set by user ###
+
+# Note: PRODUCT can be k3s or rke2
 ENV_PRODUCT="{{PRODUCT}}"
 ENV_TFVARS="{{PRODUCT}}".tfvars
-# Note: PRODUCT can be k3s or rke2
 
+# test pkg name # which is located on /entrypoint/
+TEST_DIR=upgradecluster
 
-#ACCESS_KEY_LOCAL="˜/aws-key.pem"
-ACCESS_KEY_LOCAL="~/aws-key.pem"
+# test tag name # only needed for tests pkg with multiple test cases.
+# can find example here - https://github.com/rancher/distros-test-framework/blob/83e207dcf2aa4964db881f87e55f1eb0b031887f/entrypoint/upgradecluster/upgrademanual_test.go#L1
+TEST_TAG=upgrademanual
 
 
+# aws key path locally stored
+ACCESS_KEY_LOCAL="~/{key-name}.pem"
+
+##### Variables that would not need to change ######
+### Optional variables to be set by user ###
 
 ######  Test runner config variables  ########
 # image name #
-IMG_NAME=er
+IMG_NAME=test
 
 # container name , default == distros #
 TAG_NAME=
 
-# test pkg name #
-TEST_DIR=createcluster
-
 # test state img name #
-TEST_STATE=2
-
-# test tag name #
-TEST_TAG=upgrademanual
+TEST_STATE=
 
 # log level could be "" = (INFO default), INFO, DEBUG or WARN #
-LOG_LEVEL=info
-######  Test runner config variables  ########
+LOG_LEVEL=debug
 
+# test state img name #
+TEST_STATE=
+
+
+#######  custom tfvars override   ###########
+INSTALL_VERSION=v1.30.2+k3s1  OR
+INSTALL_VERSION=v1.30.2+rke2r1
+#
+RKE2_CHANNEL=testing
+K3S_CHANNEL=testing
+NO_OF_SERVER_NODES=1
+NO_OF_WORKER_NODES=1
+SERVER_FLAGS=protect-kernel-defaults: true\\nselinux: true
+WORKER_FLAGS=protect-kernel-defaults: true\\nselinux: true
+ARCH=arm
+DATASTORE_TYPE=etcd
+#######  custom vars tfvars override   ###########
 
 
 ####### Version bump test variables + cli flags  ###########
@@ -46,22 +68,3 @@ DELETE_WORKLAOD=
 ####### Version bump test variables  ###########
 
 
-
-#######  custom tfvars override   ###########
-INSTALL_VERSION=v1.30.2+k3s1  OR
-INSTALL_VERSION=v1.30.2+rke2r1
-
-RKE2_CHANNEL=testing
-K3S_CHANNEL=testing
-NO_OF_SERVER_NODES=1
-NO_OF_WORKER_NODES=1
-SERVER_FLAGS=protect-kernel-defaults: true\\nselinux: true
-WORKER_FLAGS=protect-kernel-defaults: true\\nselinux: true
-VOLUME_SIZE=40
-NODE_OS=
-AWS_AMI=
-AWS_USER=
-INSTALL_MODE=
-DATASTORE_TYPE=
-VOLUME_SIZE=40
-#######  custom vars tfvars override   ###########

entrypoint/clusterrestore/clusterrestore_suite_test.go

@@ -2,7 +2,6 @@ package clusterrestore
 
 import (
 	"flag"
-	"fmt"
 	"os"
 	"strings"
 	"testing"
@@ -30,7 +29,8 @@ func TestMain(m *testing.M) {
 	var err error
 	flags = &customflag.ServiceFlag
 	flag.Var(&flags.Destroy, "destroy", "Destroy cluster after test")
-	flag.StringVar(&flags.S3Flags.Bucket, "s3Bucket", "distros_qa", "s3 bucket to store snapshots")
+	flag.Var(&flags.Channel, "channel", "channel to use on install")
+	flag.StringVar(&flags.S3Flags.Bucket, "s3Bucket", "distrosqa", "s3 bucket to store snapshots")
 	flag.StringVar(&flags.S3Flags.Folder, "s3Folder", "snapshots", "s3 folder to store snapshots")
 	flag.Parse()
 
@@ -94,7 +94,6 @@ func FailWithReport(message string, callerSkip ...int) {
 
 func checkUnsupportedFlags() {
 	serverFlags := os.Getenv("server_flags")
-	fmt.Printf("server flags: %s\n", serverFlags)
 
 	if strings.Contains(serverFlags, "profile") ||
 		strings.Contains(serverFlags, "selinux") ||
@@ -108,8 +107,8 @@ func checkUnsupportedFlags() {
 func cleanS3Snapshot() {
 	shared.LogLevel("info", "cleaning s3 snapshots")
 
-	err := awsClient.DeleteS3Object(customflag.ServiceFlag.S3Flags.Bucket, "on-demand-ip")
+	err := awsClient.DeleteS3Object(customflag.ServiceFlag.S3Flags.Bucket, customflag.ServiceFlag.S3Flags.Folder)
 	if err != nil {
-		shared.LogLevel("error", "error deleting object: %s", err)
+		shared.LogLevel("error", "error deleting object: %v", err)
 	}
 }

pkg/aws/s3.go

@@ -23,10 +23,10 @@ func (c Client) GetObjects(bucket string) ([]*s3.Object, error) {
 	return output.Contents, nil
 }
 
-func (c Client) DeleteS3Object(bucket, prefix string) error {
+func (c Client) DeleteS3Object(bucket, folder string) error {
 	input := &s3.ListObjectsV2Input{
 		Bucket: aws.String(bucket),
-		Prefix: aws.String(prefix),
+		Prefix: aws.String(folder),
 	}
 
 	objList, err := c.s3.ListObjectsV2(input)
@@ -35,14 +35,14 @@ func (c Client) DeleteS3Object(bucket, prefix string) error {
 	}
 
 	if len(objList.Contents) == 0 {
-		return fmt.Errorf("no objects found with prefix %s", prefix)
+		return fmt.Errorf("no objects found with prefix %s", *input.Prefix)
 	}
 
 	sort.Slice(objList.Contents, func(i, j int) bool {
 		return objList.Contents[i].LastModified.After(*objList.Contents[j].LastModified)
 	})
-	key := *objList.Contents[0].Key
 
+	key := aws.StringValue(objList.Contents[0].Key)
 	delInput := &s3.DeleteObjectInput{
 		Bucket: aws.String(bucket),
 		Key:    aws.String(key),

pkg/testcase/certrotate.go

@@ -10,27 +10,64 @@ import (
 )
 
 func TestCertRotate(cluster *shared.Cluster) {
-	certRotate(cluster.Config.Product, cluster.ServerIPs)
-
-	ip, manageError := shared.ManageService(cluster.Config.Product, "restart", "agent", cluster.AgentIPs)
-	Expect(manageError).NotTo(HaveOccurred(), "error restarting agent node ip"+ip)
+	ms := shared.NewManageService(5, 5)
+	certRotate(ms, cluster.Config.Product, cluster.ServerIPs)
+
+	actions := []shared.ServiceAction{
+		{Service: cluster.Config.Product,
+			Action:   "restart",
+			NodeType: "agent",
+		},
+		{
+			Service:  cluster.Config.Product,
+			Action:   "status",
+			NodeType: "agent",
+		},
+	}
+	for _, agentIP := range cluster.AgentIPs {
+		output, err := ms.ManageService(agentIP, actions)
+		if output != "" {
+			Expect(output).To(ContainSubstring("active "), fmt.Sprintf("error restarting %s agent service for node ip: %s",
+				cluster.Config.Product, agentIP))
+		}
+		Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("error restarting %s service on %s", cluster.Config.Product, agentIP))
+	}
 
 	verifyTLSDirContent(cluster.Config.Product, cluster.ServerIPs)
 }
 
 // certRotate Rotate certificate for etcd only and cp only nodes.
-func certRotate(product string, ips []string) {
-	ip, stopError := shared.ManageService(product, "stop", "server", ips)
-	Expect(stopError).NotTo(HaveOccurred(),
-		fmt.Sprintf("error stopping %s service for node ip: %s", product, ip))
-
-	ip, rotateError := shared.CertRotate(product, ips)
-	Expect(rotateError).NotTo(HaveOccurred(),
-		fmt.Sprintf("error running certificate rotate for %s service on %s", product, ip))
-
-	ip, startError := shared.ManageService(product, "start", "server", ips)
-	Expect(startError).NotTo(HaveOccurred(),
-		fmt.Sprintf("error starting %s service for node ip: %s", product, ip))
+func certRotate(ms *shared.ManageService, product string, ips []string) {
+	for _, ip := range ips {
+		actions := []shared.ServiceAction{
+			{
+				Service:  product,
+				Action:   "stop",
+				NodeType: "server",
+			},
+			{
+				Service: product,
+				Action:  "rotate",
+			},
+			{
+				Service:  product,
+				Action:   "start",
+				NodeType: "server",
+			},
+			{
+				Service:  product,
+				Action:   "status",
+				NodeType: "server",
+			},
+		}
+
+		output, err := ms.ManageService(ip, actions)
+		if output != "" {
+			Expect(output).To(ContainSubstring("active "),
+				fmt.Sprintf("error restarting %s service for node ip: %s", product, ip))
+		}
+		Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("error rotating certificate for %s service on %s", product, ip))
+	}
 }
 
 // verifyIdenticalFiles Verify the actual and expected identical file lists match.

pkg/testcase/clusterreset.go

@@ -12,7 +12,8 @@ func TestClusterReset(cluster *shared.Cluster) {
 	killall(cluster)
 	shared.LogLevel("info", "%s-service killed", cluster.Config.Product)
 
-	stopServer(cluster)
+	ms := shared.NewManageService(5, 5)
+	stopServer(cluster, ms)
 	shared.LogLevel("info", "%s-service stopped", cluster.Config.Product)
 
 	productLocationCmd, findErr := shared.FindPath(cluster.Config.Product, cluster.ServerIPs[0])
@@ -27,7 +28,7 @@ func TestClusterReset(cluster *shared.Cluster) {
 	deleteDataDirectories(cluster)
 	shared.LogLevel("info", "data directories deleted")
 
-	restartServer(cluster)
+	restartServer(cluster, ms)
 	shared.LogLevel("info", "%s-service restarted", cluster.Config.Product)
 }
 
@@ -68,17 +69,22 @@ func killall(cluster *shared.Cluster) {
 	Expect(res).To(SatisfyAny(ContainSubstring("timed out"), ContainSubstring("refused")))
 }
 
-func stopServer(cluster *shared.Cluster) {
-	_, stopErr := shared.ManageService(cluster.Config.Product, "stop", "server", []string{cluster.ServerIPs[0]})
+func stopServer(cluster *shared.Cluster, ms *shared.ManageService) {
+	action := shared.ServiceAction{
+		Service:  cluster.Config.Product,
+		Action:   "stop",
+		NodeType: "server",
+	}
+	_, stopErr := ms.ManageService(cluster.ServerIPs[0], []shared.ServiceAction{action})
 	Expect(stopErr).NotTo(HaveOccurred())
 
-	// due to the stop command, the service will be inactive, so ssh command to node  will fail.
+	// due to the stop command, this should fail.
 	cmd := fmt.Sprintf("sudo systemctl status %s-server", cluster.Config.Product)
 	_, err := shared.RunCommandOnNode(cmd, cluster.ServerIPs[0])
 	Expect(err).To(HaveOccurred())
 }
 
-func restartServer(cluster *shared.Cluster) {
+func restartServer(cluster *shared.Cluster, ms *shared.ManageService) {
 	var startFirst []string
 	var startLast []string
 
@@ -91,11 +97,20 @@ func restartServer(cluster *shared.Cluster) {
 		startLast = append(startLast, serverIP)
 	}
 
-	_, startErr := shared.ManageService(cluster.Config.Product, "restart", "server", startFirst)
-	Expect(startErr).NotTo(HaveOccurred())
+	action := shared.ServiceAction{
+		Service:  cluster.Config.Product,
+		Action:   "restart",
+		NodeType: "server",
+	}
+	for _, ip := range startFirst {
+		_, startErr := ms.ManageService(ip, []shared.ServiceAction{action})
+		Expect(startErr).NotTo(HaveOccurred())
+	}
 
-	_, startLastErr := shared.ManageService(cluster.Config.Product, "restart", "server", startLast)
-	Expect(startLastErr).NotTo(HaveOccurred())
+	for _, ip := range startLast {
+		_, startLastErr := ms.ManageService(ip, []shared.ServiceAction{action})
+		Expect(startLastErr).NotTo(HaveOccurred())
+	}
 }
 
 func deleteDataDirectories(cluster *shared.Cluster) {