Skip to content

Commit

Permalink
e2e: adjust migration e2e tests and introduce helper functions
Browse files Browse the repository at this point in the history 
This commit adjust existing migration e2e tests to a couple of tests
to cover the scenarios. The seperate filesystem and block tests have
been shrinked to single one and also introduced a couple of helper
functions to setup and teardown migraition specific secret,configmap
and sc. The static pv function has been renamed to a general name
while the tests were adjusted.

Signed-off-by: Humble Chirammal <[email protected]>
  • Loading branch information
@humblec @mergify
humblec authored and mergify[bot] committed Dec 20, 2021
1 parent 784c3ef commit cf6fdcb
Show file tree
Hide file tree
Showing 3 changed files with 110 additions and 210 deletions.
136 changes: 45 additions & 91 deletions e2e/migration.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,104 +4,13 @@ import (
"context"
"encoding/hex"
"fmt"
"strconv"
"strings"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/test/e2e/framework"
)

func validateRBDStaticMigrationPVDeletion(f *framework.Framework, appPath, scName string, isBlock bool) error {
opt := make(map[string]string)
var (
rbdImageName = "kubernetes-dynamic-pvc-e0b45b52-7e09-47d3-8f1b-806995fa4412"
pvName = "pv-name"
pvcName = "pvc-name"
namespace = f.UniqueName
sc = scName
provisionerAnnKey = "pv.kubernetes.io/provisioned-by"
provisionerAnnValue = "rbd.csi.ceph.com"
)

c := f.ClientSet
PVAnnMap := make(map[string]string)
PVAnnMap[provisionerAnnKey] = provisionerAnnValue
mons, err := getMons(rookNamespace, c)
if err != nil {
return fmt.Errorf("failed to get mons: %w", err)
}
mon := strings.Join(mons, ",")
size := staticPVSize
// create rbd image
cmd := fmt.Sprintf(
"rbd create %s --size=%s --image-feature=layering %s",
rbdImageName,
staticPVSize,
rbdOptions(defaultRBDPool))

_, stdErr, err := execCommandInToolBoxPod(f, cmd, rookNamespace)
if err != nil {
return err
}
if stdErr != "" {
return fmt.Errorf("failed to create rbd image %s", stdErr)
}

opt["migration"] = "true"
opt["clusterID"] = getMonsHash(mon)
opt["imageFeatures"] = staticPVImageFeature
opt["pool"] = defaultRBDPool
opt["staticVolume"] = strconv.FormatBool(true)
opt["imageName"] = rbdImageName

// Make volumeID similar to the migration volumeID
volID := composeIntreeMigVolID(mon, rbdImageName)
pv := getStaticPV(
pvName,
volID,
size,
rbdNodePluginSecretName,
cephCSINamespace,
sc,
provisionerAnnValue,
isBlock,
opt,
PVAnnMap,
deletePolicy)

_, err = c.CoreV1().PersistentVolumes().Create(context.TODO(), pv, metav1.CreateOptions{})
if err != nil {
return fmt.Errorf("PV Create API error: %w", err)
}

pvc := getStaticPVC(pvcName, pvName, size, namespace, sc, isBlock)

_, err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(context.TODO(), pvc, metav1.CreateOptions{})
if err != nil {
return fmt.Errorf("PVC Create API error: %w", err)
}
// bind pvc to app
app, err := loadApp(appPath)
if err != nil {
return err
}

app.Namespace = namespace
app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvcName
err = createApp(f.ClientSet, app, deployTimeout)
if err != nil {
return err
}

err = deletePVCAndApp("", f, pvc, app)
if err != nil {
return fmt.Errorf("failed to delete PVC and application: %w", err)
}

return err
}

// composeIntreeMigVolID create a volID similar to intree migration volID
// the migration volID format looks like below
// mig-mons-<hash>-image-<UUID_<poolhash>
Expand Down Expand Up @@ -190,6 +99,7 @@ func createMigrationUserSecretAndSC(f *framework.Framework, scName string) error
return nil
}

// createMigrationSC create a SC with migration specific secrets and clusterid.
func createMigrationSC(f *framework.Framework, scName string) error {
err := deleteResource(rbdExamplePath + "storageclass.yaml")
if err != nil {
Expand All @@ -203,6 +113,13 @@ func createMigrationSC(f *framework.Framework, scName string) error {
param["csi.storage.k8s.io/controller-expand-secret-name"] = rbdMigrationProvisionerSecretName
param["csi.storage.k8s.io/node-stage-secret-namespace"] = cephCSINamespace
param["csi.storage.k8s.io/node-stage-secret-name"] = rbdMigrationNodePluginSecretName
mons, err := getMons(rookNamespace, f.ClientSet)
if err != nil {
return fmt.Errorf("failed to get mons: %w", err)
}
mon := strings.Join(mons, ",")
param["migration"] = "true"
param["clusterID"] = getMonsHash(mon)
err = createRBDStorageClass(f.ClientSet, f, scName, nil, param, deletePolicy)
if err != nil {
return fmt.Errorf("failed to create storageclass: %w", err)
Expand Down Expand Up @@ -274,3 +191,40 @@ func deleteProvNodeMigrationSecret(f *framework.Framework, provisionerSecret, no

return nil
}

// setupMigrationCMSecretAndSC create custom configmap, secret and SC for migration tests.
func setupMigrationCMSecretAndSC(f *framework.Framework, scName string) error {
c := f.ClientSet
if scName == "" {
scName = defaultSCName
}
err := generateClusterIDConfigMapForMigration(f, c)
if err != nil {
return fmt.Errorf("failed to generate clusterID configmap: %w", err)
}

err = createMigrationUserSecretAndSC(f, scName)
if err != nil {
return fmt.Errorf("failed to create storageclass: %w", err)
}

return nil
}

// tearDownMigrationSetup deletes custom configmap and secret.
func tearDownMigrationSetup(f *framework.Framework) error {
err := deleteConfigMap(rbdDirPath)
if err != nil {
return fmt.Errorf("failed to delete configmap: %w", err)
}
err = createConfigMap(rbdDirPath, f.ClientSet, f)
if err != nil {
return fmt.Errorf("failed to create configmap: %w", err)
}
err = deleteProvNodeMigrationSecret(f, true, true)
if err != nil {
return fmt.Errorf("failed to delete migration users and Secrets associated: %w", err)
}

return nil
}
123 changes: 39 additions & 84 deletions e2e/rbd.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -402,37 +402,58 @@ var _ = Describe("RBD", func() {
}
})

// todo: may be remove the below deletion test later once the migration nodestage tests are adjusted
// also to have deletion validation through the same.
By("validate RBD migration+static Block PVC Deletion", func() {
err := generateClusterIDConfigMapForMigration(f, c)
By("validate RBD migration PVC", func() {
err := setupMigrationCMSecretAndSC(f, "")
if err != nil {
e2elog.Failf("failed to generate clusterID configmap: %v", err)
e2elog.Failf("failed to setup migration prerequisites: %v", err)
}
err = validatePVCAndAppBinding(pvcPath, appPath, f)
if err != nil {
e2elog.Failf("failed to validate pvc and application binding: %v", err)
}
// validate created backend rbd images
validateRBDImageCount(f, 0, defaultRBDPool)

// create a sc with different migration secret
err = createMigrationUserSecretAndSC(f, "migrationsc")
// Block PVC resize
err = resizePVCAndValidateSize(rawPvcPath, rawAppPath, f)
if err != nil {
e2elog.Failf("failed to create storageclass: %v", err)
e2elog.Failf("failed to resize block PVC: %v", err)
}
err = validateRBDStaticMigrationPVDeletion(f, rawAppPath, "migrationsc", true)

// FileSystem PVC resize
err = resizePVCAndValidateSize(pvcPath, appPath, f)
if err != nil {
e2elog.Failf("failed to validate rbd migrated static block pv: %v", err)
e2elog.Failf("failed to resize filesystem PVC: %v", err)
}
// validate created backend rbd images
validateRBDImageCount(f, 0, defaultRBDPool)
err = deleteConfigMap(rbdDirPath)
err = deleteResource(rbdExamplePath + "storageclass.yaml")
if err != nil {
e2elog.Failf("failed to delete configmap: %v", err)
e2elog.Failf("failed to delete storageclass: %v", err)
}
err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy)
if err != nil {
e2elog.Failf("failed to create storageclass: %v", err)
}
err = createConfigMap(rbdDirPath, f.ClientSet, f)
err = tearDownMigrationSetup(f)
if err != nil {
e2elog.Failf("failed to create configmap: %v", err)
e2elog.Failf("failed to tear down migration setup: %v", err)
}
})

err = deleteProvNodeMigrationSecret(f, true, true)
By("validate RBD migration+static FileSystem", func() {
err := setupMigrationCMSecretAndSC(f, "migrationsc")
if err != nil {
e2elog.Failf("failed to setup migration prerequisites: %v", err)
}
// validate filesystem pvc mount
err = validateRBDStaticMigrationPVC(f, appPath, "migrationsc", false)
if err != nil {
e2elog.Failf("failed to delete migration users and Secrets associated: %v", err)
e2elog.Failf("failed to validate rbd migrated static file mode pvc: %v", err)
}
// validate created backend rbd images
validateRBDImageCount(f, 0, defaultRBDPool)
err = tearDownMigrationSetup(f)
if err != nil {
e2elog.Failf("failed to tear down migration setup: %v", err)
}
err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy)
if err != nil {
Expand Down Expand Up @@ -2117,72 +2138,6 @@ var _ = Describe("RBD", func() {
validateRBDImageCount(f, 0, defaultRBDPool)
})

By("validate RBD migration+static FileSystem PVC", func() {
err := generateClusterIDConfigMapForMigration(f, c)
if err != nil {
e2elog.Failf("failed to generate clusterID configmap: %v", err)
}
// create node user and migration secret.
err = createProvNodeCephUserAndSecret(f, false, true)
if err != nil {
e2elog.Failf("failed to create users and secret: %v", err)
}

err = validateRBDStaticMigrationPV(f, appPath, rbdMigrationNodePluginSecretName, false)
if err != nil {
e2elog.Failf("failed to validate rbd migrated static pv: %v", err)
}
// validate created backend rbd images
validateRBDImageCount(f, 0, defaultRBDPool)

err = deleteProvNodeMigrationSecret(f, false, true)
if err != nil {
e2elog.Failf("failed to delete users and secret: %v", err)
}

err = deleteConfigMap(rbdDirPath)
if err != nil {
e2elog.Failf("failed to delete configmap: %v", err)
}
err = createConfigMap(rbdDirPath, f.ClientSet, f)
if err != nil {
e2elog.Failf("failed to create configmap: %v", err)
}
})

By("validate RBD migration+static Block PVC", func() {
err := generateClusterIDConfigMapForMigration(f, c)
if err != nil {
e2elog.Failf("failed to generate clusterID configmap: %v", err)
}
// create node user and migration secret.
err = createProvNodeCephUserAndSecret(f, false, true)
if err != nil {
e2elog.Failf("failed to create users and secret: %v", err)
}

err = validateRBDStaticMigrationPV(f, rawAppPath, rbdMigrationNodePluginSecretName, true)
if err != nil {
e2elog.Failf("failed to validate rbd migrated static block pv: %v", err)
}
// validate created backend rbd images
validateRBDImageCount(f, 0, defaultRBDPool)

err = deleteProvNodeMigrationSecret(f, false, true)
if err != nil {
e2elog.Failf("failed to delete users and secret: %v", err)
}

err = deleteConfigMap(rbdDirPath)
if err != nil {
e2elog.Failf("failed to delete configmap: %v", err)
}
err = createConfigMap(rbdDirPath, f.ClientSet, f)
if err != nil {
e2elog.Failf("failed to create configmap: %v", err)
}
})

By("validate failure of RBD static PVC without imageFeatures parameter", func() {
err := validateRBDStaticPV(f, rawAppPath, true, true)
if err != nil {
Expand Down
Loading

0 comments on commit cf6fdcb

Please sign in to comment.