From 0964543f0ed8b3a1b758783c44b7c76d37f4b4c9 Mon Sep 17 00:00:00 2001 From: sinhaashish Date: Tue, 21 Jan 2025 05:41:52 +0000 Subject: [PATCH] fix(controller): prevent zfs volume cr deletion if snapshot exists Signed-off-by: sinhaashish --- pkg/builder/volbuilder/build.go | 12 ++ pkg/driver/controller.go | 103 ++++++++++++-- pkg/response/delete.go | 20 +++ pkg/zfs/volume.go | 64 ++++++++- tests/provision_test.go | 101 +++++++++---- tests/pv/build.go | 80 +++++++++++ tests/pv/kubernetes.go | 178 +++++++++++++++++++++++ tests/pv/persistentvolume.go | 19 +++ tests/sc/build.go | 7 + tests/suite_test.go | 41 ++++-- tests/utils.go | 242 ++++++++++++++++++++++++-------- 11 files changed, 754 insertions(+), 113 deletions(-) create mode 100644 tests/pv/build.go create mode 100644 tests/pv/kubernetes.go create mode 100644 tests/pv/persistentvolume.go diff --git a/pkg/builder/volbuilder/build.go b/pkg/builder/volbuilder/build.go index d1913e4f3..8dd98ccfa 100644 --- a/pkg/builder/volbuilder/build.go +++ b/pkg/builder/volbuilder/build.go @@ -21,6 +21,9 @@ import ( apis "github.com/openebs/zfs-localpv/pkg/apis/openebs.io/zfs/v1" ) +// MarkForDeletionAnnotation is the annotation key +const MarkForDeletionAnnotation string = "openebs.io/marked-for-deletion" + // Builder is the builder object for ZFSVolume type Builder struct { volume *ZFSVolume @@ -154,6 +157,15 @@ func (b *Builder) WithVolBlockSize(bs string) *Builder { return b } +// WithAnnotation sets the annotation of ZFSVolume +func (b *Builder) WithAnnotation() *Builder { + if b.volume.Object.Annotations == nil { + b.volume.Object.Annotations = make(map[string]string) + } + b.volume.Object.Annotations[MarkForDeletionAnnotation] = "true" + return b +} + // WithVolumeType sets if ZFSVolume needs to be thin provisioned func (b *Builder) WithVolumeType(vtype string) *Builder { b.volume.Object.Spec.VolumeType = vtype diff --git a/pkg/driver/controller.go b/pkg/driver/controller.go index 990eee458..9efbed79d 100644 --- a/pkg/driver/controller.go +++ b/pkg/driver/controller.go @@ -503,14 +503,14 @@ func (cs *controller) DeleteVolume( defer unlock() // verify if the volume has already been deleted - vol, err := zfs.GetVolume(volumeID) + vol, err := zfs.GetZFSVolume(volumeID) if vol != nil && vol.DeletionTimestamp != nil { - goto deleteResponse + return csipayload.NewDeleteVolumeResponseBuilder().Build(), nil } if err != nil { if k8serror.IsNotFound(err) { - goto deleteResponse + return csipayload.NewDeleteVolumeResponseBuilder().Build(), nil } return nil, errors.Wrapf( err, @@ -524,19 +524,45 @@ func (cs *controller) DeleteVolume( return nil, status.Error(codes.Internal, "can not delete, volume creation is in progress") } - // Delete the corresponding ZV CR - err = zfs.DeleteVolume(volumeID) + // Fetch the list of snapshot for the given volume + snapList, err := zfs.GetSnapshotForVolume(volumeID) if err != nil { - return nil, errors.Wrapf( - err, - "failed to handle delete volume request for {%s}", - volumeID, + return nil, status.Errorf( + codes.NotFound, + "failed to handle delete volume request for {%s}, "+ + "validation failed checking for snapshots. Error: %s", + req.VolumeId, + err.Error(), ) } + // Delete the corresponding ZV CR only if there are no snapshots present for the volume + if len(snapList.Items) == 0 { + err = zfs.DeleteVolume(volumeID) + if err != nil { + return nil, errors.Wrapf( + err, + "failed to handle delete volume request for {%s}", + volumeID, + ) + } + } else { + // add annotation to the volume to indicate that it is eligible for deletion + // once all the snapshots are deleted and the reclaim policy is not Retain + // this volume will be deleted + err = zfs.MarkForDeletion(volumeID) + if err != nil { + return nil, errors.Wrapf( + err, + "failed to annotate volume on deletion request for {%s}", + volumeID, + ) + } + + } + sendEventOrIgnore("", volumeID, vol.Spec.Capacity, analytics.VolumeDeprovision) -deleteResponse: return csipayload.NewDeleteVolumeResponseBuilder().Build(), nil } @@ -815,8 +841,36 @@ func (cs *controller) DeleteSnapshot( // should succeed when an invalid snapshot id is used return &csi.DeleteSnapshotResponse{}, nil } + volumeID := snapshotID[0] unlock := cs.volumeLock.LockVolumeWithSnapshot(snapshotID[0], snapshotID[1]) defer unlock() + + // verify if the snapshot has already been deleted + _, err := zfs.GetZFSSnapshot(snapshotID[1]) + + if err != nil { + if k8serror.IsNotFound(err) { + return csipayload.NewDeleteSnapshotResponseBuilder().Build(), nil + } + return nil, errors.Wrapf( + err, + "failed to get snapshot for {%s}", + snapshotID[1], + ) + } + + // Fetch the list of snapshot for the given volume + snapList, err := zfs.GetSnapshotForVolume(volumeID) + if err != nil { + return nil, status.Errorf( + codes.FailedPrecondition, + "failed to handle delete snapshot request for {%s}, "+ + "validation failed checking for snapshot list for volume. Error: %s", + volumeID, + err.Error(), + ) + } + if err := zfs.DeleteSnapshot(snapshotID[1]); err != nil { return nil, status.Errorf( codes.Internal, @@ -825,7 +879,34 @@ func (cs *controller) DeleteSnapshot( err.Error(), ) } - return &csi.DeleteSnapshotResponse{}, nil + + eligibleForDeletion, err := zfs.IsVolumeEligibleForDeletion(volumeID) + if err != nil { + return nil, status.Errorf( + codes.FailedPrecondition, + "failed to handle delete snapshot request for {%s}, "+ + "validation failed checking for eligible for deletion. Error: %s", + volumeID, + err.Error(), + ) + } + + klog.Infof(" The snap list size %v and eligibleForDeletion %v", len(snapList.Items), eligibleForDeletion) + // Delete the corresponding ZV CR only if this is the last snapshot + // for the volume and the corresponding pvc is deleted + if len(snapList.Items) == 1 && eligibleForDeletion { + err = zfs.DeleteVolume(volumeID) + if err != nil { + return nil, errors.Wrapf( + err, + "failed to handle delete volume request for {%s}", + volumeID, + ) + } + klog.Infof("volume %s deleted after the deletion of last snapshot %s ", volumeID, snapshotID[1]) + } + + return csipayload.NewDeleteSnapshotResponseBuilder().Build(), nil } // ListSnapshots lists all snapshots for the diff --git a/pkg/response/delete.go b/pkg/response/delete.go index 29a3d4015..18b7b244b 100644 --- a/pkg/response/delete.go +++ b/pkg/response/delete.go @@ -39,3 +39,23 @@ func NewDeleteVolumeResponseBuilder() *DeleteVolumeResponseBuilder { func (b *DeleteVolumeResponseBuilder) Build() *csi.DeleteVolumeResponse { return b.response } + +// DeleteSnapshotResponseBuilder helps building an +// instance of csi DeleteSnapshotResponse +type DeleteSnapshotResponseBuilder struct { + response *csi.DeleteSnapshotResponse +} + +// NewDeleteSnapshotResponseBuilder returns a new +// instance of DeleteSnapshotResponseBuilder +func NewDeleteSnapshotResponseBuilder() *DeleteSnapshotResponseBuilder { + return &DeleteSnapshotResponseBuilder{ + response: &csi.DeleteSnapshotResponse{}, + } +} + +// Build returns the constructed instance +// of csi DeleteSnapshotResponse +func (b *DeleteSnapshotResponseBuilder) Build() *csi.DeleteSnapshotResponse { + return b.response +} diff --git a/pkg/zfs/volume.go b/pkg/zfs/volume.go index 446e4a667..90062d283 100644 --- a/pkg/zfs/volume.go +++ b/pkg/zfs/volume.go @@ -27,6 +27,8 @@ import ( "github.com/openebs/zfs-localpv/pkg/builder/restorebuilder" "github.com/openebs/zfs-localpv/pkg/builder/snapbuilder" "github.com/openebs/zfs-localpv/pkg/builder/volbuilder" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/klog/v2" ) @@ -213,13 +215,6 @@ func DeleteSnapshot(snapname string) (err error) { return } -// GetVolume the corresponding ZFSVolume CR -func GetVolume(volumeID string) (*apis.ZFSVolume, error) { - return volbuilder.NewKubeclient(). - WithNamespace(OpenEBSNamespace). - Get(volumeID, metav1.GetOptions{}) -} - // DeleteVolume deletes the corresponding ZFSVol CR func DeleteVolume(volumeID string) (err error) { err = volbuilder.NewKubeclient().WithNamespace(OpenEBSNamespace).Delete(volumeID) @@ -251,6 +246,18 @@ func GetZFSVolume(volumeID string) (*apis.ZFSVolume, error) { return vol, err } +// UpdateZFSVolumeAnnotation updtates the ZFSVolume CR with the given annotation +func UpdateZFSVolumeAnnotation(vol *apis.ZFSVolume) error { + newVol, err := volbuilder.BuildFrom(vol). + WithAnnotation().Build() + + if err != nil { + return err + } + _, err = volbuilder.NewKubeclient().WithNamespace(OpenEBSNamespace).Update(newVol) + return err +} + // GetZFSVolumeState returns ZFSVolume OwnerNode and State for // the given volume. CreateVolume request may call it again and // again until volume is "Ready". @@ -441,3 +448,46 @@ func IsVolumeReady(vol *apis.ZFSVolume) bool { return false } + +// GetSnapshotForVolume fetches all the snapshots for the given volume +func GetSnapshotForVolume(volumeID string) (*apis.ZFSSnapshotList, error) { + listOptions := metav1.ListOptions{ + LabelSelector: ZFSVolKey + "=" + volumeID, + } + snapList, err := snapbuilder.NewKubeclient().WithNamespace(OpenEBSNamespace).List(listOptions) + return snapList, err +} + +// MarkForDeletion marks the volume for deletion by adding the annotation +func MarkForDeletion(volumeName string) error { + zv, err := GetZFSVolume(volumeName) + if err != nil { + klog.Errorf("failed to get ZV %s: %v", volumeName, err) + return err + } + + err = UpdateZFSVolumeAnnotation(zv) + if err != nil { + klog.Errorf("Failed to annotate the ZV with marked for deletion %s: %v", volumeName, err) + return err + } + return nil +} + +// IsVolumeEligibleForDeletion checks if the volume can be deleted or not +func IsVolumeEligibleForDeletion(volumeName string) (bool, error) { + + zfsVol, err := GetZFSVolume(volumeName) + if err != nil { + return false, status.Errorf( + codes.Internal, + "failed to get ZFSVolume %s: %v", + volumeName, + err, + ) + } + if zfsVol.Annotations[volbuilder.MarkForDeletionAnnotation] == "true" { + return true, nil + } + return false, nil +} diff --git a/tests/provision_test.go b/tests/provision_test.go index 35246080a..bef1b275d 100644 --- a/tests/provision_test.go +++ b/tests/provision_test.go @@ -41,7 +41,7 @@ func exhaustiveVolumeTests(parameters map[string]string) { snapshotAndCloneCreate() // btrfs does not support online resize if fstype != "btrfs" { - By("Resizing the PVC", resizeAndVerifyPVC) + By("Resizing the PVC", func() { resizeAndVerifyPVC(pvcNameFS) }) } snapshotAndCloneCleanUp() cleanUp() @@ -51,8 +51,8 @@ func exhaustiveVolumeTests(parameters map[string]string) { func create(parameters map[string]string) { By("####### Creating the storage class : " + parameters["fstype"] + " #######") createFstypeStorageClass(parameters) - By("creating and verifying PVC bound status", createAndVerifyPVC) - By("Creating and deploying app pod", createDeployVerifyApp) + By("creating and verifying PVC bound status", func() { createAndVerifyPVC(pvcNameFS) }) + By("Creating and deploying app pod", func() { createDeployVerifyApp(appNameFS, pvcNameFS) }) By("verifying ZFSVolume object", VerifyZFSVolume) By("verifying storage class parameters") VerifyStorageClassParams(parameters) @@ -60,53 +60,102 @@ func create(parameters map[string]string) { // Creates the snapshot/clone resources func snapshotAndCloneCreate() { - createSnapshot(pvcName, snapName) - verifySnapshotCreated(snapName) - createClone(clonePvcName, snapName, scObj.Name) - By("Creating and deploying clone app pod", createDeployVerifyCloneApp) + createSnapshot(pvcNameFS, snapNameFS) + verifySnapshotCreated(snapNameFS) + createClone(clonePvcNameFS, snapNameFS, scObj.Name) + By("Creating and deploying clone app pod", func() { createDeployVerifyCloneApp(cloneAppNameFS, clonePvcNameFS) }) } // Removes the snapshot/clone resources func snapshotAndCloneCleanUp() { - deleteAppDeployment(cloneAppName) - deletePVC(clonePvcName) - deleteSnapshot(pvcName, snapName) + deleteAppDeployment(cloneAppNameFS) + deletePVC(clonePvcNameFS) + deleteSnapshot(pvcNameFS, snapNameFS) } // Removes the resources func cleanUp() { - deleteAppDeployment(appName) - deletePVC(pvcName) + deleteAppDeployment(appNameFS) + deletePVC(pvcNameFS) By("Deleting storage class", deleteStorageClass) } func blockVolCreationTest() { By("Creating default storage class", createStorageClass) - By("creating and verifying PVC bound status", createAndVerifyBlockPVC) + By("creating and verifying PVC bound status", func() { createAndVerifyPVC(pvcNameBlock) }) - By("Creating and deploying app pod", createDeployVerifyBlockApp) + By("Creating and deploying app pod", func() { createDeployVerifyApp(appNameBlock, pvcNameBlock) }) By("verifying ZFSVolume object", VerifyZFSVolume) By("verifying ZFSVolume property change", VerifyZFSVolumePropEdit) - By("Deleting application deployment") - createSnapshot(pvcName, snapName) - verifySnapshotCreated(snapName) - createClone(clonePvcName, snapName, scObj.Name) - By("Creating and deploying clone app pod", createDeployVerifyCloneApp) + createSnapshot(pvcNameBlock, snapNameBlock) + verifySnapshotCreated(snapNameBlock) + createClone(clonePvcNameBlock, snapNameBlock, scObj.Name) + By("Creating and deploying clone app pod", func() { createDeployVerifyCloneApp(cloneAppNameBlock, clonePvcNameBlock) }) - By("Deleting clone and main application deployment") - deleteAppDeployment(cloneAppName) - deleteAppDeployment(appName) + By("Deleting main application deployment") + deleteAppDeployment(appNameBlock) - By("Deleting snapshot, main pvc and clone pvc") - deletePVC(clonePvcName) - deleteSnapshot(pvcName, snapName) - deletePVC(pvcName) + zvName := getZVName(pvcNameBlock) + By("Deleting main pvc") + deletePVC(pvcNameBlock) + By("Verifying ZFSVolume object after pvc deletion when snapshot is present", VerifyZFSVolume) + + By("Deleting clone application deployment") + deleteAppDeployment(cloneAppNameBlock) + + By("Deleting snapshot and clone pvc") + + deletePVC(clonePvcNameBlock) + By("Verifying that ZV is present after pvc deletion ", func() { IsZVPresentConsistently(zvName) }) + deleteSnapshot(pvcNameBlock, snapNameBlock) + By("Verifying that ZV is deleted after snapshot deletion ", func() { IsZVDeletedEventually(zvName) }) + + By("Deleting storage class", deleteStorageClass) +} + +func blockVolCreationWithReclaimRetainTest() { + By("Creating storage class retain reclaim policy", createStorageClassWithReclaimPolicy) + By("creating and verifying PVC bound status", func() { createAndVerifyPVC(pvcNameBlock) }) + + By("Creating and deploying app pod", func() { createDeployVerifyApp(appNameBlock, pvcNameBlock) }) + By("verifying ZFSVolume object", VerifyZFSVolume) + + createSnapshot(pvcNameBlock, snapNameBlock) + verifySnapshotCreated(snapNameBlock) + + By("Deleting main application deployment") + deleteAppDeployment(appNameBlock) + + zvName := getZVName(pvcNameBlock) + By("Deleting main pvc") + deletePVC(pvcNameBlock) + + By("Verifying ZFSVolume object after pvc deletion when snapshot is present", VerifyZFSVolume) + + By("Verifying that ZV is present after pvc deletion ", func() { IsZVPresentConsistently(zvName) }) + By("Deleting snapshot") + deleteSnapshot(pvcNameBlock, snapNameBlock) + By("Verifying that ZV is present after pvc deletion ", func() { IsZVPresentConsistently(zvName) }) + deletePV(zvName) + By("Verifying that ZV is present after pv deletion ", func() { IsZVPresentConsistently(zvName) }) + By("Create and Verifying PV from the retained ZV ", func() { + createAndVerifyPVFromRetainedZV(pvFromRetainZV, zvName) + }) + By("creating and verifying PVC bound status from retained ZV", func() { createAndVerifyPVC(pvcFromRetainZV) }) + + zvNewName := getZVName(pvcFromRetainZV) + deletePVC(pvcFromRetainZV) + deletePV(zvNewName) + By("Deleting the ZV for cleanup ", func() { DeleteZV(zvName) }) By("Deleting storage class", deleteStorageClass) + } func volumeCreationTest() { By("Running volume creation test", fsVolCreationTest) By("Running block volume creation test", blockVolCreationTest) + By("Running block volume creation test with retain reclaim policy ", blockVolCreationWithReclaimRetainTest) + } diff --git a/tests/pv/build.go b/tests/pv/build.go new file mode 100644 index 000000000..f0e46d096 --- /dev/null +++ b/tests/pv/build.go @@ -0,0 +1,80 @@ +package pv + +import ( + "github.com/openebs/lib-csi/pkg/common/errors" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" +) + +// PV is a wrapper over persistentvolume api +type pvBuildOption func(*PV) + +// PV is a wrapper over persistentvolume api +// object. It provides build, validations and other common +// logic to be used by various feature specific callers. +type PV struct { + object *corev1.PersistentVolume +} + +// Builder is the builder object for PV +type Builder struct { + pv *PV + errs []error +} + +// NewBuilder returns new instance of Builder +func NewBuilder() *Builder { + return &Builder{pv: &PV{object: &corev1.PersistentVolume{}}} +} + +// WithName sets the Name field of PV with provided value. +func (b *Builder) WithName(name string) *Builder { + if len(name) == 0 { + b.errs = append(b.errs, errors.New("failed to build PV object: missing PV name")) + return b + } + b.pv.object.Name = name + return b +} + +// WithStorageClass sets the StorageClass field of PV with provided arguments +func (b *Builder) WithStorageClass(scName string) *Builder { + if len(scName) == 0 { + b.errs = append(b.errs, errors.New("failed to build PV object: missing storageclass name")) + return b + } + b.pv.object.Spec.StorageClassName = scName + return b +} + +// WithCapacity sets the Capacity field in PV with provided arguments +func (b *Builder) WithCapacity(capacity string) *Builder { + resCapacity, err := resource.ParseQuantity(capacity) + if err != nil { + b.errs = append(b.errs, errors.Wrapf(err, "failed to build PV object: failed to parse capacity {%s}", capacity)) + return b + } + resourceList := corev1.ResourceList{ + corev1.ResourceName(corev1.ResourceStorage): resCapacity, + } + b.pv.object.Spec.Capacity = resourceList + return b +} + +// WithAccessModes sets the AccessMode field in PV with provided arguments +func (b *Builder) WithAccessModes(accessMode []corev1.PersistentVolumeAccessMode) *Builder { + if len(accessMode) == 0 { + b.errs = append(b.errs, errors.New("failed to build PV object: missing accessmodes")) + return b + } + b.pv.object.Spec.AccessModes = accessMode + return b +} + +// Build returns the PV API instance +func (b *Builder) Build() (*corev1.PersistentVolume, error) { + if len(b.errs) > 0 { + return nil, errors.Errorf("%+v", b.errs) + } + return b.pv.object, nil +} diff --git a/tests/pv/kubernetes.go b/tests/pv/kubernetes.go new file mode 100644 index 000000000..a34912c0d --- /dev/null +++ b/tests/pv/kubernetes.go @@ -0,0 +1,178 @@ +package pv + +import ( + "context" + "strings" + + "github.com/openebs/lib-csi/pkg/common/errors" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + client "github.com/openebs/lib-csi/pkg/common/kubernetes/client" + "k8s.io/client-go/kubernetes" +) + +// getClientsetFn is a typed function that +// abstracts fetching of clientset +type getClientsetFn func() (clientset *kubernetes.Clientset, err error) + +// getClientsetFromPathFn is a typed function that +// abstracts fetching of clientset from kubeConfigPath +type getClientsetForPathFn func(kubeConfigPath string) (clientset *kubernetes.Clientset, err error) + +// getpvcFn is a typed function that +// abstracts fetching of pvc +type getFn func(cli *kubernetes.Clientset, name string, opts metav1.GetOptions) (*corev1.PersistentVolume, error) + +// createFn is a typed function that abstracts +// creation of pvc +type createFn func(cli *kubernetes.Clientset, pvc *corev1.PersistentVolume) (*corev1.PersistentVolume, error) + +// deleteFn is a typed function that abstracts +// deletion of pvcs +type deleteFn func(cli *kubernetes.Clientset, name string, deleteOpts *metav1.DeleteOptions) error + +// Kubeclient enables kubernetes API operations +// on pvc instance +type Kubeclient struct { + // clientset refers to pvc clientset + // that will be responsible to + // make kubernetes API calls + clientset *kubernetes.Clientset + + // kubeconfig path to get kubernetes clientset + kubeConfigPath string + + // functions useful during mocking + getClientset getClientsetFn + getClientsetForPath getClientsetForPathFn + get getFn + create createFn + del deleteFn +} + +// KubeclientBuildOption abstracts creating an +// instance of kubeclient +type KubeclientBuildOption func(*Kubeclient) + +// withDefaults sets the default options +// of kubeclient instance +func (k *Kubeclient) withDefaults() { + if k.getClientset == nil { + k.getClientset = func() (clients *kubernetes.Clientset, err error) { + return client.New().Clientset() + } + } + + if k.getClientsetForPath == nil { + k.getClientsetForPath = func(kubeConfigPath string) (clients *kubernetes.Clientset, err error) { + return client.New(client.WithKubeConfigPath(kubeConfigPath)).Clientset() + } + } + + if k.get == nil { + k.get = func(cli *kubernetes.Clientset, name string, opts metav1.GetOptions) (*corev1.PersistentVolume, error) { + return cli.CoreV1().PersistentVolumes().Get(context.TODO(), name, opts) + } + } + + if k.create == nil { + k.create = func(cli *kubernetes.Clientset, pv *corev1.PersistentVolume) (*corev1.PersistentVolume, error) { + return cli.CoreV1().PersistentVolumes().Create(context.TODO(), pv, metav1.CreateOptions{}) + } + } + + if k.del == nil { + k.del = func(cli *kubernetes.Clientset, name string, deleteOpts *metav1.DeleteOptions) error { + return cli.CoreV1().PersistentVolumes().Delete(context.TODO(), name, *deleteOpts) + } + } + +} + +// WithClientSet sets the kubernetes client against +// the kubeclient instance +func WithClientSet(c *kubernetes.Clientset) KubeclientBuildOption { + return func(k *Kubeclient) { + k.clientset = c + } +} + +// WithKubeConfigPath sets the kubeConfig path +// against client instance +func WithKubeConfigPath(path string) KubeclientBuildOption { + return func(k *Kubeclient) { + k.kubeConfigPath = path + } +} + +// NewKubeClient returns a new instance of kubeclient meant for +// pvc operations +func NewKubeClient(opts ...KubeclientBuildOption) *Kubeclient { + k := &Kubeclient{} + for _, o := range opts { + o(k) + } + k.withDefaults() + return k +} + +func (k *Kubeclient) getClientsetForPathOrDirect() (*kubernetes.Clientset, error) { + if k.kubeConfigPath != "" { + return k.getClientsetForPath(k.kubeConfigPath) + } + return k.getClientset() +} + +// getClientsetOrCached returns either a new instance +// of kubernetes client or its cached copy +func (k *Kubeclient) getClientsetOrCached() (*kubernetes.Clientset, error) { + if k.clientset != nil { + return k.clientset, nil + } + + cs, err := k.getClientsetForPathOrDirect() + if err != nil { + return nil, errors.Wrapf(err, "failed to get clientset") + } + k.clientset = cs + return k.clientset, nil +} + +// Get returns a pvc resource +// instances present in kubernetes cluster +func (k *Kubeclient) Get(name string, opts metav1.GetOptions) (*corev1.PersistentVolume, error) { + if strings.TrimSpace(name) == "" { + return nil, errors.New("failed to get pvc: missing pv name") + } + cli, err := k.getClientsetOrCached() + if err != nil { + return nil, errors.Wrapf(err, "failed to get pv {%s}", name) + } + return k.get(cli, name, opts) +} + +// Delete deletes a pvc instance from the +// kubecrnetes cluster +func (k *Kubeclient) Delete(name string, deleteOpts *metav1.DeleteOptions) error { + if strings.TrimSpace(name) == "" { + return errors.New("failed to delete pv: missing pv name") + } + cli, err := k.getClientsetOrCached() + if err != nil { + return errors.Wrapf(err, "failed to delete pv {%s}", name) + } + return k.del(cli, name, deleteOpts) +} + +// Create creates a pv in specified namespace in kubernetes cluster +func (k *Kubeclient) Create(pv *corev1.PersistentVolume) (*corev1.PersistentVolume, error) { + if pv == nil { + return nil, errors.New("failed to create pv: nil pv object") + } + cli, err := k.getClientsetOrCached() + if err != nil { + return nil, errors.Wrapf(err, "failed to create pv {%s} ", pv.Name) + } + return k.create(cli, pv) +} diff --git a/tests/pv/persistentvolume.go b/tests/pv/persistentvolume.go new file mode 100644 index 000000000..3f5401b7e --- /dev/null +++ b/tests/pv/persistentvolume.go @@ -0,0 +1,19 @@ +package pv + +import ( + corev1 "k8s.io/api/core/v1" +) + +// IsAvailable returns true if the pv is bounded +func (p *PV) IsAvailable() bool { + return p.object.Status.Phase == corev1.VolumeAvailable +} + +// NewForAPIObject returns a new instance of PV +func NewForAPIObject(obj *corev1.PersistentVolume, opts ...pvBuildOption) *PV { + p := &PV{object: obj} + for _, o := range opts { + o(p) + } + return p +} diff --git a/tests/sc/build.go b/tests/sc/build.go index 07386a936..2484e4fa9 100644 --- a/tests/sc/build.go +++ b/tests/sc/build.go @@ -18,6 +18,7 @@ package sc import ( "github.com/openebs/lib-csi/pkg/common/errors" + corev1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" ) @@ -89,6 +90,12 @@ func (b *Builder) WithProvisioner(provisioner string) *Builder { return b } +// WithReclaimPolicy sets the ReclaimPolicy field of storageclass with provided argument. +func (b *Builder) WithReclaimPolicy(reclaimPolicy *corev1.PersistentVolumeReclaimPolicy) *Builder { + b.sc.object.ReclaimPolicy = reclaimPolicy + return b +} + // WithVolumeExpansion sets the AllowedVolumeExpansion field of storageclass with provided argument. func (b *Builder) WithVolumeExpansion(expansionAllowed bool) *Builder { b.sc.object.AllowVolumeExpansion = &expansionAllowed diff --git a/tests/suite_test.go b/tests/suite_test.go index 16dfd82f7..020467fc4 100644 --- a/tests/suite_test.go +++ b/tests/suite_test.go @@ -25,6 +25,7 @@ import ( "github.com/openebs/zfs-localpv/pkg/builder/volbuilder" "github.com/openebs/zfs-localpv/tests/deploy" "github.com/openebs/zfs-localpv/tests/pod" + "github.com/openebs/zfs-localpv/tests/pv" "github.com/openebs/zfs-localpv/tests/pvc" "github.com/openebs/zfs-localpv/tests/sc" appsv1 "k8s.io/api/apps/v1" @@ -41,18 +42,33 @@ const ( ) var ( - ZFSClient *volbuilder.Kubeclient - SCClient *sc.Kubeclient - PVCClient *pvc.Kubeclient - DeployClient *deploy.Kubeclient - PodClient *pod.KubeClient - scName = "zfspv-sc" - ZFSProvisioner = "zfs.csi.openebs.io" - pvcName = "zfspv-pvc" - snapName = "zfspv-snap" - appName = "busybox-zfspv" - clonePvcName = "zfspv-pvc-clone" - cloneAppName = "busybox-zfspv-clone" + ZFSClient *volbuilder.Kubeclient + SCClient *sc.Kubeclient + PVCClient *pvc.Kubeclient + PVClient *pv.Kubeclient + DeployClient *deploy.Kubeclient + PodClient *pod.KubeClient + scName = "zfspv-sc" + ZFSProvisioner = "zfs.csi.openebs.io" + RetainReclaimPolicy = corev1.PersistentVolumeReclaimPolicy(corev1.PersistentVolumeReclaimRetain) + + pvcNameFS = "zfspv-pvc-fs" + pvcNameBlock = "zfspv-pvc-block" + + appNameFS = "busybox-zfspv-fs" + appNameBlock = "busybox-zfspv-block" + + snapNameFS = "zfspv-snap-fs" + snapNameBlock = "zfspv-snap-block" + + clonePvcNameFS = "zfspv-pvc-clone-fs" + clonePvcNameBlock = "zfspv-pvc-clone-block" + + cloneAppNameFS = "busybox-zfspv-clone-fs" + cloneAppNameBlock = "busybox-zfspv-clone-block" + + pvFromRetainZV = "pv-from-retain-zv" + pvcFromRetainZV = "pvc-from-retain-zv" scObj *storagev1.StorageClass deployObj *appsv1.Deployment @@ -76,6 +92,7 @@ func init() { } SCClient = sc.NewKubeClient(sc.WithKubeConfigPath(KubeConfigPath)) PVCClient = pvc.NewKubeClient(pvc.WithKubeConfigPath(KubeConfigPath)) + PVClient = pv.NewKubeClient(pv.WithKubeConfigPath(KubeConfigPath)) DeployClient = deploy.NewKubeClient(deploy.WithKubeConfigPath(KubeConfigPath)) PodClient = pod.NewKubeClient(pod.WithKubeConfigPath(KubeConfigPath)) ZFSClient = volbuilder.NewKubeclient(volbuilder.WithKubeConfigPath(KubeConfigPath)) diff --git a/tests/utils.go b/tests/utils.go index 3bd459e9e..60c0e114e 100644 --- a/tests/utils.go +++ b/tests/utils.go @@ -32,6 +32,7 @@ import ( "github.com/openebs/zfs-localpv/tests/k8svolume" "github.com/openebs/zfs-localpv/tests/pod" "github.com/openebs/zfs-localpv/tests/pts" + "github.com/openebs/zfs-localpv/tests/pv" "github.com/openebs/zfs-localpv/tests/pvc" "github.com/openebs/zfs-localpv/tests/sc" corev1 "k8s.io/api/core/v1" @@ -53,6 +54,18 @@ func IsPVCBoundEventually(pvcName string) bool { Should(gomega.BeTrue()) } +// IsPVAvailableEventually checks if the pv is bound or not eventually +func IsPVAvailableEventually(pvName string) bool { + return gomega.Eventually(func() bool { + volume, err := PVClient. + Get(pvName, metav1.GetOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + return pv.NewForAPIObject(volume).IsAvailable() + }, + 60, 5). + Should(gomega.BeTrue()) +} + // IsPVCResizedEventually checks if the pvc is bound or not eventually func IsPVCResizedEventually(pvcName string, newCapacity string) bool { newStorage, err := resource.ParseQuantity(newCapacity) @@ -117,7 +130,7 @@ func GetVolumeProperty(vol *apis.ZFSVolume, prop string) (string, error) { // else returns false func IsPVCDeletedEventually(pvcName string) bool { return gomega.Eventually(func() bool { - _, err := PVCClient. + _, err := PVCClient.WithNamespace(OpenEBSNamespace). Get(pvcName, metav1.GetOptions{}) return k8serrors.IsNotFound(err) }, @@ -125,6 +138,32 @@ func IsPVCDeletedEventually(pvcName string) bool { Should(gomega.BeTrue()) } +// IsPVDeletedEventually tries to get the deleted pv +// and returns true if pv is not found +// else returns false +func IsPVDeletedEventually(pvName string) bool { + return gomega.Eventually(func() bool { + _, err := PVClient. + Get(pvName, metav1.GetOptions{}) + return k8serrors.IsNotFound(err) + }, + 120, 10). + Should(gomega.BeTrue()) +} + +// IsZVDeletedEventually tries to get the deleted zv +// and returns true if zv is not found +// else returns false +func IsZVDeletedEventually(zvName string) bool { + return gomega.Eventually(func() bool { + _, err := ZFSClient.WithNamespace(OpenEBSNamespace). + Get(zvName, metav1.GetOptions{}) + return k8serrors.IsNotFound(err) + }, + 120, 10). + Should(gomega.BeTrue()) +} + // VerifyStorageClassParams verifies the volume properties set at creation time func VerifyStorageClassParams(property map[string]string) { vol, err := ZFSClient.WithNamespace(OpenEBSNamespace). @@ -197,6 +236,28 @@ func createFstypeStorageClass(addons map[string]string) { gomega.Expect(err).To(gomega.BeNil(), "while creating a ext4 storageclass {%s}", scName) } +func createStorageClassWithReclaimPolicy() { + var ( + err error + ) + + parameters := map[string]string{ + "poolname": POOLNAME, + } + + ginkgo.By("building a default storage class") + scObj, err = sc.NewBuilder(). + WithGenerateName(scName). + WithParametersNew(parameters). + WithProvisioner(ZFSProvisioner). + WithReclaimPolicy(&RetainReclaimPolicy).Build() + gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), + "while building default storageclass obj with prefix {%s}", scName) + + scObj, err = SCClient.Create(scObj) + gomega.Expect(err).To(gomega.BeNil(), "while creating a default storageclass {%s}", scName) +} + func createStorageClass() { var ( err error @@ -349,18 +410,29 @@ func deleteStorageClass() { "while deleting zfs storageclass {%s}", scObj.Name) } -func createAndVerifyPVC() { +func createAndVerifyPVC(pvcName string) { var ( - err error - pvcName = "zfspv-pvc" + err error ) - ginkgo.By("building a pvc") + ginkgo.By("building a pvc " + pvcName) pvcObj, err = pvc.NewBuilder(). WithName(pvcName). WithNamespace(OpenEBSNamespace). WithStorageClass(scObj.Name). WithAccessModes(accessModes). WithCapacity(capacity).Build() + + if pvcName == "zfspv-pvc-block" { + volmode := corev1.PersistentVolumeBlock + pvcObj.Spec.VolumeMode = &volmode + } + + if pvcName == "pvc-from-retain-zv" { + volmode := corev1.PersistentVolumeBlock + pvcObj.Spec.VolumeMode = &volmode + pvcObj.Spec.VolumeName = "pv-from-retain-zv" + } + gomega.Expect(err).ShouldNot( gomega.HaveOccurred(), "while building pvc {%s} in namespace {%s}", @@ -377,12 +449,6 @@ func createAndVerifyPVC() { OpenEBSNamespace, ) - ginkgo.By("verifying pvc status as bound") - - status := IsPVCBoundEventually(pvcName) - gomega.Expect(status).To(gomega.Equal(true), - "while checking status equal to bound") - pvcObj, err = PVCClient.WithNamespace(OpenEBSNamespace).Get(pvcObj.Name, metav1.GetOptions{}) gomega.Expect(err).To( gomega.BeNil(), @@ -390,39 +456,6 @@ func createAndVerifyPVC() { pvcName, OpenEBSNamespace, ) -} - -func createAndVerifyBlockPVC() { - var ( - err error - pvcName = "zfspv-pvc" - ) - - volmode := corev1.PersistentVolumeBlock - - ginkgo.By("building a pvc") - pvcObj, err = pvc.NewBuilder(). - WithName(pvcName). - WithNamespace(OpenEBSNamespace). - WithStorageClass(scObj.Name). - WithAccessModes(accessModes). - WithVolumeMode(&volmode). - WithCapacity(capacity).Build() - gomega.Expect(err).ShouldNot( - gomega.HaveOccurred(), - "while building pvc {%s} in namespace {%s}", - pvcName, - OpenEBSNamespace, - ) - - ginkgo.By("creating above pvc") - pvcObj, err = PVCClient.WithNamespace(OpenEBSNamespace).Create(pvcObj) - gomega.Expect(err).To( - gomega.BeNil(), - "while creating pvc {%s} in namespace {%s}", - pvcName, - OpenEBSNamespace, - ) ginkgo.By("verifying pvc status as bound") @@ -439,10 +472,9 @@ func createAndVerifyBlockPVC() { ) } -func resizeAndVerifyPVC() { +func resizeAndVerifyPVC(pvcName string) { var ( - err error - pvcName = "zfspv-pvc" + err error ) ginkgo.By("updating the pvc with new size") pvcObj, err = PVCClient.WithNamespace(OpenEBSNamespace).Get(pvcObj.Name, metav1.GetOptions{}) @@ -476,13 +508,17 @@ func resizeAndVerifyPVC() { OpenEBSNamespace, ) } -func createDeployVerifyApp() { +func createDeployVerifyApp(appName, pvcName string) { ginkgo.By("creating and deploying app pod") - createAndDeployAppPod(appName, pvcName) + if pvcName == "zfspv-pvc-block" || pvcName == "pvc-name-for-del-test" { + createAndDeployBlockAppPod(appName, pvcName) + } else { + createAndDeployAppPod(appName, pvcName) + } ginkgo.By("verifying app pod is running", func() { verifyAppPodRunning(appName) }) } -func createDeployVerifyCloneApp() { +func createDeployVerifyCloneApp(cloneAppName, clonePvcName string) { ginkgo.By("creating and deploying app pod") createAndDeployAppPod(cloneAppName, clonePvcName) ginkgo.By("verifying app pod is running", func() { verifyAppPodRunning(cloneAppName) }) @@ -545,7 +581,7 @@ func createAndDeployAppPod(appname string, pvcname string) { ) } -func createAndDeployBlockAppPod() { +func createAndDeployBlockAppPod(appName, pvcName string) { var err error labels := map[string]string{ "app": "busybox", @@ -585,7 +621,7 @@ func createAndDeployBlockAppPod() { WithVolumeBuilders( k8svolume.NewBuilder(). WithName("datavol1"). - WithPVCSource(pvcObj.Name), + WithPVCSource(pvcName), ). WithTerminationGracePeriodSeconds(5), ). @@ -602,11 +638,6 @@ func createAndDeployBlockAppPod() { ) } -func createDeployVerifyBlockApp() { - ginkgo.By("creating and deploying app pod", createAndDeployBlockAppPod) - ginkgo.By("verifying app pod is running", func() { verifyAppPodRunning(appName) }) -} - func verifyAppPodRunning(appname string) { var err error gomega.Eventually(func() bool { @@ -643,6 +674,32 @@ func deletePVC(pvcname string) { gomega.Expect(status).To(gomega.Equal(true), "while trying to get deleted pvc") } +func deletePV(pvName string) { + err := PVClient.Delete(pvName, &metav1.DeleteOptions{}) + gomega.Expect(err).To( + gomega.BeNil(), + "while deleting pv {%s} ", + pvName, + ) + ginkgo.By("verifying deleted pv") + status := IsPVDeletedEventually(pvName) + gomega.Expect(status).To(gomega.Equal(true), "while trying to get deleted pv") +} + +// DeleteZV deletes the zv +func DeleteZV(zvName string) { + err := ZFSClient.WithNamespace(OpenEBSNamespace).Delete(zvName) + gomega.Expect(err).To( + gomega.BeNil(), + "while deleting zv {%s} in namespace {%s}", + zvName, + OpenEBSNamespace, + ) + ginkgo.By("verifying deleted zv") + status := IsZVDeletedEventually(zvName) + gomega.Expect(status).To(gomega.Equal(true), "while trying to get deleted zv") +} + func getStoragClassParams() []map[string]string { return []map[string]string{ { @@ -713,3 +770,74 @@ func getStoragClassParams() []map[string]string { }, } } + +// IsZVPresentConsistently checks if the zfs volume is present or not consistently +func IsZVPresentConsistently(zvName string) bool { + return gomega.Consistently(func() bool { + volume, err := ZFSClient.WithNamespace(OpenEBSNamespace).Get(zvName, metav1.GetOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + return volume.Name == zvName + }, + 30, 5). // Check consistency for 60 seconds, polling every 5 seconds + Should(gomega.BeTrue()) +} + +// getZVName return the zv name +func getZVName(pvcName string) string { + pvcObj, _ = PVCClient.WithNamespace(OpenEBSNamespace).Get(pvcName, metav1.GetOptions{}) + return pvcObj.Spec.VolumeName + +} + +func createAndVerifyPVFromRetainedZV(pvName, volumeHandle string) { + var ( + err error + ) + + ginkgo.By("building a pv from retained zv") + pvObj, err := pv.NewBuilder(). + WithName(pvName). + WithStorageClass(scObj.Name). + WithAccessModes(accessModes). + WithCapacity(capacity).Build() + + source := corev1.CSIPersistentVolumeSource{Driver: ZFSProvisioner, VolumeHandle: volumeHandle} + volmode := corev1.PersistentVolumeBlock + pvObj.Spec.VolumeMode = &volmode + pvObj.Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimRetain + pvObj.Spec.CSI = &source + + gomega.Expect(err).ShouldNot( + gomega.HaveOccurred(), + "while building pv {%s} ", + pvName, + ) + + ginkgo.By("creating above pv") + pvObj, err = PVClient.Create(pvObj) + gomega.Expect(err).To( + gomega.BeNil(), + "while creating pv {%s} ", + pvName, + ) + + ginkgo.By("verifying pvc status as bound ") + + pvObj, err = PVClient.Get(pvName, metav1.GetOptions{}) + gomega.Expect(err).To( + gomega.BeNil(), + "while retrieving pvc {%s}", + pvName, + ) + + status := IsPVAvailableEventually(pvName) + gomega.Expect(status).To(gomega.Equal(true), + "while checking status equal to available") + + pvObj, err = PVClient.Get(pvName, metav1.GetOptions{}) + gomega.Expect(err).To( + gomega.BeNil(), + "while retrieving pvc {%s}", + pvName, + ) +}