From 916c067fbb646ef7385ed7ea7442b35670d8ba53 Mon Sep 17 00:00:00 2001 From: Max Petyurenko Date: Tue, 20 Jun 2023 17:06:49 +0300 Subject: [PATCH 1/5] Add LUKS encryption support. --- CHANGELOG.md | 4 + README.md | 15 + VERSION | 2 +- cmd/do-csi-plugin/Dockerfile | 1 + .../releases/csi-digitalocean-dev/driver.yaml | 36 + .../csi-digitalocean-v4.7.0/crds.yaml | 865 ++++++++++++++++++ .../csi-digitalocean-v4.7.0/driver.yaml | 533 +++++++++++ .../snapshot-controller.yaml | 98 ++ .../snapshot-validation-webhook.yaml | 128 +++ driver/controller.go | 65 +- driver/driver_test.go | 9 +- driver/luks_util.go | 402 ++++++++ driver/luks_util_test.go | 110 +++ driver/mounter.go | 151 ++- driver/node.go | 63 +- 15 files changed, 2425 insertions(+), 57 deletions(-) create mode 100644 deploy/kubernetes/releases/csi-digitalocean-v4.7.0/crds.yaml create mode 100644 deploy/kubernetes/releases/csi-digitalocean-v4.7.0/driver.yaml create mode 100644 deploy/kubernetes/releases/csi-digitalocean-v4.7.0/snapshot-controller.yaml create mode 100644 deploy/kubernetes/releases/csi-digitalocean-v4.7.0/snapshot-validation-webhook.yaml create mode 100644 driver/luks_util.go create mode 100644 driver/luks_util_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 946de23a3..f16c2b530 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ ## unreleased +## v4.7.0 - 2023.06.23 +* Add LUKS encryption support + [[GH-514]](https://github.com/digitalocean/csi-digitalocean/pull/514) + ## v4.6.1 - 2023.05.17 * Update CSI sidecars to latest [[GH-502]](https://github.com/digitalocean/csi-digitalocean/pull/502) diff --git a/README.md b/README.md index 66b0647a8..161516ff0 100644 --- a/README.md +++ b/README.md @@ -277,6 +277,21 @@ $ kubectl exec -ti my-csi-app /bin/sh hello-world ``` +## Volume parameters + +This plugin supports the following `StorageClass` parameters: + +For LUKS encryption: + +* `dobs.csi.digitalocean.com/luks-encrypted`: set to the string `"true"` if the volume should be encrypted + with LUKS +* `dobs.csi.digitalocean.com/luks-cipher`: cipher to use; must be supported by the kernel and luks +* `dobs.csi.digitalocean.com/luks-key-size`: key-size to use + +For LUKS encrypted volumes, a secret that contains the LUKS key needs to be referenced through +the `csi.storage.k8s.io/node-stage-secret-name` and `csi.storage.k8s.io/node-stage-secret-namespace` +parameter. See the included `StorageClass` definition. + ## Upgrading When upgrading to a new Kubernetes minor version, you should upgrade the CSI diff --git a/VERSION b/VERSION index c0c6ad02b..79214dce1 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v4.6.1 +v4.7.0 diff --git a/cmd/do-csi-plugin/Dockerfile b/cmd/do-csi-plugin/Dockerfile index bac4cddb5..87bbc2b13 100644 --- a/cmd/do-csi-plugin/Dockerfile +++ b/cmd/do-csi-plugin/Dockerfile @@ -17,6 +17,7 @@ FROM amd64/alpine:3.16 # e2fsprogs-extra is required for resize2fs used for the resize operation # blkid: block device identification tool from util-linux RUN apk add --no-cache ca-certificates \ + cryptsetup \ e2fsprogs \ findmnt \ xfsprogs \ diff --git a/deploy/kubernetes/releases/csi-digitalocean-dev/driver.yaml b/deploy/kubernetes/releases/csi-digitalocean-dev/driver.yaml index b9075d4fe..00a1558af 100644 --- a/deploy/kubernetes/releases/csi-digitalocean-dev/driver.yaml +++ b/deploy/kubernetes/releases/csi-digitalocean-dev/driver.yaml @@ -38,6 +38,36 @@ deletionPolicy: Delete --- +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: do-block-storage-luks-global +provisioner: dobs.csi.digitalocean.com +allowVolumeExpansion: true +parameters: + dobs.csi.digitalocean.com/luks-encrypted: "true" + dobs.csi.digitalocean.com/luks-cipher: "aes-xts-plain64" + dobs.csi.digitalocean.com/luks-key-size: "512" + csi.storage.k8s.io/node-stage-secret-namespace: "kube-system" + csi.storage.k8s.io/node-stage-secret-name: "luks-key" + +--- + +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: do-block-storage-luks +provisioner: dobs.csi.digitalocean.com +allowVolumeExpansion: true +parameters: + dobs.csi.digitalocean.com/luks-encrypted: "true" + dobs.csi.digitalocean.com/luks-cipher: "aes-xts-plain64" + dobs.csi.digitalocean.com/luks-key-size: "512" + csi.storage.k8s.io/node-stage-secret-namespace: ${pvc.namespace} + csi.storage.k8s.io/node-stage-secret-name: ${pvc.name}-luks-key + +--- + kind: StorageClass apiVersion: storage.k8s.io/v1 metadata: @@ -443,6 +473,8 @@ spec: mountPropagation: "Bidirectional" - name: device-dir mountPath: /dev + - name: tmpfs + mountPath: /tmp volumes: - name: registration-dir hostPath: @@ -462,6 +494,10 @@ spec: - name: udev-rules-dir hostPath: path: /etc/udev/rules.d/ + # to make sure temporary stored luks keys never touch a disk + - name: tmpfs + emptyDir: + medium: Memory --- apiVersion: v1 diff --git a/deploy/kubernetes/releases/csi-digitalocean-v4.7.0/crds.yaml b/deploy/kubernetes/releases/csi-digitalocean-v4.7.0/crds.yaml new file mode 100644 index 000000000..f79f0a4df --- /dev/null +++ b/deploy/kubernetes/releases/csi-digitalocean-v4.7.0/crds.yaml @@ -0,0 +1,865 @@ +# Copyright 2022 DigitalOcean +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +############################################## +########### ############ +########### Snapshot CRDs ############ +########### ############ +############################################## +# +# Source: https://github.com/kubernetes-csi/external-snapshotter +# + +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.8.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/665" + creationTimestamp: null + name: volumesnapshotclasses.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotClass + listKind: VolumeSnapshotClassList + plural: volumesnapshotclasses + shortNames: + - vsclass + - vsclasses + singular: volumesnapshotclass + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .driver + name: Driver + type: string + - description: Determines whether a VolumeSnapshotContent created through the + VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. + jsonPath: .deletionPolicy + name: DeletionPolicy + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: VolumeSnapshotClass specifies parameters that a underlying storage + system uses when creating a volume snapshot. A specific VolumeSnapshotClass + is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses + are non-namespaced + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + deletionPolicy: + description: deletionPolicy determines whether a VolumeSnapshotContent + created through the VolumeSnapshotClass should be deleted when its bound + VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". + "Retain" means that the VolumeSnapshotContent and its physical snapshot + on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent + and its physical snapshot on underlying storage system are deleted. + Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the storage driver that handles this + VolumeSnapshotClass. Required. + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + parameters: + additionalProperties: + type: string + description: parameters is a key-value map with storage driver specific + parameters for creating snapshots. These values are opaque to Kubernetes. + type: object + required: + - deletionPolicy + - driver + type: object + served: true + storage: true + subresources: {} + - additionalPrinterColumns: + - jsonPath: .driver + name: Driver + type: string + - description: Determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. + jsonPath: .deletionPolicy + name: DeletionPolicy + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + # This indicates the v1beta1 version of the custom resource is deprecated. + # API requests to this version receive a warning in the server response. + deprecated: true + # This overrides the default warning returned to clients making v1beta1 API requests. + deprecationWarning: "snapshot.storage.k8s.io/v1beta1 VolumeSnapshotClass is deprecated; use snapshot.storage.k8s.io/v1 VolumeSnapshotClass" + schema: + openAPIV3Schema: + description: VolumeSnapshotClass specifies parameters that a underlying storage system uses when creating a volume snapshot. A specific VolumeSnapshotClass is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses are non-namespaced + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + deletionPolicy: + description: deletionPolicy determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the storage driver that handles this VolumeSnapshotClass. Required. + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + parameters: + additionalProperties: + type: string + description: parameters is a key-value map with storage driver specific parameters for creating snapshots. These values are opaque to Kubernetes. + type: object + required: + - deletionPolicy + - driver + type: object + served: false + storage: false + subresources: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- + +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.8.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/665" + creationTimestamp: null + name: volumesnapshotcontents.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotContent + listKind: VolumeSnapshotContentList + plural: volumesnapshotcontents + shortNames: + - vsc + - vscs + singular: volumesnapshotcontent + scope: Cluster + versions: + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: Represents the complete size of the snapshot in bytes + jsonPath: .status.restoreSize + name: RestoreSize + type: integer + - description: Determines whether this VolumeSnapshotContent and its physical + snapshot on the underlying storage system should be deleted when its bound + VolumeSnapshot is deleted. + jsonPath: .spec.deletionPolicy + name: DeletionPolicy + type: string + - description: Name of the CSI driver used to create the physical snapshot on + the underlying storage system. + jsonPath: .spec.driver + name: Driver + type: string + - description: Name of the VolumeSnapshotClass to which this snapshot belongs. + jsonPath: .spec.volumeSnapshotClassName + name: VolumeSnapshotClass + type: string + - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent + object is bound. + jsonPath: .spec.volumeSnapshotRef.name + name: VolumeSnapshot + type: string + - description: Namespace of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. + jsonPath: .spec.volumeSnapshotRef.namespace + name: VolumeSnapshotNamespace + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: VolumeSnapshotContent represents the actual "on-disk" snapshot + object in the underlying storage system + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: spec defines properties of a VolumeSnapshotContent created + by the underlying storage system. Required. + properties: + deletionPolicy: + description: deletionPolicy determines whether this VolumeSnapshotContent + and its physical snapshot on the underlying storage system should + be deleted when its bound VolumeSnapshot is deleted. Supported values + are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent + and its physical snapshot on underlying storage system are kept. + "Delete" means that the VolumeSnapshotContent and its physical snapshot + on underlying storage system are deleted. For dynamically provisioned + snapshots, this field will automatically be filled in by the CSI + snapshotter sidecar with the "DeletionPolicy" field defined in the + corresponding VolumeSnapshotClass. For pre-existing snapshots, users + MUST specify this field when creating the VolumeSnapshotContent + object. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the CSI driver used to create the + physical snapshot on the underlying storage system. This MUST be + the same as the name returned by the CSI GetPluginName() call for + that driver. Required. + type: string + source: + description: source specifies whether the snapshot is (or should be) + dynamically provisioned or already exists, and just requires a Kubernetes + object representation. This field is immutable after creation. Required. + properties: + snapshotHandle: + description: snapshotHandle specifies the CSI "snapshot_id" of + a pre-existing snapshot on the underlying storage system for + which a Kubernetes object representation was (or should be) + created. This field is immutable. + type: string + volumeHandle: + description: volumeHandle specifies the CSI "volume_id" of the + volume from which a snapshot should be dynamically taken from. + This field is immutable. + type: string + type: object + oneOf: + - required: ["snapshotHandle"] + - required: ["volumeHandle"] + sourceVolumeMode: + description: SourceVolumeMode is the mode of the volume whose snapshot + is taken. Can be either “Filesystem” or “Block”. If not specified, + it indicates the source volume's mode is unknown. This field is + immutable. This field is an alpha field. + type: string + volumeSnapshotClassName: + description: name of the VolumeSnapshotClass from which this snapshot + was (or will be) created. Note that after provisioning, the VolumeSnapshotClass + may be deleted or recreated with different set of values, and as + such, should not be referenced post-snapshot creation. + type: string + volumeSnapshotRef: + description: volumeSnapshotRef specifies the VolumeSnapshot object + to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName + field must reference to this VolumeSnapshotContent's name for the + bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent + object, name and namespace of the VolumeSnapshot object MUST be + provided for binding to happen. This field is immutable after creation. + Required. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of + an entire object, this string should contain a valid JSON/Go + field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen + only to have some well-defined way of referencing a part of + an object. TODO: this design is not final and this field is + subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + required: + - deletionPolicy + - driver + - source + - volumeSnapshotRef + type: object + status: + description: status represents the current information of a snapshot. + properties: + creationTime: + description: creationTime is the timestamp when the point-in-time + snapshot is taken by the underlying storage system. In dynamic snapshot + creation case, this field will be filled in by the CSI snapshotter + sidecar with the "creation_time" value returned from CSI "CreateSnapshot" + gRPC call. For a pre-existing snapshot, this field will be filled + with the "creation_time" value returned from the CSI "ListSnapshots" + gRPC call if the driver supports it. If not specified, it indicates + the creation time is unknown. The format of this field is a Unix + nanoseconds time encoded as an int64. On Unix, the command `date + +%s%N` returns the current time in nanoseconds since 1970-01-01 + 00:00:00 UTC. + format: int64 + type: integer + error: + description: error is the last observed error during snapshot creation, + if any. Upon success after retry, this error field will be cleared. + properties: + message: + description: 'message is a string detailing the encountered error + during snapshot creation if specified. NOTE: message may be + logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if a snapshot is ready to be used + to restore a volume. In dynamic snapshot creation case, this field + will be filled in by the CSI snapshotter sidecar with the "ready_to_use" + value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing + snapshot, this field will be filled with the "ready_to_use" value + returned from the CSI "ListSnapshots" gRPC call if the driver supports + it, otherwise, this field will be set to "True". If not specified, + it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + description: restoreSize represents the complete size of the snapshot + in bytes. In dynamic snapshot creation case, this field will be + filled in by the CSI snapshotter sidecar with the "size_bytes" value + returned from CSI "CreateSnapshot" gRPC call. For a pre-existing + snapshot, this field will be filled with the "size_bytes" value + returned from the CSI "ListSnapshots" gRPC call if the driver supports + it. When restoring a volume from this snapshot, the size of the + volume MUST NOT be smaller than the restoreSize if it is specified, + otherwise the restoration will fail. If not specified, it indicates + that the size is unknown. + format: int64 + minimum: 0 + type: integer + snapshotHandle: + description: snapshotHandle is the CSI "snapshot_id" of a snapshot + on the underlying storage system. If not specified, it indicates + that dynamic snapshot creation has either failed or it is still + in progress. + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: Represents the complete size of the snapshot in bytes + jsonPath: .status.restoreSize + name: RestoreSize + type: integer + - description: Determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. + jsonPath: .spec.deletionPolicy + name: DeletionPolicy + type: string + - description: Name of the CSI driver used to create the physical snapshot on the underlying storage system. + jsonPath: .spec.driver + name: Driver + type: string + - description: Name of the VolumeSnapshotClass to which this snapshot belongs. + jsonPath: .spec.volumeSnapshotClassName + name: VolumeSnapshotClass + type: string + - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. + jsonPath: .spec.volumeSnapshotRef.name + name: VolumeSnapshot + type: string + - description: Namespace of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. + jsonPath: .spec.volumeSnapshotRef.namespace + name: VolumeSnapshotNamespace + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + # This indicates the v1beta1 version of the custom resource is deprecated. + # API requests to this version receive a warning in the server response. + deprecated: true + # This overrides the default warning returned to clients making v1beta1 API requests. + deprecationWarning: "snapshot.storage.k8s.io/v1beta1 VolumeSnapshotContent is deprecated; use snapshot.storage.k8s.io/v1 VolumeSnapshotContent" + schema: + openAPIV3Schema: + description: VolumeSnapshotContent represents the actual "on-disk" snapshot object in the underlying storage system + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: spec defines properties of a VolumeSnapshotContent created by the underlying storage system. Required. + properties: + deletionPolicy: + description: deletionPolicy determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. For dynamically provisioned snapshots, this field will automatically be filled in by the CSI snapshotter sidecar with the "DeletionPolicy" field defined in the corresponding VolumeSnapshotClass. For pre-existing snapshots, users MUST specify this field when creating the VolumeSnapshotContent object. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the CSI driver used to create the physical snapshot on the underlying storage system. This MUST be the same as the name returned by the CSI GetPluginName() call for that driver. Required. + type: string + source: + description: source specifies whether the snapshot is (or should be) dynamically provisioned or already exists, and just requires a Kubernetes object representation. This field is immutable after creation. Required. + properties: + snapshotHandle: + description: snapshotHandle specifies the CSI "snapshot_id" of a pre-existing snapshot on the underlying storage system for which a Kubernetes object representation was (or should be) created. This field is immutable. + type: string + volumeHandle: + description: volumeHandle specifies the CSI "volume_id" of the volume from which a snapshot should be dynamically taken from. This field is immutable. + type: string + type: object + volumeSnapshotClassName: + description: name of the VolumeSnapshotClass from which this snapshot was (or will be) created. Note that after provisioning, the VolumeSnapshotClass may be deleted or recreated with different set of values, and as such, should not be referenced post-snapshot creation. + type: string + volumeSnapshotRef: + description: volumeSnapshotRef specifies the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName field must reference to this VolumeSnapshotContent's name for the bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent object, name and namespace of the VolumeSnapshot object MUST be provided for binding to happen. This field is immutable after creation. Required. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. For example, if the object reference is to a container within a pod, this would take on a value like: "spec.containers{name}" (where "name" refers to the name of the container that triggered the event) or if no container name is specified "spec.containers[2]" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. TODO: this design is not final and this field is subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + required: + - deletionPolicy + - driver + - source + - volumeSnapshotRef + type: object + status: + description: status represents the current information of a snapshot. + properties: + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it indicates the creation time is unknown. The format of this field is a Unix nanoseconds time encoded as an int64. On Unix, the command `date +%s%N` returns the current time in nanoseconds since 1970-01-01 00:00:00 UTC. + format: int64 + type: integer + error: + description: error is the last observed error during snapshot creation, if any. Upon success after retry, this error field will be cleared. + properties: + message: + description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if a snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + description: restoreSize represents the complete size of the snapshot in bytes. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown. + format: int64 + minimum: 0 + type: integer + snapshotHandle: + description: snapshotHandle is the CSI "snapshot_id" of a snapshot on the underlying storage system. If not specified, it indicates that dynamic snapshot creation has either failed or it is still in progress. + type: string + type: object + required: + - spec + type: object + served: false + storage: false + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- + +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.8.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/665" + creationTimestamp: null + name: volumesnapshots.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshot + listKind: VolumeSnapshotList + plural: volumesnapshots + shortNames: + - vs + singular: volumesnapshot + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: If a new snapshot needs to be created, this contains the name of + the source PVC from which this snapshot was (or will be) created. + jsonPath: .spec.source.persistentVolumeClaimName + name: SourcePVC + type: string + - description: If a snapshot already exists, this contains the name of the existing + VolumeSnapshotContent object representing the existing snapshot. + jsonPath: .spec.source.volumeSnapshotContentName + name: SourceSnapshotContent + type: string + - description: Represents the minimum size of volume required to rehydrate from + this snapshot. + jsonPath: .status.restoreSize + name: RestoreSize + type: string + - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot. + jsonPath: .spec.volumeSnapshotClassName + name: SnapshotClass + type: string + - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot + object intends to bind to. Please note that verification of binding actually + requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure + both are pointing at each other. Binding MUST be verified prior to usage of + this object. + jsonPath: .status.boundVolumeSnapshotContentName + name: SnapshotContent + type: string + - description: Timestamp when the point-in-time snapshot was taken by the underlying + storage system. + jsonPath: .status.creationTime + name: CreationTime + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: VolumeSnapshot is a user's request for either creating a point-in-time + snapshot of a persistent volume, or binding to a pre-existing snapshot. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: 'spec defines the desired characteristics of a snapshot requested + by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots + Required.' + properties: + source: + description: source specifies where a snapshot will be created from. + This field is immutable after creation. Required. + properties: + persistentVolumeClaimName: + description: persistentVolumeClaimName specifies the name of the + PersistentVolumeClaim object representing the volume from which + a snapshot should be created. This PVC is assumed to be in the + same namespace as the VolumeSnapshot object. This field should + be set if the snapshot does not exists, and needs to be created. + This field is immutable. + type: string + volumeSnapshotContentName: + description: volumeSnapshotContentName specifies the name of a + pre-existing VolumeSnapshotContent object representing an existing + volume snapshot. This field should be set if the snapshot already + exists and only needs a representation in Kubernetes. This field + is immutable. + type: string + type: object + oneOf: + - required: ["persistentVolumeClaimName"] + - required: ["volumeSnapshotContentName"] + volumeSnapshotClassName: + description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass + requested by the VolumeSnapshot. VolumeSnapshotClassName may be + left nil to indicate that the default SnapshotClass should be used. + A given cluster may have multiple default Volume SnapshotClasses: + one default per CSI Driver. If a VolumeSnapshot does not specify + a SnapshotClass, VolumeSnapshotSource will be checked to figure + out what the associated CSI Driver is, and the default VolumeSnapshotClass + associated with that CSI Driver will be used. If more than one VolumeSnapshotClass + exist for a given CSI Driver and more than one have been marked + as default, CreateSnapshot will fail and generate an event. Empty + string is not allowed for this field.' + type: string + required: + - source + type: object + status: + description: status represents the current information of a snapshot. + Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent + objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent + point at each other) before using this object. + properties: + boundVolumeSnapshotContentName: + description: 'boundVolumeSnapshotContentName is the name of the VolumeSnapshotContent + object to which this VolumeSnapshot object intends to bind to. If + not specified, it indicates that the VolumeSnapshot object has not + been successfully bound to a VolumeSnapshotContent object yet. NOTE: + To avoid possible security issues, consumers must verify binding + between VolumeSnapshot and VolumeSnapshotContent objects is successful + (by validating that both VolumeSnapshot and VolumeSnapshotContent + point at each other) before using this object.' + type: string + creationTime: + description: creationTime is the timestamp when the point-in-time + snapshot is taken by the underlying storage system. In dynamic snapshot + creation case, this field will be filled in by the snapshot controller + with the "creation_time" value returned from CSI "CreateSnapshot" + gRPC call. For a pre-existing snapshot, this field will be filled + with the "creation_time" value returned from the CSI "ListSnapshots" + gRPC call if the driver supports it. If not specified, it may indicate + that the creation time of the snapshot is unknown. + format: date-time + type: string + error: + description: error is the last observed error during snapshot creation, + if any. This field could be helpful to upper level controllers(i.e., + application controller) to decide whether they should continue on + waiting for the snapshot to be created based on the type of error + reported. The snapshot controller will keep retrying when an error + occurs during the snapshot creation. Upon success, this error field + will be cleared. + properties: + message: + description: 'message is a string detailing the encountered error + during snapshot creation if specified. NOTE: message may be + logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if the snapshot is ready to be used + to restore a volume. In dynamic snapshot creation case, this field + will be filled in by the snapshot controller with the "ready_to_use" + value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing + snapshot, this field will be filled with the "ready_to_use" value + returned from the CSI "ListSnapshots" gRPC call if the driver supports + it, otherwise, this field will be set to "True". If not specified, + it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + type: string + description: restoreSize represents the minimum size of volume required + to create a volume from this snapshot. In dynamic snapshot creation + case, this field will be filled in by the snapshot controller with + the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. + For a pre-existing snapshot, this field will be filled with the + "size_bytes" value returned from the CSI "ListSnapshots" gRPC call + if the driver supports it. When restoring a volume from this snapshot, + the size of the volume MUST NOT be smaller than the restoreSize + if it is specified, otherwise the restoration will fail. If not + specified, it indicates that the size is unknown. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: If a new snapshot needs to be created, this contains the name of the source PVC from which this snapshot was (or will be) created. + jsonPath: .spec.source.persistentVolumeClaimName + name: SourcePVC + type: string + - description: If a snapshot already exists, this contains the name of the existing VolumeSnapshotContent object representing the existing snapshot. + jsonPath: .spec.source.volumeSnapshotContentName + name: SourceSnapshotContent + type: string + - description: Represents the minimum size of volume required to rehydrate from this snapshot. + jsonPath: .status.restoreSize + name: RestoreSize + type: string + - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot. + jsonPath: .spec.volumeSnapshotClassName + name: SnapshotClass + type: string + - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot object intends to bind to. Please note that verification of binding actually requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure both are pointing at each other. Binding MUST be verified prior to usage of this object. + jsonPath: .status.boundVolumeSnapshotContentName + name: SnapshotContent + type: string + - description: Timestamp when the point-in-time snapshot was taken by the underlying storage system. + jsonPath: .status.creationTime + name: CreationTime + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + # This indicates the v1beta1 version of the custom resource is deprecated. + # API requests to this version receive a warning in the server response. + deprecated: true + # This overrides the default warning returned to clients making v1beta1 API requests. + deprecationWarning: "snapshot.storage.k8s.io/v1beta1 VolumeSnapshot is deprecated; use snapshot.storage.k8s.io/v1 VolumeSnapshot" + schema: + openAPIV3Schema: + description: VolumeSnapshot is a user's request for either creating a point-in-time snapshot of a persistent volume, or binding to a pre-existing snapshot. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: 'spec defines the desired characteristics of a snapshot requested by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots Required.' + properties: + source: + description: source specifies where a snapshot will be created from. This field is immutable after creation. Required. + properties: + persistentVolumeClaimName: + description: persistentVolumeClaimName specifies the name of the PersistentVolumeClaim object representing the volume from which a snapshot should be created. This PVC is assumed to be in the same namespace as the VolumeSnapshot object. This field should be set if the snapshot does not exists, and needs to be created. This field is immutable. + type: string + volumeSnapshotContentName: + description: volumeSnapshotContentName specifies the name of a pre-existing VolumeSnapshotContent object representing an existing volume snapshot. This field should be set if the snapshot already exists and only needs a representation in Kubernetes. This field is immutable. + type: string + type: object + volumeSnapshotClassName: + description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass requested by the VolumeSnapshot. VolumeSnapshotClassName may be left nil to indicate that the default SnapshotClass should be used. A given cluster may have multiple default Volume SnapshotClasses: one default per CSI Driver. If a VolumeSnapshot does not specify a SnapshotClass, VolumeSnapshotSource will be checked to figure out what the associated CSI Driver is, and the default VolumeSnapshotClass associated with that CSI Driver will be used. If more than one VolumeSnapshotClass exist for a given CSI Driver and more than one have been marked as default, CreateSnapshot will fail and generate an event. Empty string is not allowed for this field.' + type: string + required: + - source + type: object + status: + description: status represents the current information of a snapshot. Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object. + properties: + boundVolumeSnapshotContentName: + description: 'boundVolumeSnapshotContentName is the name of the VolumeSnapshotContent object to which this VolumeSnapshot object intends to bind to. If not specified, it indicates that the VolumeSnapshot object has not been successfully bound to a VolumeSnapshotContent object yet. NOTE: To avoid possible security issues, consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object.' + type: string + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it may indicate that the creation time of the snapshot is unknown. + format: date-time + type: string + error: + description: error is the last observed error during snapshot creation, if any. This field could be helpful to upper level controllers(i.e., application controller) to decide whether they should continue on waiting for the snapshot to be created based on the type of error reported. The snapshot controller will keep retrying when an error occurs during the snapshot creation. Upon success, this error field will be cleared. + properties: + message: + description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if the snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + type: string + description: restoreSize represents the minimum size of volume required to create a volume from this snapshot. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + required: + - spec + type: object + served: false + storage: false + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/deploy/kubernetes/releases/csi-digitalocean-v4.7.0/driver.yaml b/deploy/kubernetes/releases/csi-digitalocean-v4.7.0/driver.yaml new file mode 100644 index 000000000..6913961d3 --- /dev/null +++ b/deploy/kubernetes/releases/csi-digitalocean-v4.7.0/driver.yaml @@ -0,0 +1,533 @@ +# Copyright 2022 DigitalOcean +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Install the CSI Driver. This simplifies driver discovery and enables us to +# customize Kubernetes behavior +# https://kubernetes-csi.github.io/docs/csi-driver-object.html +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: dobs.csi.digitalocean.com +spec: + attachRequired: true + podInfoOnMount: true + +--- + +kind: VolumeSnapshotClass +apiVersion: snapshot.storage.k8s.io/v1 +metadata: + name: do-block-storage + annotations: + snapshot.storage.kubernetes.io/is-default-class: "true" +driver: dobs.csi.digitalocean.com +deletionPolicy: Delete + +--- + +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: do-block-storage-luks-global +provisioner: dobs.csi.digitalocean.com +allowVolumeExpansion: true +parameters: + dobs.csi.digitalocean.com/luks-encrypted: "true" + dobs.csi.digitalocean.com/luks-cipher: "aes-xts-plain64" + dobs.csi.digitalocean.com/luks-key-size: "512" + csi.storage.k8s.io/node-stage-secret-namespace: "kube-system" + csi.storage.k8s.io/node-stage-secret-name: "luks-key" + +--- + +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: do-block-storage-luks +provisioner: dobs.csi.digitalocean.com +allowVolumeExpansion: true +parameters: + dobs.csi.digitalocean.com/luks-encrypted: "true" + dobs.csi.digitalocean.com/luks-cipher: "aes-xts-plain64" + dobs.csi.digitalocean.com/luks-key-size: "512" + csi.storage.k8s.io/node-stage-secret-namespace: ${pvc.namespace} + csi.storage.k8s.io/node-stage-secret-name: ${pvc.name}-luks-key + +--- + +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: do-block-storage + annotations: + storageclass.kubernetes.io/is-default-class: "true" +provisioner: dobs.csi.digitalocean.com +allowVolumeExpansion: true + +--- + +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: do-block-storage-retain +provisioner: dobs.csi.digitalocean.com +reclaimPolicy: Retain +allowVolumeExpansion: true + +--- + +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: do-block-storage-xfs +provisioner: dobs.csi.digitalocean.com +parameters: + fstype: xfs +allowVolumeExpansion: true + +--- + +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: do-block-storage-xfs-retain +provisioner: dobs.csi.digitalocean.com +parameters: + fstype: xfs +reclaimPolicy: Retain +allowVolumeExpansion: true + +--- + +############################################## +########### ############ +########### Controller plugin ############ +########### ############ +############################################## + +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: csi-do-controller + namespace: kube-system +spec: + serviceName: "csi-do" + selector: + matchLabels: + app: csi-do-controller + replicas: 1 + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: csi-do-plugin + labels: + app: csi-do-controller + role: csi-do + spec: + priorityClassName: system-cluster-critical + serviceAccount: csi-do-controller-sa + containers: + - name: csi-provisioner + image: registry.k8s.io/sig-storage/csi-provisioner:v3.5.0 + args: + - "--csi-address=$(ADDRESS)" + - "--default-fstype=ext4" + - "--v=5" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: csi-attacher + image: registry.k8s.io/sig-storage/csi-attacher:v4.3.0 + args: + - "--csi-address=$(ADDRESS)" + - "--v=5" + - "--reconcile-sync=30m" + - "--timeout=2m" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: csi-snapshotter + image: registry.k8s.io/sig-storage/csi-snapshotter:v6.2.1 + args: + - "--csi-address=$(ADDRESS)" + - "--v=5" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + imagePullPolicy: IfNotPresent + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: csi-resizer + image: registry.k8s.io/sig-storage/csi-resizer:v1.8.0 + args: + - "--csi-address=$(ADDRESS)" + - "--timeout=30s" + - "--v=5" + # DO volumes support online resize. + - "--handle-volume-inuse-error=false" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: csi-do-plugin + image: digitalocean/do-csi-plugin:v4.7.0 + args : + - "--endpoint=$(CSI_ENDPOINT)" + - "--token=$(DIGITALOCEAN_ACCESS_TOKEN)" + - "--url=$(DIGITALOCEAN_API_URL)" + - "--default-volumes-page-size=200" + env: + - name: CSI_ENDPOINT + value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + - name: DIGITALOCEAN_API_URL + value: https://api.digitalocean.com/ + - name: DIGITALOCEAN_ACCESS_TOKEN + valueFrom: + secretKeyRef: + name: digitalocean + key: access-token + imagePullPolicy: "Always" + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + volumes: + - name: socket-dir + emptyDir: {} + +--- + +kind: ServiceAccount +apiVersion: v1 +metadata: + name: csi-do-controller-sa + namespace: kube-system + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-do-provisioner-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["get", "list"] + - apiGroups: [ "storage.k8s.io" ] + resources: [ "csinodes" ] + verbs: [ "get", "list", "watch" ] + - apiGroups: [ "" ] + resources: [ "nodes" ] + verbs: [ "get", "list", "watch" ] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch"] + +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-do-provisioner-binding +subjects: + - kind: ServiceAccount + name: csi-do-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-do-provisioner-role + apiGroup: rbac.authorization.k8s.io + +--- +# Attacher must be able to work with PVs, nodes and VolumeAttachments +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-do-attacher-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["patch"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-do-attacher-binding +subjects: + - kind: ServiceAccount + name: csi-do-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-do-attacher-role + apiGroup: rbac.authorization.k8s.io + +--- + +# Snapshotter sidecar + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-do-snapshotter-role +rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update", "patch"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-do-snapshotter-binding +subjects: + - kind: ServiceAccount + name: csi-do-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-do-snapshotter-role + apiGroup: rbac.authorization.k8s.io + +--- + +# Resizer must be able to work with PVCs, PVs, SCs. +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-do-resizer-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-do-resizer-binding +subjects: + - kind: ServiceAccount + name: csi-do-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-do-resizer-role + apiGroup: rbac.authorization.k8s.io + +--- + +######################################## +########### ############ +########### Node plugin ############ +########### ############ +######################################## + +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: csi-do-node + namespace: kube-system +spec: + selector: + matchLabels: + app: csi-do-node + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: csi-do-plugin + labels: + app: csi-do-node + role: csi-do + spec: + priorityClassName: system-node-critical + serviceAccount: csi-do-node-sa + hostNetwork: true + initContainers: + # Delete automount udev rule running on all DO droplets. The rule mounts + # devices briefly and may conflict with CSI-managed droplets (leading to + # "resource busy" errors). We can safely delete it in DOKS. + - name: automount-udev-deleter + image: alpine:3 + args: + - "rm" + - "-f" + - "/etc/udev/rules.d/99-digitalocean-automount.rules" + volumeMounts: + - name: udev-rules-dir + mountPath: /etc/udev/rules.d/ + containers: + - name: csi-node-driver-registrar + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.8.0 + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)" + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/dobs.csi.digitalocean.com/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: plugin-dir + mountPath: /csi/ + - name: registration-dir + mountPath: /registration/ + - name: csi-do-plugin + image: digitalocean/do-csi-plugin:v4.7.0 + args : + - "--endpoint=$(CSI_ENDPOINT)" + - "--url=$(DIGITALOCEAN_API_URL)" + env: + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: DIGITALOCEAN_API_URL + value: https://api.digitalocean.com/ + imagePullPolicy: "Always" + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: pods-mount-dir + mountPath: /var/lib/kubelet + # needed so that any mounts setup inside this container are + # propagated back to the host machine. + mountPropagation: "Bidirectional" + - name: device-dir + mountPath: /dev + - name: tmpfs + mountPath: /tmp + volumes: + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: DirectoryOrCreate + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/dobs.csi.digitalocean.com + type: DirectoryOrCreate + - name: pods-mount-dir + hostPath: + path: /var/lib/kubelet + type: Directory + - name: device-dir + hostPath: + path: /dev + - name: udev-rules-dir + hostPath: + path: /etc/udev/rules.d/ + # to make sure temporary stored luks keys never touch a disk + - name: tmpfs + emptyDir: + medium: Memory +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-do-node-sa + namespace: kube-system + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-do-node-driver-registrar-role + namespace: kube-system +rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-do-node-driver-registrar-binding +subjects: + - kind: ServiceAccount + name: csi-do-node-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-do-node-driver-registrar-role + apiGroup: rbac.authorization.k8s.io diff --git a/deploy/kubernetes/releases/csi-digitalocean-v4.7.0/snapshot-controller.yaml b/deploy/kubernetes/releases/csi-digitalocean-v4.7.0/snapshot-controller.yaml new file mode 100644 index 000000000..d61c26e7e --- /dev/null +++ b/deploy/kubernetes/releases/csi-digitalocean-v4.7.0/snapshot-controller.yaml @@ -0,0 +1,98 @@ +# Copyright 2022 DigitalOcean +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# +# snapshotter controller +# source: # Source: https://github.com/kubernetes-csi/external-snapshotter +# + +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: snapshot-controller + namespace: kube-system +spec: + serviceName: "snapshot-controller" + replicas: 1 + selector: + matchLabels: + app: snapshot-controller + template: + metadata: + labels: + app: snapshot-controller + spec: + serviceAccountName: snapshot-controller + containers: + - name: snapshot-controller + image: registry.k8s.io/sig-storage/snapshot-controller:v6.2.1 + args: + - "--v=5" + imagePullPolicy: IfNotPresent + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: snapshot-controller + namespace: kube-system + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: snapshot-controller-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status"] + verbs: ["update", "patch"] + +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: snapshot-controller-binding +subjects: + - kind: ServiceAccount + name: snapshot-controller + namespace: kube-system +roleRef: + kind: ClusterRole + name: snapshot-controller-role + apiGroup: rbac.authorization.k8s.io diff --git a/deploy/kubernetes/releases/csi-digitalocean-v4.7.0/snapshot-validation-webhook.yaml b/deploy/kubernetes/releases/csi-digitalocean-v4.7.0/snapshot-validation-webhook.yaml new file mode 100644 index 000000000..af864470e --- /dev/null +++ b/deploy/kubernetes/releases/csi-digitalocean-v4.7.0/snapshot-validation-webhook.yaml @@ -0,0 +1,128 @@ +# Copyright 2022 DigitalOcean +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: "validation-webhook.snapshot.storage.k8s.io" +webhooks: + - name: "validation-webhook.snapshot.storage.k8s.io" + rules: + - apiGroups: ["snapshot.storage.k8s.io"] + apiVersions: ["v1", "v1beta1"] + operations: ["CREATE", "UPDATE"] + resources: ["volumesnapshots", "volumesnapshotcontents"] + scope: "*" + clientConfig: + service: + namespace: "kube-system" + name: "snapshot-validation-service" + path: "/volumesnapshot" + # XXX Uncomment and populate the CA bundle field accordingly if a dedicated + # CA is to be used. + # caBundle: ${CA_BUNDLE} + admissionReviewVersions: ["v1", "v1beta1"] + sideEffects: None + failurePolicy: Fail + timeoutSeconds: 5 + +--- + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: snapshot-validation + namespace: kube-system + labels: + app: snapshot-validation +spec: + replicas: 1 + selector: + matchLabels: + app: snapshot-validation + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + type: RollingUpdate + template: + metadata: + labels: + app: snapshot-validation + spec: + serviceAccountName: snapshot-validation + containers: + - name: snapshot-validation + image: registry.k8s.io/sig-storage/snapshot-validation-webhook:v6.2.1 + imagePullPolicy: IfNotPresent + args: ['--tls-cert-file=/etc/snapshot-validation-webhook/certs/cert.pem', '--tls-private-key-file=/etc/snapshot-validation-webhook/certs/key.pem'] + ports: + - containerPort: 443 + volumeMounts: + - name: snapshot-validation-webhook-certs + mountPath: /etc/snapshot-validation-webhook/certs + readOnly: true + volumes: + - name: snapshot-validation-webhook-certs + secret: + # XXX Populate the secret properly with a certificate and key + secretName: snapshot-validation-secret + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: snapshot-validation + namespace: kube-system + +--- + +apiVersion: v1 +kind: Service +metadata: + name: snapshot-validation-service + namespace: kube-system +spec: + selector: + app: snapshot-validation + ports: + - protocol: TCP + port: 443 + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: snapshot-validation +rules: + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: snapshot-validation +subjects: + - kind: ServiceAccount + name: snapshot-validation + namespace: kube-system +roleRef: + kind: ClusterRole + name: snapshot-validation + apiGroup: rbac.authorization.k8s.io diff --git a/driver/controller.go b/driver/controller.go index 35c93fd22..a011dd337 100644 --- a/driver/controller.go +++ b/driver/controller.go @@ -44,6 +44,12 @@ const ( tiB ) +const ( + // PublishInfoVolumeName is used to pass the volume name from + // `ControllerPublishVolume` to `NodeStageVolume or `NodePublishVolume` + PublishInfoVolumeName = DefaultDriverName + "/volume-name" +) + const ( // minimumVolumeSizeInBytes is used to validate that the user is not trying // to create a volume that is smaller than what we support @@ -115,12 +121,17 @@ func (d *Driver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) } volumeName := req.Name + luksEncrypted := "false" + if req.Parameters[LuksEncryptedAttribute] == "true" { + luksEncrypted = "true" + } log := d.log.WithFields(logrus.Fields{ "volume_name": volumeName, "storage_size_giga_bytes": size / giB, "method": "create_volume", "volume_capabilities": req.VolumeCapabilities, + "luks_encrypted": luksEncrypted, }) log.Info("create volume called") @@ -133,6 +144,26 @@ func (d *Driver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) return nil, status.Error(codes.Internal, err.Error()) } + csiVolume := csi.Volume{ + AccessibleTopology: []*csi.Topology{ + { + Segments: map[string]string{ + "region": d.region, + }, + }, + }, + CapacityBytes: size, + VolumeContext: map[string]string{ + LuksEncryptedAttribute: luksEncrypted, + PublishInfoVolumeName: volumeName, + }, + } + + if luksEncrypted == "true" { + csiVolume.VolumeContext[LuksCipherAttribute] = req.Parameters[LuksCipherAttribute] + csiVolume.VolumeContext[LuksKeySizeAttribute] = req.Parameters[LuksKeySizeAttribute] + } + // volume already exist, do nothing if len(volumes) != 0 { if len(volumes) > 1 { @@ -145,12 +176,9 @@ func (d *Driver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) } log.Info("volume already created") - return &csi.CreateVolumeResponse{ - Volume: &csi.Volume{ - VolumeId: vol.ID, - CapacityBytes: vol.SizeGigaBytes * giB, - }, - }, nil + csiVolume.VolumeId = vol.ID + csiVolume.CapacityBytes = vol.SizeGigaBytes * giB + return &csi.CreateVolumeResponse{Volume: &csiVolume}, nil } volumeReq := &godo.VolumeCreateRequest{ @@ -220,19 +248,8 @@ func (d *Driver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) log.Info("resize completed") } - resp := &csi.CreateVolumeResponse{ - Volume: &csi.Volume{ - VolumeId: vol.ID, - CapacityBytes: size, - AccessibleTopology: []*csi.Topology{ - { - Segments: map[string]string{ - "region": d.region, - }, - }, - }, - }, - } + csiVolume.VolumeId = vol.ID + resp := &csi.CreateVolumeResponse{Volume: &csiVolume} // external-provisioner expects a content source to be returned if the PVC // specified a data source, which corresponds to us having received a @@ -349,6 +366,9 @@ func (d *Driver) ControllerPublishVolume(ctx context.Context, req *csi.Controlle return &csi.ControllerPublishVolumeResponse{ PublishContext: map[string]string{ d.publishInfoVolumeName: vol.Name, + LuksEncryptedAttribute: req.VolumeContext[LuksEncryptedAttribute], + LuksCipherAttribute: req.VolumeContext[LuksCipherAttribute], + LuksKeySizeAttribute: req.VolumeContext[LuksKeySizeAttribute], }, }, nil } @@ -374,6 +394,9 @@ func (d *Driver) ControllerPublishVolume(ctx context.Context, req *csi.Controlle return &csi.ControllerPublishVolumeResponse{ PublishContext: map[string]string{ d.publishInfoVolumeName: vol.Name, + LuksEncryptedAttribute: req.VolumeContext[LuksEncryptedAttribute], + LuksCipherAttribute: req.VolumeContext[LuksCipherAttribute], + LuksKeySizeAttribute: req.VolumeContext[LuksKeySizeAttribute], }, }, nil } @@ -406,6 +429,9 @@ func (d *Driver) ControllerPublishVolume(ctx context.Context, req *csi.Controlle return &csi.ControllerPublishVolumeResponse{ PublishContext: map[string]string{ d.publishInfoVolumeName: vol.Name, + LuksEncryptedAttribute: req.VolumeContext[LuksEncryptedAttribute], + LuksCipherAttribute: req.VolumeContext[LuksCipherAttribute], + LuksKeySizeAttribute: req.VolumeContext[LuksKeySizeAttribute], }, }, nil } @@ -838,6 +864,7 @@ func (d *Driver) ListSnapshots(ctx context.Context, req *csi.ListSnapshotsReques untypedSnapshots, nextToken, err := listResources(ctx, log, startingToken, req.MaxEntries, func(ctx context.Context, listOpts *godo.ListOptions) ([]interface{}, *godo.Response, error) { snapshots, resp, err := d.snapshots.ListVolume(ctx, listOpts) + if err != nil { return nil, resp, err } diff --git a/driver/driver_test.go b/driver/driver_test.go index a7666eacd..2e9b169e8 100644 --- a/driver/driver_test.go +++ b/driver/driver_test.go @@ -523,16 +523,16 @@ type fakeMounter struct { mounted map[string]string } -func (f *fakeMounter) Format(source string, fsType string) error { +func (f *fakeMounter) Format(source string, fsType string, context LuksContext) error { return nil } -func (f *fakeMounter) Mount(source string, target string, fsType string, options ...string) error { +func (f *fakeMounter) Mount(source string, target string, fsType string, context LuksContext, options ...string) error { f.mounted[target] = source return nil } -func (f *fakeMounter) Unmount(target string) error { +func (f *fakeMounter) Unmount(target string, context LuksContext) error { delete(f.mounted, target) return nil } @@ -549,9 +549,10 @@ func (f *fakeMounter) IsAttached(source string) error { return nil } -func (f *fakeMounter) IsFormatted(source string) (bool, error) { +func (f *fakeMounter) IsFormatted(source string, context LuksContext) (bool, error) { return true, nil } + func (f *fakeMounter) IsMounted(target string) (bool, error) { _, ok := f.mounted[target] return ok, nil diff --git a/driver/luks_util.go b/driver/luks_util.go new file mode 100644 index 000000000..c0dd33ecc --- /dev/null +++ b/driver/luks_util.go @@ -0,0 +1,402 @@ +/* +Copyright 2019 linkyard ag +Copyright cloudscale.ch + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package driver + +import ( + "errors" + "fmt" + "github.com/sirupsen/logrus" + "io/ioutil" + "os" + "os/exec" + "strings" +) + +const ( + // LuksEncryptedAttribute is used to pass the information if the volume should be + // encrypted with luks to `NodeStageVolume` + LuksEncryptedAttribute = DefaultDriverName + "/luks-encrypted" + + // LuksCipherAttribute is used to pass the information about the luks encryption + // cypher to `NodeStageVolume` + LuksCipherAttribute = DefaultDriverName + "/luks-cipher" + + // LuksKeySizeAttribute is used to pass the information about the luks key size + // to `NodeStageVolume` + LuksKeySizeAttribute = DefaultDriverName + "/luks-key-size" + + // LuksKeyAttribute is the key of the luks key used in the map of secrets passed from the CO + LuksKeyAttribute = "luksKey" +) + +type VolumeLifecycle string + +const ( + VolumeLifecycleNodeStageVolume VolumeLifecycle = "NodeStageVolume" + VolumeLifecycleNodePublishVolume VolumeLifecycle = "NodePublishVolume" + VolumeLifecycleNodeUnstageVolume VolumeLifecycle = "NodeUnstageVolume" + VolumeLifecycleNodeUnpublishVolume VolumeLifecycle = "NodeUnpublishVolume" +) + +type LuksContext struct { + EncryptionEnabled bool + EncryptionKey string + EncryptionCipher string + EncryptionKeySize string + VolumeName string + VolumeLifecycle VolumeLifecycle +} + +func (ctx *LuksContext) validate() error { + if !ctx.EncryptionEnabled { + return nil + } + + var appendFn = func(x string, xs string) string { + if xs != "" { + xs += "; " + } + xs += x + return xs + } + + errorMsg := "" + if ctx.VolumeName == "" { + errorMsg = appendFn("no volume name provided", errorMsg) + } + if ctx.EncryptionKey == "" { + errorMsg = appendFn("no encryption key provided", errorMsg) + } + if ctx.EncryptionCipher == "" { + errorMsg = appendFn("no encryption cipher provided", errorMsg) + } + if ctx.EncryptionKeySize == "" { + errorMsg = appendFn("no encryption key size provided", errorMsg) + } + if errorMsg == "" { + return nil + } + return errors.New(errorMsg) +} + +func getLuksContext(secrets map[string]string, context map[string]string, lifecycle VolumeLifecycle) LuksContext { + if context[LuksEncryptedAttribute] != "true" { + return LuksContext{ + EncryptionEnabled: false, + VolumeLifecycle: lifecycle, + } + } + + luksKey := secrets[LuksKeyAttribute] + luksCipher := context[LuksCipherAttribute] + luksKeySize := context[LuksKeySizeAttribute] + volumeName := context[PublishInfoVolumeName] + + return LuksContext{ + EncryptionEnabled: true, + EncryptionKey: luksKey, + EncryptionCipher: luksCipher, + EncryptionKeySize: luksKeySize, + VolumeName: volumeName, + VolumeLifecycle: lifecycle, + } +} + +func luksFormat(source string, mkfsCmd string, mkfsArgs []string, ctx LuksContext, log *logrus.Entry) error { + cryptsetupCmd, err := getCryptsetupCmd() + if err != nil { + return err + } + filename, err := writeLuksKey(ctx.EncryptionKey, log) + if err != nil { + return err + } + + defer func() { + e := os.Remove(filename) + if e != nil { + log.Errorf("cannot delete temporary file %s: %s", filename, e.Error()) + } + }() + + // initialize the luks partition + cryptsetupArgs := []string{ + "-v", + "--type=luks1", + "--batch-mode", + "--cipher", ctx.EncryptionCipher, + "--key-size", ctx.EncryptionKeySize, + "--key-file", filename, + "luksFormat", source, + } + + log.WithFields(logrus.Fields{ + "cmd": cryptsetupCmd, + "args": cryptsetupArgs, + }).Info("executing cryptsetup luksFormat command") + + out, err := exec.Command(cryptsetupCmd, cryptsetupArgs...).CombinedOutput() + if err != nil { + return fmt.Errorf("cryptsetup luksFormat failed: %v cmd: '%s %s' output: %q", + err, cryptsetupCmd, strings.Join(cryptsetupArgs, " "), string(out)) + } + + // format the disk with the desired filesystem + + // open the luks partition and set up a mapping + err = luksOpen(source, filename, ctx, log) + if err != nil { + return fmt.Errorf("cryptsetup luksOpen failed: %v cmd: '%s %s' output: %q", + err, cryptsetupCmd, strings.Join(cryptsetupArgs, " "), string(out)) + } + + defer func() { + e := luksClose(ctx.VolumeName, log) + if e != nil { + log.Errorf("cannot close luks device: %s", e.Error()) + } + }() + + // replace the source volume with the mapped one in the arguments to mkfs + mkfsNewArgs := make([]string, len(mkfsArgs)) + for i, elem := range mkfsArgs { + if elem != source { + mkfsNewArgs[i] = elem + } else { + mkfsArgs[i] = "/dev/mapper/" + ctx.VolumeName + } + } + + log.WithFields(logrus.Fields{ + "cmd": mkfsCmd, + "args": mkfsArgs, + }).Info("executing format command") + + out, err = exec.Command(mkfsCmd, mkfsArgs...).CombinedOutput() + if err != nil { + return fmt.Errorf("formatting disk failed: %v cmd: '%s %s' output: %q", + err, mkfsCmd, strings.Join(mkfsArgs, " "), string(out)) + } + + return nil +} + +// prepares a luks-encrypted volume for mounting and returns the path of the mapped volume +func luksPrepareMount(source string, ctx LuksContext, log *logrus.Entry) (string, error) { + filename, err := writeLuksKey(ctx.EncryptionKey, log) + if err != nil { + return "", err + } + defer func() { + e := os.Remove(filename) + if e != nil { + log.Errorf("cannot delete temporary file %s: %s", filename, e.Error()) + } + }() + + err = luksOpen(source, filename, ctx, log) + if err != nil { + return "", err + } + return "/dev/mapper/" + ctx.VolumeName, nil +} + +func luksClose(volume string, log *logrus.Entry) error { + cryptsetupCmd, err := getCryptsetupCmd() + if err != nil { + return err + } + cryptsetupArgs := []string{"--batch-mode", "close", volume} + + log.WithFields(logrus.Fields{ + "cmd": cryptsetupCmd, + "args": cryptsetupArgs, + }).Info("executing cryptsetup close command") + + out, err := exec.Command(cryptsetupCmd, cryptsetupArgs...).CombinedOutput() + if err != nil { + return fmt.Errorf("removing luks mapping failed: %v cmd: '%s %s' output: %q", + err, cryptsetupCmd, strings.Join(cryptsetupArgs, " "), string(out)) + } + return nil +} + +// checks if the given volume is formatted by checking if it is a luks volume and +// if the luks volume, once opened, contains a filesystem +func isLuksVolumeFormatted(volume string, ctx LuksContext, log *logrus.Entry) (bool, error) { + isLuks, err := isLuks(volume) + if err != nil { + return false, err + } + if !isLuks { + return false, nil + } + + filename, err := writeLuksKey(ctx.EncryptionKey, log) + if err != nil { + return false, err + } + defer func() { + e := os.Remove(filename) + if e != nil { + log.Errorf("cannot delete temporary file %s: %s", filename, e.Error()) + } + }() + + err = luksOpen(volume, filename, ctx, log) + if err != nil { + return false, err + } + defer func() { + e := luksClose(ctx.VolumeName, log) + if e != nil { + log.Errorf("cannot close luks device: %s", e.Error()) + } + }() + + return isVolumeFormatted(volume, log) +} + +func luksOpen(volume string, keyFile string, ctx LuksContext, log *logrus.Entry) error { + // check if the luks volume is already open + if _, err := os.Stat("/dev/mapper/" + ctx.VolumeName); !os.IsNotExist(err) { + log.WithFields(logrus.Fields{ + "volume": volume, + }).Info("luks volume is already open") + return nil + } + + cryptsetupCmd, err := getCryptsetupCmd() + if err != nil { + return err + } + cryptsetupArgs := []string{ + "--batch-mode", + "luksOpen", + "--key-file", keyFile, + volume, ctx.VolumeName, + } + log.WithFields(logrus.Fields{ + "cmd": cryptsetupCmd, + "args": cryptsetupArgs, + }).Info("executing cryptsetup luksOpen command") + out, err := exec.Command(cryptsetupCmd, cryptsetupArgs...).CombinedOutput() + if err != nil { + return fmt.Errorf("cryptsetup luksOpen failed: %v cmd: '%s %s' output: %q", + err, cryptsetupCmd, strings.Join(cryptsetupArgs, " "), string(out)) + } + return nil +} + +// runs cryptsetup resize for a given volume (/dev/mapper/pvc-xyz) +func luksResize(volume string) error { + cryptsetupCmd, err := getCryptsetupCmd() + if err != nil { + return err + } + cryptsetupArgs := []string{"--batch-mode", "resize", volume} + + _, err = exec.Command(cryptsetupCmd, cryptsetupArgs...).CombinedOutput() + return err +} + +// runs cryptsetup isLuks for a given volume +func isLuks(volume string) (bool, error) { + cryptsetupCmd, err := getCryptsetupCmd() + if err != nil { + return false, err + } + cryptsetupArgs := []string{"--batch-mode", "isLuks", volume} + + // cryptsetup isLuks exits with code 0 if the target is a luks volume; otherwise it returns + // a non-zero exit code which exec.Command interprets as an error + _, err = exec.Command(cryptsetupCmd, cryptsetupArgs...).CombinedOutput() + if err != nil { + return false, nil + } + return true, nil +} + +// check is a given mapping under /dev/mapper is a luks volume +func isLuksMapping(volume string) (bool, string, error) { + if strings.HasPrefix(volume, "/dev/mapper/") { + mappingName := volume[len("/dev/mapper/"):] + cryptsetupCmd, err := getCryptsetupCmd() + if err != nil { + return false, mappingName, err + } + cryptsetupArgs := []string{"status", mappingName} + + out, err := exec.Command(cryptsetupCmd, cryptsetupArgs...).CombinedOutput() + if err != nil { + return false, mappingName, nil + } + for _, statusLine := range strings.Split(string(out), "\n") { + if strings.Contains(statusLine, "type:") { + if strings.Contains(strings.ToLower(statusLine), "luks") { + return true, mappingName, nil + } + return false, mappingName, nil + } + } + + } + return false, "", nil +} + +func getCryptsetupCmd() (string, error) { + cryptsetupCmd := "cryptsetup" + _, err := exec.LookPath(cryptsetupCmd) + if err != nil { + if err == exec.ErrNotFound { + return "", fmt.Errorf("%q executable not found in $PATH", cryptsetupCmd) + } + return "", err + } + return cryptsetupCmd, nil +} + +// writes the given luks encryption key to a temporary file and returns the name of the temporary +// file +func writeLuksKey(key string, log *logrus.Entry) (string, error) { + if !checkTmpFs("/tmp") { + return "", errors.New("temporary directory /tmp is not a tmpfs volume; refusing to write luks key to a volume backed by a disk") + } + tmpFile, err := ioutil.TempFile("/tmp", "luks-") + if err != nil { + return "", err + } + _, err = tmpFile.WriteString(key) + if err != nil { + log.WithField("tmp_file", tmpFile.Name()).Warnf("Unable to write luks key file: %s", err.Error()) + return "", err + } + return tmpFile.Name(), nil +} + +// makes sure that the given directory is a tmpfs +func checkTmpFs(dir string) bool { + out, err := exec.Command("sh", "-c", "df -T "+dir+" | tail -n1 | awk '{print $2}'").CombinedOutput() + if err != nil { + return false + } + if len(out) == 0 { + return false + } + return strings.TrimSpace(string(out)) == "tmpfs" +} diff --git a/driver/luks_util_test.go b/driver/luks_util_test.go new file mode 100644 index 000000000..c43cbeb12 --- /dev/null +++ b/driver/luks_util_test.go @@ -0,0 +1,110 @@ +package driver + +import ( + "context" + "errors" + "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/sirupsen/logrus" + "strings" + "testing" +) + +type TestPodVolume struct { + ClaimName string + SizeGB int + StorageClass string + LuksKey string +} + +type TestPodDescriptor struct { + Kind string + Name string + Volumes []TestPodVolume +} + +type DiskInfo struct { + PVCName string `json:"pvcName"` + DeviceName string `json:"deviceName"` + DeviceSize int `json:"deviceSize"` + Filesystem string `json:"filesystem"` + FilesystemUUID string `json:"filesystemUUID"` + FilesystemSize int `json:"filesystemSize"` + DeviceSource string `json:"deviceSource"` + Luks string `json:"luks,omitempty"` + Cipher string `json:"cipher,omitempty"` + Keysize int `json:"keysize,omitempty"` +} + +// creates a kubernetes pod from the given TestPodDescriptor +func TestCreateLuksVolume(t *testing.T) { + tests := []struct { + name string + listVolumesErr error + getSnapshotErr error + }{ + { + name: "listing volumes failing", + listVolumesErr: errors.New("failed to list volumes"), + }, + { + name: "fetching snapshot failing", + getSnapshotErr: errors.New("failed to get snapshot"), + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + d := &Driver{ + storage: &fakeStorageDriver{ + listVolumesErr: test.listVolumesErr, + }, + snapshots: &fakeSnapshotsDriver{ + getSnapshotErr: test.getSnapshotErr, + }, + log: logrus.New().WithField("test_enabed", true), + } + + _, err := d.CreateVolume(context.Background(), &csi.CreateVolumeRequest{ + Name: "name", + Parameters: map[string]string{ + LuksCipherAttribute: "cipher", + LuksEncryptedAttribute: "true", + LuksKeyAttribute: "key", + LuksKeySizeAttribute: "23234", + }, + VolumeCapabilities: []*csi.VolumeCapability{ + &csi.VolumeCapability{ + AccessType: &csi.VolumeCapability_Mount{ + Mount: &csi.VolumeCapability_MountVolume{}, + }, + AccessMode: &csi.VolumeCapability_AccessMode{ + Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, + }, + }, + }, + VolumeContentSource: &csi.VolumeContentSource{ + Type: &csi.VolumeContentSource_Snapshot{ + Snapshot: &csi.VolumeContentSource_SnapshotSource{ + SnapshotId: "snapshotId", + }, + }, + }, + }) + + var wantErr error + switch { + case test.listVolumesErr != nil: + wantErr = test.listVolumesErr + case test.getSnapshotErr != nil: + wantErr = test.getSnapshotErr + } + + if wantErr == nil && err != nil { + t.Errorf("got error %q, want none", err) + } + if wantErr != nil && !strings.Contains(err.Error(), wantErr.Error()) { + t.Errorf("want error %q to include %q", err, wantErr) + } + }) + } +} diff --git a/driver/mounter.go b/driver/mounter.go index a7cecd85d..0bce82527 100644 --- a/driver/mounter.go +++ b/driver/mounter.go @@ -78,20 +78,20 @@ const ( // more than just mounting functionality by now. type Mounter interface { // Format formats the source with the given filesystem type - Format(source, fsType string) error + Format(source, fsType string, luksContext LuksContext) error // Mount mounts source to target with the given fstype and options. - Mount(source, target, fsType string, options ...string) error + Mount(source, target, fsType string, luksContext LuksContext, options ...string) error // Unmount unmounts the given target - Unmount(target string) error + Unmount(target string, luksContext LuksContext) error // IsAttached checks whether the source device is in the running state. IsAttached(source string) error // IsFormatted checks whether the source device is formatted or not. It // returns true if the source device is already formatted. - IsFormatted(source string) (bool, error) + IsFormatted(source string, luksContext LuksContext) (bool, error) // IsMounted checks whether the target path is a correct mount (i.e: // propagated). It returns true if it's mounted. An error is returned in @@ -131,7 +131,7 @@ func newMounter(log *logrus.Entry) *mounter { } } -func (m *mounter) Format(source, fsType string) error { +func (m *mounter) Format(source, fsType string, luksContext LuksContext) error { mkfsCmd := fmt.Sprintf("mkfs.%s", fsType) _, err := exec.LookPath(mkfsCmd) @@ -157,21 +157,34 @@ func (m *mounter) Format(source, fsType string) error { mkfsArgs = []string{"-F", source} } - m.log.WithFields(logrus.Fields{ - "cmd": mkfsCmd, - "args": mkfsArgs, - }).Info("executing format command") + if !luksContext.EncryptionEnabled { + m.log.WithFields(logrus.Fields{ + "cmd": mkfsCmd, + "args": mkfsArgs, + }).Info("executing format command") - out, err := exec.Command(mkfsCmd, mkfsArgs...).CombinedOutput() - if err != nil { - return fmt.Errorf("formatting disk failed: %v cmd: '%s %s' output: %q", - err, mkfsCmd, strings.Join(mkfsArgs, " "), string(out)) - } + out, err := exec.Command(mkfsCmd, mkfsArgs...).CombinedOutput() + if err != nil { + return fmt.Errorf("formatting disk failed: %v cmd: '%s %s' output: %q", + err, mkfsCmd, strings.Join(mkfsArgs, " "), string(out)) + } + return nil + } else { + err := luksContext.validate() + if err != nil { + return err + } - return nil + err = luksFormat(source, mkfsCmd, mkfsArgs, luksContext, m.log) + if err != nil { + return err + } + + return nil + } } -func (m *mounter) Mount(source, target, fsType string, opts ...string) error { +func (m *mounter) Mount(source, target, fsType string, luksContext LuksContext, opts ...string) error { mountCmd := "mount" mountArgs := []string{} @@ -211,7 +224,21 @@ func (m *mounter) Mount(source, target, fsType string, opts ...string) error { mountArgs = append(mountArgs, "-o", strings.Join(opts, ",")) } - mountArgs = append(mountArgs, source) + if luksContext.EncryptionEnabled && luksContext.VolumeLifecycle == VolumeLifecycleNodeStageVolume { + luksSource, err := luksPrepareMount(source, luksContext, m.log) + if err != nil { + m.log.WithFields(logrus.Fields{ + "error": err.Error(), + "volume": luksContext.VolumeName, + }).Error("failed to prepare luks volume for mounting") + return err + } + + mountArgs = append(mountArgs, luksSource) + } else { + mountArgs = append(mountArgs, source) + } + mountArgs = append(mountArgs, target) m.log.WithFields(logrus.Fields{ @@ -228,8 +255,77 @@ func (m *mounter) Mount(source, target, fsType string, opts ...string) error { return nil } -func (m *mounter) Unmount(target string) error { - return mount.CleanupMountPoint(target, m.kMounter, true) +func (m *mounter) Unmount(target string, luksContext LuksContext) error { + if target == "" { + return errors.New("target is not specified for unmounting the volume") + } + + // if this is the unmount call after the mount-bind has been removed, + // a luks volume needs to be closed after unmounting; get the source + // of the mount to check if that is a luks volume + mountSources, err := getMountSources(target) + if err != nil { + return err + } + + if len(mountSources) == 0 { + return fmt.Errorf("unable to determine mount sources of target %s", target) + } + + umountCmd := "umount" + umountArgs := []string{target} + + m.log.WithFields(logrus.Fields{ + "cmd": umountCmd, + "args": umountArgs, + }).Info("executing umount command") + + out, err := exec.Command(umountCmd, umountArgs...).CombinedOutput() + if err != nil { + return fmt.Errorf("unmounting failed: %v cmd: '%s %s' output: %q", + err, umountCmd, target, string(out)) + } + + merror := mount.CleanupMountPoint(target, m.kMounter, true) + + // if this is the unstaging process, check if the source is a luks volume and close it + if luksContext.VolumeLifecycle == VolumeLifecycleNodeUnstageVolume { + for _, source := range mountSources { + isLuksMapping, mappingName, err := isLuksMapping(source) + if err != nil { + return err + } + if isLuksMapping { + err := luksClose(mappingName, m.log) + if err != nil { + return err + } + } + } + } + + return merror +} + +// gets the mount sources of a mountpoint +func getMountSources(target string) ([]string, error) { + _, err := exec.LookPath("findmnt") + if err != nil { + if err == exec.ErrNotFound { + return nil, fmt.Errorf("%q executable not found in $PATH", "findmnt") + } + return nil, err + } + out, err := exec.Command("sh", "-c", fmt.Sprintf("findmnt -o SOURCE -n -M %s", target)).CombinedOutput() + if err != nil { + // findmnt exits with non zero exit status if it couldn't find anything + if strings.TrimSpace(string(out)) == "" { + return nil, nil + } + return nil, fmt.Errorf("checking mounted failed: %v cmd: %q output: %q", + err, "findmnt", string(out)) + } + return strings.Split(string(out), "\n"), nil } func (m *mounter) IsAttached(source string) error { @@ -256,7 +352,20 @@ func (m *mounter) IsAttached(source string) error { return nil } -func (m *mounter) IsFormatted(source string) (bool, error) { +func (m *mounter) IsFormatted(source string, luksContext LuksContext) (bool, error) { + if !luksContext.EncryptionEnabled { + return isVolumeFormatted(source, m.log) + } + + formatted, err := isLuksVolumeFormatted(source, luksContext, m.log) + if err != nil { + return false, err + } + + return formatted, nil +} + +func isVolumeFormatted(source string, log *logrus.Entry) (bool, error) { if source == "" { return false, errors.New("source is not specified") } @@ -272,7 +381,7 @@ func (m *mounter) IsFormatted(source string) (bool, error) { blkidArgs := []string{source} - m.log.WithFields(logrus.Fields{ + log.WithFields(logrus.Fields{ "cmd": blkidCmd, "args": blkidArgs, }).Info("checking if source is formatted") diff --git a/driver/node.go b/driver/node.go index 6e03a744c..48088a492 100644 --- a/driver/node.go +++ b/driver/node.go @@ -88,6 +88,11 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe }) log.Info("node stage volume called") + publishContext := req.GetPublishContext() + if publishContext == nil { + return nil, status.Error(codes.InvalidArgument, "PublishContext must be provided") + } + volumeName := "" if volName, ok := req.GetPublishContext()[d.publishInfoVolumeName]; !ok { return nil, status.Error(codes.InvalidArgument, "Could not find the volume by name") @@ -103,6 +108,9 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe } source := getDeviceByIDPath(volumeName) + + luksContext := getLuksContext(req.Secrets, req.VolumeContext, VolumeLifecycleNodeStageVolume) + target := req.StagingTargetPath mnt := req.VolumeCapability.GetMount() @@ -121,6 +129,7 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe "source": source, "fs_type": fsType, "mount_options": options, + "luks_encrypted": luksContext.EncryptionEnabled, }) var noFormat bool @@ -139,14 +148,14 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe } } - formatted, err := d.mounter.IsFormatted(source) + formatted, err := d.mounter.IsFormatted(source, luksContext) if err != nil { return nil, err } if !formatted { log.Info("formatting the volume for staging") - if err := d.mounter.Format(source, fsType); err != nil { + if err := d.mounter.Format(source, fsType, luksContext); err != nil { return nil, status.Error(codes.Internal, err.Error()) } } else { @@ -162,14 +171,15 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe } if !mounted { - if err := d.mounter.Mount(source, target, fsType, options...); err != nil { + if err := d.mounter.Mount(source, target, fsType, luksContext, options...); err != nil { return nil, status.Error(codes.Internal, err.Error()) } } else { log.Info("source device is already mounted to the target path") } - if _, err := os.Stat(source); err == nil { + + if _, err := os.Stat(source); err == nil && !luksContext.EncryptionEnabled { r := mountutil.NewResizeFs(utilexec.New()) needResize, err := r.NeedResize(source, target) @@ -199,6 +209,8 @@ func (d *Driver) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVolu return nil, status.Error(codes.InvalidArgument, "NodeUnstageVolume Staging Target Path must be provided") } + luksContext := LuksContext{VolumeLifecycle: VolumeLifecycleNodeUnstageVolume} + log := d.log.WithFields(logrus.Fields{ "volume_id": req.VolumeId, "staging_target_path": req.StagingTargetPath, @@ -213,7 +225,7 @@ func (d *Driver) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVolu if mounted { log.Info("unmounting the staging target path") - err := d.mounter.Unmount(req.StagingTargetPath) + err := d.mounter.Unmount(req.StagingTargetPath, luksContext) if err != nil { return nil, err } @@ -243,11 +255,19 @@ func (d *Driver) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolu return nil, status.Error(codes.InvalidArgument, "NodePublishVolume Volume Capability must be provided") } + publishContext := req.GetPublishContext() + if publishContext == nil { + return nil, status.Error(codes.InvalidArgument, "PublishContext must be provided") + } + + luksContext := getLuksContext(req.Secrets, publishContext, VolumeLifecycleNodePublishVolume) + log := d.log.WithFields(logrus.Fields{ "volume_id": req.VolumeId, "staging_target_path": req.StagingTargetPath, "target_path": req.TargetPath, "method": "node_publish_volume", + "luks_encrypted": luksContext.EncryptionEnabled, }) log.Info("node publish volume called") @@ -259,9 +279,9 @@ func (d *Driver) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolu var err error switch req.GetVolumeCapability().GetAccessType().(type) { case *csi.VolumeCapability_Block: - err = d.nodePublishVolumeForBlock(req, options, log) + err = d.nodePublishVolumeForBlock(req, luksContext, options, log) case *csi.VolumeCapability_Mount: - err = d.nodePublishVolumeForFileSystem(req, options, log) + err = d.nodePublishVolumeForFileSystem(req, luksContext, options, log) default: return nil, status.Error(codes.InvalidArgument, "Unknown access type") } @@ -284,6 +304,8 @@ func (d *Driver) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublish return nil, status.Error(codes.InvalidArgument, "NodeUnpublishVolume Target Path must be provided") } + luksContext := LuksContext{VolumeLifecycle: VolumeLifecycleNodeUnpublishVolume} + log := d.log.WithFields(logrus.Fields{ "volume_id": req.VolumeId, "target_path": req.TargetPath, @@ -291,7 +313,7 @@ func (d *Driver) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublish }) log.Info("node unpublish volume called") - err := d.mounter.Unmount(req.TargetPath) + err := d.mounter.Unmount(req.TargetPath, luksContext) if err != nil { return nil, err } @@ -482,6 +504,22 @@ func (d *Driver) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandVolume return nil, status.Errorf(codes.NotFound, "NodeExpandVolume device path for volume path %q not found", volumePath) } + isLuks, _, err := isLuksMapping(devicePath) + if err != nil { + return nil, status.Errorf(codes.Internal, "NodeExpandVolume unable to test if volume %q at %q is encrypted with luks: %v", volumePath, devicePath, err) + } + + // the luks container must be resized if the volume was resized while the disk was mounted + if isLuks { + log.WithFields(logrus.Fields{ + "device_path": devicePath, + }).Info("resizing luks container") + err := luksResize(devicePath) + if err != nil { + return nil, status.Errorf(codes.Internal, "NodeExpandVolume unable resize luks container for volume %q at %q: %v", volumePath, devicePath, err) + } + } + r := mount.NewResizeFs(utilexec.New()) log = log.WithFields(logrus.Fields{ "device_path": devicePath, @@ -492,10 +530,11 @@ func (d *Driver) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandVolume } log.Info("volume was resized") + return &csi.NodeExpandVolumeResponse{}, nil } -func (d *Driver) nodePublishVolumeForFileSystem(req *csi.NodePublishVolumeRequest, mountOptions []string, log *logrus.Entry) error { +func (d *Driver) nodePublishVolumeForFileSystem(req *csi.NodePublishVolumeRequest, luksContext LuksContext, mountOptions []string, log *logrus.Entry) error { source := req.StagingTargetPath target := req.TargetPath @@ -523,7 +562,7 @@ func (d *Driver) nodePublishVolumeForFileSystem(req *csi.NodePublishVolumeReques if !mounted { log.Info("mounting the volume") - if err := d.mounter.Mount(source, target, fsType, mountOptions...); err != nil { + if err := d.mounter.Mount(source, target, fsType, luksContext, mountOptions...); err != nil { return status.Error(codes.Internal, err.Error()) } } else { @@ -533,7 +572,7 @@ func (d *Driver) nodePublishVolumeForFileSystem(req *csi.NodePublishVolumeReques return nil } -func (d *Driver) nodePublishVolumeForBlock(req *csi.NodePublishVolumeRequest, mountOptions []string, log *logrus.Entry) error { +func (d *Driver) nodePublishVolumeForBlock(req *csi.NodePublishVolumeRequest, luksContext LuksContext, mountOptions []string, log *logrus.Entry) error { volumeName, ok := req.GetPublishContext()[d.publishInfoVolumeName] if !ok { return status.Error(codes.InvalidArgument, fmt.Sprintf("Could not find the volume name from the publish context %q", d.publishInfoVolumeName)) @@ -559,7 +598,7 @@ func (d *Driver) nodePublishVolumeForBlock(req *csi.NodePublishVolumeRequest, mo if !mounted { log.Info("mounting the volume") - if err := d.mounter.Mount(source, target, "", mountOptions...); err != nil { + if err := d.mounter.Mount(source, target, "", luksContext, mountOptions...); err != nil { return status.Errorf(codes.Internal, err.Error()) } } else { From b121391c6e4c350a4fb4c919c0224e0ba6a9d09b Mon Sep 17 00:00:00 2001 From: Max Petyurenko Date: Thu, 13 Jul 2023 14:39:46 +0300 Subject: [PATCH 2/5] Revert bump-version --- CHANGELOG.md | 4 - VERSION | 2 +- .../csi-digitalocean-v4.7.0/crds.yaml | 865 ------------------ .../csi-digitalocean-v4.7.0/driver.yaml | 533 ----------- .../snapshot-controller.yaml | 98 -- .../snapshot-validation-webhook.yaml | 128 --- 6 files changed, 1 insertion(+), 1629 deletions(-) delete mode 100644 deploy/kubernetes/releases/csi-digitalocean-v4.7.0/crds.yaml delete mode 100644 deploy/kubernetes/releases/csi-digitalocean-v4.7.0/driver.yaml delete mode 100644 deploy/kubernetes/releases/csi-digitalocean-v4.7.0/snapshot-controller.yaml delete mode 100644 deploy/kubernetes/releases/csi-digitalocean-v4.7.0/snapshot-validation-webhook.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index f16c2b530..946de23a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,5 @@ ## unreleased -## v4.7.0 - 2023.06.23 -* Add LUKS encryption support - [[GH-514]](https://github.com/digitalocean/csi-digitalocean/pull/514) - ## v4.6.1 - 2023.05.17 * Update CSI sidecars to latest [[GH-502]](https://github.com/digitalocean/csi-digitalocean/pull/502) diff --git a/VERSION b/VERSION index 79214dce1..c0c6ad02b 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v4.7.0 +v4.6.1 diff --git a/deploy/kubernetes/releases/csi-digitalocean-v4.7.0/crds.yaml b/deploy/kubernetes/releases/csi-digitalocean-v4.7.0/crds.yaml deleted file mode 100644 index f79f0a4df..000000000 --- a/deploy/kubernetes/releases/csi-digitalocean-v4.7.0/crds.yaml +++ /dev/null @@ -1,865 +0,0 @@ -# Copyright 2022 DigitalOcean -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -############################################## -########### ############ -########### Snapshot CRDs ############ -########### ############ -############################################## -# -# Source: https://github.com/kubernetes-csi/external-snapshotter -# - -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.8.0 - api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/665" - creationTimestamp: null - name: volumesnapshotclasses.snapshot.storage.k8s.io -spec: - group: snapshot.storage.k8s.io - names: - kind: VolumeSnapshotClass - listKind: VolumeSnapshotClassList - plural: volumesnapshotclasses - shortNames: - - vsclass - - vsclasses - singular: volumesnapshotclass - scope: Cluster - versions: - - additionalPrinterColumns: - - jsonPath: .driver - name: Driver - type: string - - description: Determines whether a VolumeSnapshotContent created through the - VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. - jsonPath: .deletionPolicy - name: DeletionPolicy - type: string - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1 - schema: - openAPIV3Schema: - description: VolumeSnapshotClass specifies parameters that a underlying storage - system uses when creating a volume snapshot. A specific VolumeSnapshotClass - is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses - are non-namespaced - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - deletionPolicy: - description: deletionPolicy determines whether a VolumeSnapshotContent - created through the VolumeSnapshotClass should be deleted when its bound - VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". - "Retain" means that the VolumeSnapshotContent and its physical snapshot - on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent - and its physical snapshot on underlying storage system are deleted. - Required. - enum: - - Delete - - Retain - type: string - driver: - description: driver is the name of the storage driver that handles this - VolumeSnapshotClass. Required. - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - parameters: - additionalProperties: - type: string - description: parameters is a key-value map with storage driver specific - parameters for creating snapshots. These values are opaque to Kubernetes. - type: object - required: - - deletionPolicy - - driver - type: object - served: true - storage: true - subresources: {} - - additionalPrinterColumns: - - jsonPath: .driver - name: Driver - type: string - - description: Determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. - jsonPath: .deletionPolicy - name: DeletionPolicy - type: string - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1beta1 - # This indicates the v1beta1 version of the custom resource is deprecated. - # API requests to this version receive a warning in the server response. - deprecated: true - # This overrides the default warning returned to clients making v1beta1 API requests. - deprecationWarning: "snapshot.storage.k8s.io/v1beta1 VolumeSnapshotClass is deprecated; use snapshot.storage.k8s.io/v1 VolumeSnapshotClass" - schema: - openAPIV3Schema: - description: VolumeSnapshotClass specifies parameters that a underlying storage system uses when creating a volume snapshot. A specific VolumeSnapshotClass is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses are non-namespaced - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - deletionPolicy: - description: deletionPolicy determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. Required. - enum: - - Delete - - Retain - type: string - driver: - description: driver is the name of the storage driver that handles this VolumeSnapshotClass. Required. - type: string - kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - parameters: - additionalProperties: - type: string - description: parameters is a key-value map with storage driver specific parameters for creating snapshots. These values are opaque to Kubernetes. - type: object - required: - - deletionPolicy - - driver - type: object - served: false - storage: false - subresources: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- - -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.8.0 - api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/665" - creationTimestamp: null - name: volumesnapshotcontents.snapshot.storage.k8s.io -spec: - group: snapshot.storage.k8s.io - names: - kind: VolumeSnapshotContent - listKind: VolumeSnapshotContentList - plural: volumesnapshotcontents - shortNames: - - vsc - - vscs - singular: volumesnapshotcontent - scope: Cluster - versions: - - additionalPrinterColumns: - - description: Indicates if the snapshot is ready to be used to restore a volume. - jsonPath: .status.readyToUse - name: ReadyToUse - type: boolean - - description: Represents the complete size of the snapshot in bytes - jsonPath: .status.restoreSize - name: RestoreSize - type: integer - - description: Determines whether this VolumeSnapshotContent and its physical - snapshot on the underlying storage system should be deleted when its bound - VolumeSnapshot is deleted. - jsonPath: .spec.deletionPolicy - name: DeletionPolicy - type: string - - description: Name of the CSI driver used to create the physical snapshot on - the underlying storage system. - jsonPath: .spec.driver - name: Driver - type: string - - description: Name of the VolumeSnapshotClass to which this snapshot belongs. - jsonPath: .spec.volumeSnapshotClassName - name: VolumeSnapshotClass - type: string - - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent - object is bound. - jsonPath: .spec.volumeSnapshotRef.name - name: VolumeSnapshot - type: string - - description: Namespace of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. - jsonPath: .spec.volumeSnapshotRef.namespace - name: VolumeSnapshotNamespace - type: string - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1 - schema: - openAPIV3Schema: - description: VolumeSnapshotContent represents the actual "on-disk" snapshot - object in the underlying storage system - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - spec: - description: spec defines properties of a VolumeSnapshotContent created - by the underlying storage system. Required. - properties: - deletionPolicy: - description: deletionPolicy determines whether this VolumeSnapshotContent - and its physical snapshot on the underlying storage system should - be deleted when its bound VolumeSnapshot is deleted. Supported values - are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent - and its physical snapshot on underlying storage system are kept. - "Delete" means that the VolumeSnapshotContent and its physical snapshot - on underlying storage system are deleted. For dynamically provisioned - snapshots, this field will automatically be filled in by the CSI - snapshotter sidecar with the "DeletionPolicy" field defined in the - corresponding VolumeSnapshotClass. For pre-existing snapshots, users - MUST specify this field when creating the VolumeSnapshotContent - object. Required. - enum: - - Delete - - Retain - type: string - driver: - description: driver is the name of the CSI driver used to create the - physical snapshot on the underlying storage system. This MUST be - the same as the name returned by the CSI GetPluginName() call for - that driver. Required. - type: string - source: - description: source specifies whether the snapshot is (or should be) - dynamically provisioned or already exists, and just requires a Kubernetes - object representation. This field is immutable after creation. Required. - properties: - snapshotHandle: - description: snapshotHandle specifies the CSI "snapshot_id" of - a pre-existing snapshot on the underlying storage system for - which a Kubernetes object representation was (or should be) - created. This field is immutable. - type: string - volumeHandle: - description: volumeHandle specifies the CSI "volume_id" of the - volume from which a snapshot should be dynamically taken from. - This field is immutable. - type: string - type: object - oneOf: - - required: ["snapshotHandle"] - - required: ["volumeHandle"] - sourceVolumeMode: - description: SourceVolumeMode is the mode of the volume whose snapshot - is taken. Can be either “Filesystem” or “Block”. If not specified, - it indicates the source volume's mode is unknown. This field is - immutable. This field is an alpha field. - type: string - volumeSnapshotClassName: - description: name of the VolumeSnapshotClass from which this snapshot - was (or will be) created. Note that after provisioning, the VolumeSnapshotClass - may be deleted or recreated with different set of values, and as - such, should not be referenced post-snapshot creation. - type: string - volumeSnapshotRef: - description: volumeSnapshotRef specifies the VolumeSnapshot object - to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName - field must reference to this VolumeSnapshotContent's name for the - bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent - object, name and namespace of the VolumeSnapshot object MUST be - provided for binding to happen. This field is immutable after creation. - Required. - properties: - apiVersion: - description: API version of the referent. - type: string - fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' - type: string - kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' - type: string - namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' - type: string - resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' - type: string - uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' - type: string - type: object - required: - - deletionPolicy - - driver - - source - - volumeSnapshotRef - type: object - status: - description: status represents the current information of a snapshot. - properties: - creationTime: - description: creationTime is the timestamp when the point-in-time - snapshot is taken by the underlying storage system. In dynamic snapshot - creation case, this field will be filled in by the CSI snapshotter - sidecar with the "creation_time" value returned from CSI "CreateSnapshot" - gRPC call. For a pre-existing snapshot, this field will be filled - with the "creation_time" value returned from the CSI "ListSnapshots" - gRPC call if the driver supports it. If not specified, it indicates - the creation time is unknown. The format of this field is a Unix - nanoseconds time encoded as an int64. On Unix, the command `date - +%s%N` returns the current time in nanoseconds since 1970-01-01 - 00:00:00 UTC. - format: int64 - type: integer - error: - description: error is the last observed error during snapshot creation, - if any. Upon success after retry, this error field will be cleared. - properties: - message: - description: 'message is a string detailing the encountered error - during snapshot creation if specified. NOTE: message may be - logged, and it should not contain sensitive information.' - type: string - time: - description: time is the timestamp when the error was encountered. - format: date-time - type: string - type: object - readyToUse: - description: readyToUse indicates if a snapshot is ready to be used - to restore a volume. In dynamic snapshot creation case, this field - will be filled in by the CSI snapshotter sidecar with the "ready_to_use" - value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing - snapshot, this field will be filled with the "ready_to_use" value - returned from the CSI "ListSnapshots" gRPC call if the driver supports - it, otherwise, this field will be set to "True". If not specified, - it means the readiness of a snapshot is unknown. - type: boolean - restoreSize: - description: restoreSize represents the complete size of the snapshot - in bytes. In dynamic snapshot creation case, this field will be - filled in by the CSI snapshotter sidecar with the "size_bytes" value - returned from CSI "CreateSnapshot" gRPC call. For a pre-existing - snapshot, this field will be filled with the "size_bytes" value - returned from the CSI "ListSnapshots" gRPC call if the driver supports - it. When restoring a volume from this snapshot, the size of the - volume MUST NOT be smaller than the restoreSize if it is specified, - otherwise the restoration will fail. If not specified, it indicates - that the size is unknown. - format: int64 - minimum: 0 - type: integer - snapshotHandle: - description: snapshotHandle is the CSI "snapshot_id" of a snapshot - on the underlying storage system. If not specified, it indicates - that dynamic snapshot creation has either failed or it is still - in progress. - type: string - type: object - required: - - spec - type: object - served: true - storage: true - subresources: - status: {} - - additionalPrinterColumns: - - description: Indicates if the snapshot is ready to be used to restore a volume. - jsonPath: .status.readyToUse - name: ReadyToUse - type: boolean - - description: Represents the complete size of the snapshot in bytes - jsonPath: .status.restoreSize - name: RestoreSize - type: integer - - description: Determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. - jsonPath: .spec.deletionPolicy - name: DeletionPolicy - type: string - - description: Name of the CSI driver used to create the physical snapshot on the underlying storage system. - jsonPath: .spec.driver - name: Driver - type: string - - description: Name of the VolumeSnapshotClass to which this snapshot belongs. - jsonPath: .spec.volumeSnapshotClassName - name: VolumeSnapshotClass - type: string - - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. - jsonPath: .spec.volumeSnapshotRef.name - name: VolumeSnapshot - type: string - - description: Namespace of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. - jsonPath: .spec.volumeSnapshotRef.namespace - name: VolumeSnapshotNamespace - type: string - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1beta1 - # This indicates the v1beta1 version of the custom resource is deprecated. - # API requests to this version receive a warning in the server response. - deprecated: true - # This overrides the default warning returned to clients making v1beta1 API requests. - deprecationWarning: "snapshot.storage.k8s.io/v1beta1 VolumeSnapshotContent is deprecated; use snapshot.storage.k8s.io/v1 VolumeSnapshotContent" - schema: - openAPIV3Schema: - description: VolumeSnapshotContent represents the actual "on-disk" snapshot object in the underlying storage system - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - spec: - description: spec defines properties of a VolumeSnapshotContent created by the underlying storage system. Required. - properties: - deletionPolicy: - description: deletionPolicy determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. For dynamically provisioned snapshots, this field will automatically be filled in by the CSI snapshotter sidecar with the "DeletionPolicy" field defined in the corresponding VolumeSnapshotClass. For pre-existing snapshots, users MUST specify this field when creating the VolumeSnapshotContent object. Required. - enum: - - Delete - - Retain - type: string - driver: - description: driver is the name of the CSI driver used to create the physical snapshot on the underlying storage system. This MUST be the same as the name returned by the CSI GetPluginName() call for that driver. Required. - type: string - source: - description: source specifies whether the snapshot is (or should be) dynamically provisioned or already exists, and just requires a Kubernetes object representation. This field is immutable after creation. Required. - properties: - snapshotHandle: - description: snapshotHandle specifies the CSI "snapshot_id" of a pre-existing snapshot on the underlying storage system for which a Kubernetes object representation was (or should be) created. This field is immutable. - type: string - volumeHandle: - description: volumeHandle specifies the CSI "volume_id" of the volume from which a snapshot should be dynamically taken from. This field is immutable. - type: string - type: object - volumeSnapshotClassName: - description: name of the VolumeSnapshotClass from which this snapshot was (or will be) created. Note that after provisioning, the VolumeSnapshotClass may be deleted or recreated with different set of values, and as such, should not be referenced post-snapshot creation. - type: string - volumeSnapshotRef: - description: volumeSnapshotRef specifies the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName field must reference to this VolumeSnapshotContent's name for the bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent object, name and namespace of the VolumeSnapshot object MUST be provided for binding to happen. This field is immutable after creation. Required. - properties: - apiVersion: - description: API version of the referent. - type: string - fieldPath: - description: 'If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. For example, if the object reference is to a container within a pod, this would take on a value like: "spec.containers{name}" (where "name" refers to the name of the container that triggered the event) or if no container name is specified "spec.containers[2]" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. TODO: this design is not final and this field is subject to change in the future.' - type: string - kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' - type: string - namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' - type: string - resourceVersion: - description: 'Specific resourceVersion to which this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' - type: string - uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' - type: string - type: object - required: - - deletionPolicy - - driver - - source - - volumeSnapshotRef - type: object - status: - description: status represents the current information of a snapshot. - properties: - creationTime: - description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it indicates the creation time is unknown. The format of this field is a Unix nanoseconds time encoded as an int64. On Unix, the command `date +%s%N` returns the current time in nanoseconds since 1970-01-01 00:00:00 UTC. - format: int64 - type: integer - error: - description: error is the last observed error during snapshot creation, if any. Upon success after retry, this error field will be cleared. - properties: - message: - description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.' - type: string - time: - description: time is the timestamp when the error was encountered. - format: date-time - type: string - type: object - readyToUse: - description: readyToUse indicates if a snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown. - type: boolean - restoreSize: - description: restoreSize represents the complete size of the snapshot in bytes. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown. - format: int64 - minimum: 0 - type: integer - snapshotHandle: - description: snapshotHandle is the CSI "snapshot_id" of a snapshot on the underlying storage system. If not specified, it indicates that dynamic snapshot creation has either failed or it is still in progress. - type: string - type: object - required: - - spec - type: object - served: false - storage: false - subresources: - status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- - -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.8.0 - api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/665" - creationTimestamp: null - name: volumesnapshots.snapshot.storage.k8s.io -spec: - group: snapshot.storage.k8s.io - names: - kind: VolumeSnapshot - listKind: VolumeSnapshotList - plural: volumesnapshots - shortNames: - - vs - singular: volumesnapshot - scope: Namespaced - versions: - - additionalPrinterColumns: - - description: Indicates if the snapshot is ready to be used to restore a volume. - jsonPath: .status.readyToUse - name: ReadyToUse - type: boolean - - description: If a new snapshot needs to be created, this contains the name of - the source PVC from which this snapshot was (or will be) created. - jsonPath: .spec.source.persistentVolumeClaimName - name: SourcePVC - type: string - - description: If a snapshot already exists, this contains the name of the existing - VolumeSnapshotContent object representing the existing snapshot. - jsonPath: .spec.source.volumeSnapshotContentName - name: SourceSnapshotContent - type: string - - description: Represents the minimum size of volume required to rehydrate from - this snapshot. - jsonPath: .status.restoreSize - name: RestoreSize - type: string - - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot. - jsonPath: .spec.volumeSnapshotClassName - name: SnapshotClass - type: string - - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot - object intends to bind to. Please note that verification of binding actually - requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure - both are pointing at each other. Binding MUST be verified prior to usage of - this object. - jsonPath: .status.boundVolumeSnapshotContentName - name: SnapshotContent - type: string - - description: Timestamp when the point-in-time snapshot was taken by the underlying - storage system. - jsonPath: .status.creationTime - name: CreationTime - type: date - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1 - schema: - openAPIV3Schema: - description: VolumeSnapshot is a user's request for either creating a point-in-time - snapshot of a persistent volume, or binding to a pre-existing snapshot. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - spec: - description: 'spec defines the desired characteristics of a snapshot requested - by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots - Required.' - properties: - source: - description: source specifies where a snapshot will be created from. - This field is immutable after creation. Required. - properties: - persistentVolumeClaimName: - description: persistentVolumeClaimName specifies the name of the - PersistentVolumeClaim object representing the volume from which - a snapshot should be created. This PVC is assumed to be in the - same namespace as the VolumeSnapshot object. This field should - be set if the snapshot does not exists, and needs to be created. - This field is immutable. - type: string - volumeSnapshotContentName: - description: volumeSnapshotContentName specifies the name of a - pre-existing VolumeSnapshotContent object representing an existing - volume snapshot. This field should be set if the snapshot already - exists and only needs a representation in Kubernetes. This field - is immutable. - type: string - type: object - oneOf: - - required: ["persistentVolumeClaimName"] - - required: ["volumeSnapshotContentName"] - volumeSnapshotClassName: - description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass - requested by the VolumeSnapshot. VolumeSnapshotClassName may be - left nil to indicate that the default SnapshotClass should be used. - A given cluster may have multiple default Volume SnapshotClasses: - one default per CSI Driver. If a VolumeSnapshot does not specify - a SnapshotClass, VolumeSnapshotSource will be checked to figure - out what the associated CSI Driver is, and the default VolumeSnapshotClass - associated with that CSI Driver will be used. If more than one VolumeSnapshotClass - exist for a given CSI Driver and more than one have been marked - as default, CreateSnapshot will fail and generate an event. Empty - string is not allowed for this field.' - type: string - required: - - source - type: object - status: - description: status represents the current information of a snapshot. - Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent - objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent - point at each other) before using this object. - properties: - boundVolumeSnapshotContentName: - description: 'boundVolumeSnapshotContentName is the name of the VolumeSnapshotContent - object to which this VolumeSnapshot object intends to bind to. If - not specified, it indicates that the VolumeSnapshot object has not - been successfully bound to a VolumeSnapshotContent object yet. NOTE: - To avoid possible security issues, consumers must verify binding - between VolumeSnapshot and VolumeSnapshotContent objects is successful - (by validating that both VolumeSnapshot and VolumeSnapshotContent - point at each other) before using this object.' - type: string - creationTime: - description: creationTime is the timestamp when the point-in-time - snapshot is taken by the underlying storage system. In dynamic snapshot - creation case, this field will be filled in by the snapshot controller - with the "creation_time" value returned from CSI "CreateSnapshot" - gRPC call. For a pre-existing snapshot, this field will be filled - with the "creation_time" value returned from the CSI "ListSnapshots" - gRPC call if the driver supports it. If not specified, it may indicate - that the creation time of the snapshot is unknown. - format: date-time - type: string - error: - description: error is the last observed error during snapshot creation, - if any. This field could be helpful to upper level controllers(i.e., - application controller) to decide whether they should continue on - waiting for the snapshot to be created based on the type of error - reported. The snapshot controller will keep retrying when an error - occurs during the snapshot creation. Upon success, this error field - will be cleared. - properties: - message: - description: 'message is a string detailing the encountered error - during snapshot creation if specified. NOTE: message may be - logged, and it should not contain sensitive information.' - type: string - time: - description: time is the timestamp when the error was encountered. - format: date-time - type: string - type: object - readyToUse: - description: readyToUse indicates if the snapshot is ready to be used - to restore a volume. In dynamic snapshot creation case, this field - will be filled in by the snapshot controller with the "ready_to_use" - value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing - snapshot, this field will be filled with the "ready_to_use" value - returned from the CSI "ListSnapshots" gRPC call if the driver supports - it, otherwise, this field will be set to "True". If not specified, - it means the readiness of a snapshot is unknown. - type: boolean - restoreSize: - type: string - description: restoreSize represents the minimum size of volume required - to create a volume from this snapshot. In dynamic snapshot creation - case, this field will be filled in by the snapshot controller with - the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. - For a pre-existing snapshot, this field will be filled with the - "size_bytes" value returned from the CSI "ListSnapshots" gRPC call - if the driver supports it. When restoring a volume from this snapshot, - the size of the volume MUST NOT be smaller than the restoreSize - if it is specified, otherwise the restoration will fail. If not - specified, it indicates that the size is unknown. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - required: - - spec - type: object - served: true - storage: true - subresources: - status: {} - - additionalPrinterColumns: - - description: Indicates if the snapshot is ready to be used to restore a volume. - jsonPath: .status.readyToUse - name: ReadyToUse - type: boolean - - description: If a new snapshot needs to be created, this contains the name of the source PVC from which this snapshot was (or will be) created. - jsonPath: .spec.source.persistentVolumeClaimName - name: SourcePVC - type: string - - description: If a snapshot already exists, this contains the name of the existing VolumeSnapshotContent object representing the existing snapshot. - jsonPath: .spec.source.volumeSnapshotContentName - name: SourceSnapshotContent - type: string - - description: Represents the minimum size of volume required to rehydrate from this snapshot. - jsonPath: .status.restoreSize - name: RestoreSize - type: string - - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot. - jsonPath: .spec.volumeSnapshotClassName - name: SnapshotClass - type: string - - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot object intends to bind to. Please note that verification of binding actually requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure both are pointing at each other. Binding MUST be verified prior to usage of this object. - jsonPath: .status.boundVolumeSnapshotContentName - name: SnapshotContent - type: string - - description: Timestamp when the point-in-time snapshot was taken by the underlying storage system. - jsonPath: .status.creationTime - name: CreationTime - type: date - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1beta1 - # This indicates the v1beta1 version of the custom resource is deprecated. - # API requests to this version receive a warning in the server response. - deprecated: true - # This overrides the default warning returned to clients making v1beta1 API requests. - deprecationWarning: "snapshot.storage.k8s.io/v1beta1 VolumeSnapshot is deprecated; use snapshot.storage.k8s.io/v1 VolumeSnapshot" - schema: - openAPIV3Schema: - description: VolumeSnapshot is a user's request for either creating a point-in-time snapshot of a persistent volume, or binding to a pre-existing snapshot. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - spec: - description: 'spec defines the desired characteristics of a snapshot requested by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots Required.' - properties: - source: - description: source specifies where a snapshot will be created from. This field is immutable after creation. Required. - properties: - persistentVolumeClaimName: - description: persistentVolumeClaimName specifies the name of the PersistentVolumeClaim object representing the volume from which a snapshot should be created. This PVC is assumed to be in the same namespace as the VolumeSnapshot object. This field should be set if the snapshot does not exists, and needs to be created. This field is immutable. - type: string - volumeSnapshotContentName: - description: volumeSnapshotContentName specifies the name of a pre-existing VolumeSnapshotContent object representing an existing volume snapshot. This field should be set if the snapshot already exists and only needs a representation in Kubernetes. This field is immutable. - type: string - type: object - volumeSnapshotClassName: - description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass requested by the VolumeSnapshot. VolumeSnapshotClassName may be left nil to indicate that the default SnapshotClass should be used. A given cluster may have multiple default Volume SnapshotClasses: one default per CSI Driver. If a VolumeSnapshot does not specify a SnapshotClass, VolumeSnapshotSource will be checked to figure out what the associated CSI Driver is, and the default VolumeSnapshotClass associated with that CSI Driver will be used. If more than one VolumeSnapshotClass exist for a given CSI Driver and more than one have been marked as default, CreateSnapshot will fail and generate an event. Empty string is not allowed for this field.' - type: string - required: - - source - type: object - status: - description: status represents the current information of a snapshot. Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object. - properties: - boundVolumeSnapshotContentName: - description: 'boundVolumeSnapshotContentName is the name of the VolumeSnapshotContent object to which this VolumeSnapshot object intends to bind to. If not specified, it indicates that the VolumeSnapshot object has not been successfully bound to a VolumeSnapshotContent object yet. NOTE: To avoid possible security issues, consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object.' - type: string - creationTime: - description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it may indicate that the creation time of the snapshot is unknown. - format: date-time - type: string - error: - description: error is the last observed error during snapshot creation, if any. This field could be helpful to upper level controllers(i.e., application controller) to decide whether they should continue on waiting for the snapshot to be created based on the type of error reported. The snapshot controller will keep retrying when an error occurs during the snapshot creation. Upon success, this error field will be cleared. - properties: - message: - description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.' - type: string - time: - description: time is the timestamp when the error was encountered. - format: date-time - type: string - type: object - readyToUse: - description: readyToUse indicates if the snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown. - type: boolean - restoreSize: - type: string - description: restoreSize represents the minimum size of volume required to create a volume from this snapshot. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - required: - - spec - type: object - served: false - storage: false - subresources: - status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/deploy/kubernetes/releases/csi-digitalocean-v4.7.0/driver.yaml b/deploy/kubernetes/releases/csi-digitalocean-v4.7.0/driver.yaml deleted file mode 100644 index 6913961d3..000000000 --- a/deploy/kubernetes/releases/csi-digitalocean-v4.7.0/driver.yaml +++ /dev/null @@ -1,533 +0,0 @@ -# Copyright 2022 DigitalOcean -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# Install the CSI Driver. This simplifies driver discovery and enables us to -# customize Kubernetes behavior -# https://kubernetes-csi.github.io/docs/csi-driver-object.html -apiVersion: storage.k8s.io/v1 -kind: CSIDriver -metadata: - name: dobs.csi.digitalocean.com -spec: - attachRequired: true - podInfoOnMount: true - ---- - -kind: VolumeSnapshotClass -apiVersion: snapshot.storage.k8s.io/v1 -metadata: - name: do-block-storage - annotations: - snapshot.storage.kubernetes.io/is-default-class: "true" -driver: dobs.csi.digitalocean.com -deletionPolicy: Delete - ---- - -kind: StorageClass -apiVersion: storage.k8s.io/v1 -metadata: - name: do-block-storage-luks-global -provisioner: dobs.csi.digitalocean.com -allowVolumeExpansion: true -parameters: - dobs.csi.digitalocean.com/luks-encrypted: "true" - dobs.csi.digitalocean.com/luks-cipher: "aes-xts-plain64" - dobs.csi.digitalocean.com/luks-key-size: "512" - csi.storage.k8s.io/node-stage-secret-namespace: "kube-system" - csi.storage.k8s.io/node-stage-secret-name: "luks-key" - ---- - -kind: StorageClass -apiVersion: storage.k8s.io/v1 -metadata: - name: do-block-storage-luks -provisioner: dobs.csi.digitalocean.com -allowVolumeExpansion: true -parameters: - dobs.csi.digitalocean.com/luks-encrypted: "true" - dobs.csi.digitalocean.com/luks-cipher: "aes-xts-plain64" - dobs.csi.digitalocean.com/luks-key-size: "512" - csi.storage.k8s.io/node-stage-secret-namespace: ${pvc.namespace} - csi.storage.k8s.io/node-stage-secret-name: ${pvc.name}-luks-key - ---- - -kind: StorageClass -apiVersion: storage.k8s.io/v1 -metadata: - name: do-block-storage - annotations: - storageclass.kubernetes.io/is-default-class: "true" -provisioner: dobs.csi.digitalocean.com -allowVolumeExpansion: true - ---- - -kind: StorageClass -apiVersion: storage.k8s.io/v1 -metadata: - name: do-block-storage-retain -provisioner: dobs.csi.digitalocean.com -reclaimPolicy: Retain -allowVolumeExpansion: true - ---- - -kind: StorageClass -apiVersion: storage.k8s.io/v1 -metadata: - name: do-block-storage-xfs -provisioner: dobs.csi.digitalocean.com -parameters: - fstype: xfs -allowVolumeExpansion: true - ---- - -kind: StorageClass -apiVersion: storage.k8s.io/v1 -metadata: - name: do-block-storage-xfs-retain -provisioner: dobs.csi.digitalocean.com -parameters: - fstype: xfs -reclaimPolicy: Retain -allowVolumeExpansion: true - ---- - -############################################## -########### ############ -########### Controller plugin ############ -########### ############ -############################################## - -kind: StatefulSet -apiVersion: apps/v1 -metadata: - name: csi-do-controller - namespace: kube-system -spec: - serviceName: "csi-do" - selector: - matchLabels: - app: csi-do-controller - replicas: 1 - template: - metadata: - annotations: - kubectl.kubernetes.io/default-container: csi-do-plugin - labels: - app: csi-do-controller - role: csi-do - spec: - priorityClassName: system-cluster-critical - serviceAccount: csi-do-controller-sa - containers: - - name: csi-provisioner - image: registry.k8s.io/sig-storage/csi-provisioner:v3.5.0 - args: - - "--csi-address=$(ADDRESS)" - - "--default-fstype=ext4" - - "--v=5" - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - imagePullPolicy: "IfNotPresent" - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/sockets/pluginproxy/ - - name: csi-attacher - image: registry.k8s.io/sig-storage/csi-attacher:v4.3.0 - args: - - "--csi-address=$(ADDRESS)" - - "--v=5" - - "--reconcile-sync=30m" - - "--timeout=2m" - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - imagePullPolicy: "IfNotPresent" - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/sockets/pluginproxy/ - - name: csi-snapshotter - image: registry.k8s.io/sig-storage/csi-snapshotter:v6.2.1 - args: - - "--csi-address=$(ADDRESS)" - - "--v=5" - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - imagePullPolicy: IfNotPresent - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/sockets/pluginproxy/ - - name: csi-resizer - image: registry.k8s.io/sig-storage/csi-resizer:v1.8.0 - args: - - "--csi-address=$(ADDRESS)" - - "--timeout=30s" - - "--v=5" - # DO volumes support online resize. - - "--handle-volume-inuse-error=false" - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - imagePullPolicy: "IfNotPresent" - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/sockets/pluginproxy/ - - name: csi-do-plugin - image: digitalocean/do-csi-plugin:v4.7.0 - args : - - "--endpoint=$(CSI_ENDPOINT)" - - "--token=$(DIGITALOCEAN_ACCESS_TOKEN)" - - "--url=$(DIGITALOCEAN_API_URL)" - - "--default-volumes-page-size=200" - env: - - name: CSI_ENDPOINT - value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock - - name: DIGITALOCEAN_API_URL - value: https://api.digitalocean.com/ - - name: DIGITALOCEAN_ACCESS_TOKEN - valueFrom: - secretKeyRef: - name: digitalocean - key: access-token - imagePullPolicy: "Always" - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/sockets/pluginproxy/ - volumes: - - name: socket-dir - emptyDir: {} - ---- - -kind: ServiceAccount -apiVersion: v1 -metadata: - name: csi-do-controller-sa - namespace: kube-system - ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-do-provisioner-role -rules: - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "delete"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["get", "list"] - - apiGroups: [ "storage.k8s.io" ] - resources: [ "csinodes" ] - verbs: [ "get", "list", "watch" ] - - apiGroups: [ "" ] - resources: [ "nodes" ] - verbs: [ "get", "list", "watch" ] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch"] - ---- - -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-do-provisioner-binding -subjects: - - kind: ServiceAccount - name: csi-do-controller-sa - namespace: kube-system -roleRef: - kind: ClusterRole - name: csi-do-provisioner-role - apiGroup: rbac.authorization.k8s.io - ---- -# Attacher must be able to work with PVs, nodes and VolumeAttachments -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-do-attacher-role -rules: - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["csinodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments/status"] - verbs: ["patch"] - ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-do-attacher-binding -subjects: - - kind: ServiceAccount - name: csi-do-controller-sa - namespace: kube-system -roleRef: - kind: ClusterRole - name: csi-do-attacher-role - apiGroup: rbac.authorization.k8s.io - ---- - -# Snapshotter sidecar - -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-do-snapshotter-role -rules: - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents/status"] - verbs: ["update", "patch"] - ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-do-snapshotter-binding -subjects: - - kind: ServiceAccount - name: csi-do-controller-sa - namespace: kube-system -roleRef: - kind: ClusterRole - name: csi-do-snapshotter-role - apiGroup: rbac.authorization.k8s.io - ---- - -# Resizer must be able to work with PVCs, PVs, SCs. -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-do-resizer-role -rules: - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims/status"] - verbs: ["update", "patch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - ---- - -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-do-resizer-binding -subjects: - - kind: ServiceAccount - name: csi-do-controller-sa - namespace: kube-system -roleRef: - kind: ClusterRole - name: csi-do-resizer-role - apiGroup: rbac.authorization.k8s.io - ---- - -######################################## -########### ############ -########### Node plugin ############ -########### ############ -######################################## - -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: csi-do-node - namespace: kube-system -spec: - selector: - matchLabels: - app: csi-do-node - template: - metadata: - annotations: - kubectl.kubernetes.io/default-container: csi-do-plugin - labels: - app: csi-do-node - role: csi-do - spec: - priorityClassName: system-node-critical - serviceAccount: csi-do-node-sa - hostNetwork: true - initContainers: - # Delete automount udev rule running on all DO droplets. The rule mounts - # devices briefly and may conflict with CSI-managed droplets (leading to - # "resource busy" errors). We can safely delete it in DOKS. - - name: automount-udev-deleter - image: alpine:3 - args: - - "rm" - - "-f" - - "/etc/udev/rules.d/99-digitalocean-automount.rules" - volumeMounts: - - name: udev-rules-dir - mountPath: /etc/udev/rules.d/ - containers: - - name: csi-node-driver-registrar - image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.8.0 - args: - - "--v=5" - - "--csi-address=$(ADDRESS)" - - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)" - env: - - name: ADDRESS - value: /csi/csi.sock - - name: DRIVER_REG_SOCK_PATH - value: /var/lib/kubelet/plugins/dobs.csi.digitalocean.com/csi.sock - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - volumeMounts: - - name: plugin-dir - mountPath: /csi/ - - name: registration-dir - mountPath: /registration/ - - name: csi-do-plugin - image: digitalocean/do-csi-plugin:v4.7.0 - args : - - "--endpoint=$(CSI_ENDPOINT)" - - "--url=$(DIGITALOCEAN_API_URL)" - env: - - name: CSI_ENDPOINT - value: unix:///csi/csi.sock - - name: DIGITALOCEAN_API_URL - value: https://api.digitalocean.com/ - imagePullPolicy: "Always" - securityContext: - privileged: true - capabilities: - add: ["SYS_ADMIN"] - allowPrivilegeEscalation: true - volumeMounts: - - name: plugin-dir - mountPath: /csi - - name: pods-mount-dir - mountPath: /var/lib/kubelet - # needed so that any mounts setup inside this container are - # propagated back to the host machine. - mountPropagation: "Bidirectional" - - name: device-dir - mountPath: /dev - - name: tmpfs - mountPath: /tmp - volumes: - - name: registration-dir - hostPath: - path: /var/lib/kubelet/plugins_registry/ - type: DirectoryOrCreate - - name: plugin-dir - hostPath: - path: /var/lib/kubelet/plugins/dobs.csi.digitalocean.com - type: DirectoryOrCreate - - name: pods-mount-dir - hostPath: - path: /var/lib/kubelet - type: Directory - - name: device-dir - hostPath: - path: /dev - - name: udev-rules-dir - hostPath: - path: /etc/udev/rules.d/ - # to make sure temporary stored luks keys never touch a disk - - name: tmpfs - emptyDir: - medium: Memory ---- - -apiVersion: v1 -kind: ServiceAccount -metadata: - name: csi-do-node-sa - namespace: kube-system - ---- - -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-do-node-driver-registrar-role - namespace: kube-system -rules: - - apiGroups: [""] - resources: ["events"] - verbs: ["get", "list", "watch", "create", "update", "patch"] - ---- - -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-do-node-driver-registrar-binding -subjects: - - kind: ServiceAccount - name: csi-do-node-sa - namespace: kube-system -roleRef: - kind: ClusterRole - name: csi-do-node-driver-registrar-role - apiGroup: rbac.authorization.k8s.io diff --git a/deploy/kubernetes/releases/csi-digitalocean-v4.7.0/snapshot-controller.yaml b/deploy/kubernetes/releases/csi-digitalocean-v4.7.0/snapshot-controller.yaml deleted file mode 100644 index d61c26e7e..000000000 --- a/deploy/kubernetes/releases/csi-digitalocean-v4.7.0/snapshot-controller.yaml +++ /dev/null @@ -1,98 +0,0 @@ -# Copyright 2022 DigitalOcean -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# -# snapshotter controller -# source: # Source: https://github.com/kubernetes-csi/external-snapshotter -# - -kind: StatefulSet -apiVersion: apps/v1 -metadata: - name: snapshot-controller - namespace: kube-system -spec: - serviceName: "snapshot-controller" - replicas: 1 - selector: - matchLabels: - app: snapshot-controller - template: - metadata: - labels: - app: snapshot-controller - spec: - serviceAccountName: snapshot-controller - containers: - - name: snapshot-controller - image: registry.k8s.io/sig-storage/snapshot-controller:v6.2.1 - args: - - "--v=5" - imagePullPolicy: IfNotPresent - ---- - -apiVersion: v1 -kind: ServiceAccount -metadata: - name: snapshot-controller - namespace: kube-system - ---- - -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: snapshot-controller-role -rules: - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["create", "get", "list", "watch", "update", "delete", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents/status"] - verbs: ["patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots/status"] - verbs: ["update", "patch"] - ---- - -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: snapshot-controller-binding -subjects: - - kind: ServiceAccount - name: snapshot-controller - namespace: kube-system -roleRef: - kind: ClusterRole - name: snapshot-controller-role - apiGroup: rbac.authorization.k8s.io diff --git a/deploy/kubernetes/releases/csi-digitalocean-v4.7.0/snapshot-validation-webhook.yaml b/deploy/kubernetes/releases/csi-digitalocean-v4.7.0/snapshot-validation-webhook.yaml deleted file mode 100644 index af864470e..000000000 --- a/deploy/kubernetes/releases/csi-digitalocean-v4.7.0/snapshot-validation-webhook.yaml +++ /dev/null @@ -1,128 +0,0 @@ -# Copyright 2022 DigitalOcean -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - name: "validation-webhook.snapshot.storage.k8s.io" -webhooks: - - name: "validation-webhook.snapshot.storage.k8s.io" - rules: - - apiGroups: ["snapshot.storage.k8s.io"] - apiVersions: ["v1", "v1beta1"] - operations: ["CREATE", "UPDATE"] - resources: ["volumesnapshots", "volumesnapshotcontents"] - scope: "*" - clientConfig: - service: - namespace: "kube-system" - name: "snapshot-validation-service" - path: "/volumesnapshot" - # XXX Uncomment and populate the CA bundle field accordingly if a dedicated - # CA is to be used. - # caBundle: ${CA_BUNDLE} - admissionReviewVersions: ["v1", "v1beta1"] - sideEffects: None - failurePolicy: Fail - timeoutSeconds: 5 - ---- - -apiVersion: apps/v1 -kind: Deployment -metadata: - name: snapshot-validation - namespace: kube-system - labels: - app: snapshot-validation -spec: - replicas: 1 - selector: - matchLabels: - app: snapshot-validation - strategy: - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - type: RollingUpdate - template: - metadata: - labels: - app: snapshot-validation - spec: - serviceAccountName: snapshot-validation - containers: - - name: snapshot-validation - image: registry.k8s.io/sig-storage/snapshot-validation-webhook:v6.2.1 - imagePullPolicy: IfNotPresent - args: ['--tls-cert-file=/etc/snapshot-validation-webhook/certs/cert.pem', '--tls-private-key-file=/etc/snapshot-validation-webhook/certs/key.pem'] - ports: - - containerPort: 443 - volumeMounts: - - name: snapshot-validation-webhook-certs - mountPath: /etc/snapshot-validation-webhook/certs - readOnly: true - volumes: - - name: snapshot-validation-webhook-certs - secret: - # XXX Populate the secret properly with a certificate and key - secretName: snapshot-validation-secret - ---- - -apiVersion: v1 -kind: ServiceAccount -metadata: - name: snapshot-validation - namespace: kube-system - ---- - -apiVersion: v1 -kind: Service -metadata: - name: snapshot-validation-service - namespace: kube-system -spec: - selector: - app: snapshot-validation - ports: - - protocol: TCP - port: 443 - ---- - -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: snapshot-validation -rules: - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotclasses"] - verbs: ["get", "list", "watch"] - ---- - -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: snapshot-validation -subjects: - - kind: ServiceAccount - name: snapshot-validation - namespace: kube-system -roleRef: - kind: ClusterRole - name: snapshot-validation - apiGroup: rbac.authorization.k8s.io From 7dfa6c67bc5f892a96d0d4c8c9ae307cd1e1ecb6 Mon Sep 17 00:00:00 2001 From: Max Petyurenko Date: Thu, 13 Jul 2023 14:48:54 +0300 Subject: [PATCH 3/5] Remove LUKS storage class. --- README.md | 10 +++---- .../releases/csi-digitalocean-dev/driver.yaml | 30 ------------------- 2 files changed, 4 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 161516ff0..0570da2b6 100644 --- a/README.md +++ b/README.md @@ -285,12 +285,10 @@ For LUKS encryption: * `dobs.csi.digitalocean.com/luks-encrypted`: set to the string `"true"` if the volume should be encrypted with LUKS -* `dobs.csi.digitalocean.com/luks-cipher`: cipher to use; must be supported by the kernel and luks -* `dobs.csi.digitalocean.com/luks-key-size`: key-size to use - -For LUKS encrypted volumes, a secret that contains the LUKS key needs to be referenced through -the `csi.storage.k8s.io/node-stage-secret-name` and `csi.storage.k8s.io/node-stage-secret-namespace` -parameter. See the included `StorageClass` definition. +* `dobs.csi.digitalocean.com/luks-cipher`: cipher to use; must be supported by the kernel and luks. E.g. `aes-xts-plain64` +* `dobs.csi.digitalocean.com/luks-key-size`: key-size to use. E.g. `512` +* `csi.storage.k8s.io/node-stage-secret-name`: secret name that contains LUKS key. E.g. `${pvc.name}-luks-key` +* `csi.storage.k8s.io/node-stage-secret-namespace`: secret key namespace. E.g. `${pvc.namespace}` ## Upgrading diff --git a/deploy/kubernetes/releases/csi-digitalocean-dev/driver.yaml b/deploy/kubernetes/releases/csi-digitalocean-dev/driver.yaml index 00a1558af..7d4e2f9ad 100644 --- a/deploy/kubernetes/releases/csi-digitalocean-dev/driver.yaml +++ b/deploy/kubernetes/releases/csi-digitalocean-dev/driver.yaml @@ -38,36 +38,6 @@ deletionPolicy: Delete --- -kind: StorageClass -apiVersion: storage.k8s.io/v1 -metadata: - name: do-block-storage-luks-global -provisioner: dobs.csi.digitalocean.com -allowVolumeExpansion: true -parameters: - dobs.csi.digitalocean.com/luks-encrypted: "true" - dobs.csi.digitalocean.com/luks-cipher: "aes-xts-plain64" - dobs.csi.digitalocean.com/luks-key-size: "512" - csi.storage.k8s.io/node-stage-secret-namespace: "kube-system" - csi.storage.k8s.io/node-stage-secret-name: "luks-key" - ---- - -kind: StorageClass -apiVersion: storage.k8s.io/v1 -metadata: - name: do-block-storage-luks -provisioner: dobs.csi.digitalocean.com -allowVolumeExpansion: true -parameters: - dobs.csi.digitalocean.com/luks-encrypted: "true" - dobs.csi.digitalocean.com/luks-cipher: "aes-xts-plain64" - dobs.csi.digitalocean.com/luks-key-size: "512" - csi.storage.k8s.io/node-stage-secret-namespace: ${pvc.namespace} - csi.storage.k8s.io/node-stage-secret-name: ${pvc.name}-luks-key - ---- - kind: StorageClass apiVersion: storage.k8s.io/v1 metadata: From ec5f1453f803c1699081f12c1280ea1cd31d0344 Mon Sep 17 00:00:00 2001 From: Max Petyurenko Date: Thu, 13 Jul 2023 15:13:53 +0300 Subject: [PATCH 4/5] Return formatted error in case LUKS validation is failed. --- driver/mounter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/driver/mounter.go b/driver/mounter.go index 0bce82527..80778be20 100644 --- a/driver/mounter.go +++ b/driver/mounter.go @@ -172,7 +172,7 @@ func (m *mounter) Format(source, fsType string, luksContext LuksContext) error { } else { err := luksContext.validate() if err != nil { - return err + return fmt.Errorf("validation failed: %s", err.Error()) } err = luksFormat(source, mkfsCmd, mkfsArgs, luksContext, m.log) From 3db00e4f229966ab884e63f409047ae8b83402e5 Mon Sep 17 00:00:00 2001 From: Max Petyurenko Date: Fri, 4 Aug 2023 19:13:47 +0300 Subject: [PATCH 5/5] Add e2e test for LUKS volume. --- test/e2e/Dockerfile | 9 + test/e2e/luks/framework/framework.go | 167 +++++++++ test/e2e/luks/go.mod | 54 +++ test/e2e/luks/go.sum | 492 +++++++++++++++++++++++++++ test/e2e/luks/luks_suite_test.go | 13 + test/e2e/luks/luks_test.go | 242 +++++++++++++ test/e2e/run-versioned-e2e-tests.sh | 3 + 7 files changed, 980 insertions(+) create mode 100644 test/e2e/luks/framework/framework.go create mode 100644 test/e2e/luks/go.mod create mode 100644 test/e2e/luks/go.sum create mode 100644 test/e2e/luks/luks_suite_test.go create mode 100644 test/e2e/luks/luks_test.go diff --git a/test/e2e/Dockerfile b/test/e2e/Dockerfile index 85f26c962..db737b4bd 100644 --- a/test/e2e/Dockerfile +++ b/test/e2e/Dockerfile @@ -60,6 +60,14 @@ RUN echo "${KUBE_VERSION_1_24_E2E_BIN_SHA256_CHECKSUM}" e2e.test | sha256sum --c RUN cp e2e.test /e2e.1.24.test RUN cp ginkgo /ginkgo-1.24 +### LUKS +FROM builder AS luks + +ADD luks /luks +RUN cd /luks && go install github.com/onsi/ginkgo/v2/ginkgo +RUN cd /luks && GOOS=linux GOARCH=amd64 CGO_ENABLED=0 ginkgo build +RUN cp /luks/luks.test /e2e.luks.test + FROM golang:1.20 AS tools # See comment at the bottom on why we need tini. ARG TINI_VERSION=0.19.0 @@ -90,6 +98,7 @@ COPY --from=tests-1.25 /e2e.1.25.test / COPY --from=tests-1.25 /ginkgo-1.25 /usr/local/bin COPY --from=tests-1.24 /e2e.1.24.test / COPY --from=tests-1.24 /ginkgo-1.24 /usr/local/bin +COPY --from=luks /e2e.luks.test / COPY --from=tools /tini /sbin/ COPY --from=tools /doctl /usr/local/bin/ COPY --from=tools /kubectl /usr/local/bin/ diff --git a/test/e2e/luks/framework/framework.go b/test/e2e/luks/framework/framework.go new file mode 100644 index 000000000..970588e78 --- /dev/null +++ b/test/e2e/luks/framework/framework.go @@ -0,0 +1,167 @@ +package framework + +import ( + "context" + "flag" + "fmt" + "github.com/onsi/ginkgo/v2" + v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/util/homedir" + "os" + "path/filepath" +) + +var ( + f Framework + TestDriverPath string +) + +type Framework struct { + BaseName string + Namespace *v1.Namespace + ClientSet clientset.Interface + DriverData *DriverDefinition +} + +func init() { + flag.StringVar(&TestDriverPath, "storage.testdriver", "", "The testdriver yaml file") +} + +func NewDefaultFramework(baseName string) *Framework { + return NewFramework(baseName, nil) +} + +func NewFramework(baseName string, client clientset.Interface) *Framework { + f := &Framework{ + BaseName: baseName, + ClientSet: client, + } + + ginkgo.BeforeEach(f.BeforeEach) + + return f +} + +func (f *Framework) BeforeEach(ctx context.Context) { + ginkgo.DeferCleanup(f.AfterEach) + + driver, err := readTestDriverFile() + if err != nil { + ginkgo.Fail(fmt.Sprintf("Not able to read testdriver yaml file, %s", err.Error())) + } + + f.DriverData = driver + + var kubeconfig *string + + // Try to get kube config either from user home directory or absolute path. + if home := homedir.HomeDir(); home != "" { + kubeconfig = flag.String("kubeconfig", filepath.Join(home, ".kube", "config"), "(optional) absolute path to the kubeconfig file") + } else { + kubeconfig = flag.String("kubeconfig", "", "absolute path to the kubeconfig file") + } + + // Override with KUBECONFIG env variable. + kubeconfigEnv := os.Getenv("KUBECONFIG") + if kubeconfigEnv != "" { + *kubeconfig = kubeconfigEnv + } + + ginkgo.By(fmt.Sprintf("Creating a kubernetes client, basename %s", f.BaseName)) + + config, err := clientcmd.BuildConfigFromFlags("", *kubeconfig) + if err != nil { + ginkgo.Fail(err.Error()) + } + + f.ClientSet, err = clientset.NewForConfig(config) + if err != nil { + ginkgo.Fail(err.Error()) + } + + ns, err := f.CreateNamespace(ctx) + + f.Namespace = ns +} + +func (f *Framework) AfterEach(ctx context.Context) { + defer func() { + ginkgo.By(fmt.Sprintf("Destroying namespace: %s", f.Namespace.Name)) + + if err := f.ClientSet.CoreV1().Namespaces().Delete(ctx, f.Namespace.Name, metav1.DeleteOptions{}); err != nil { + if !apierrors.IsNotFound(err) { + ginkgo.By(fmt.Sprintf("Could not delete namespace: %s", f.Namespace.Name)) + } else { + ginkgo.By(fmt.Sprintf("Namespace was already deleted: %s", f.Namespace.Name)) + } + } + + f.Namespace = nil + f.ClientSet = nil + }() +} + +func (f *Framework) CreateNamespace(ctx context.Context) (*v1.Namespace, error) { + nsSpec := &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: f.BaseName, + }, + Status: v1.NamespaceStatus{}, + } + + ginkgo.By(fmt.Sprintf("Create namespace: %s", f.BaseName)) + + ns, err := f.ClientSet.CoreV1().Namespaces().Create(ctx, nsSpec, metav1.CreateOptions{}) + + if err != nil { + ginkgo.By(fmt.Sprintf("Failed to create namespace: %v", err)) + + return nil, err + } + + return ns, nil +} + +func readTestDriverFile() (*DriverDefinition, error) { + data, err := os.ReadFile(TestDriverPath) + if err != nil { + return nil, err + } + + driverData := &DriverDefinition{} + + if err := runtime.DecodeInto(scheme.Codecs.UniversalDecoder(), data, driverData); err != nil { + return nil, err + } + + if driverData.DriverInfo.Name == "" { + return nil, fmt.Errorf("DriverInfo.Name is not set in file: %s", TestDriverPath) + } + + return driverData, nil +} + +type DriverDefinition struct { + DriverInfo struct { + Name string + SupportedSizeRange struct { + Max string + Min string + } + } +} + +func (d *DriverDefinition) GetObjectKind() schema.ObjectKind { + return nil +} + +func (d *DriverDefinition) DeepCopyObject() runtime.Object { + return nil +} diff --git a/test/e2e/luks/go.mod b/test/e2e/luks/go.mod new file mode 100644 index 000000000..47c88582e --- /dev/null +++ b/test/e2e/luks/go.mod @@ -0,0 +1,54 @@ +module ginkgo + +go 1.20 + +require ( + github.com/onsi/ginkgo/v2 v2.11.0 + github.com/onsi/gomega v1.27.10 + k8s.io/apimachinery v0.27.4 + k8s.io/client-go v0.27.4 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/emicklei/go-restful/v3 v3.9.0 // indirect + github.com/go-logr/logr v1.2.4 // indirect + github.com/go-openapi/jsonpointer v0.19.6 // indirect + github.com/go-openapi/jsonreference v0.20.1 // indirect + github.com/go-openapi/swag v0.22.3 // indirect + github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/google/gnostic v0.5.7-v3refs // indirect + github.com/google/go-cmp v0.5.9 // indirect + github.com/google/gofuzz v1.1.0 // indirect + github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect + github.com/google/uuid v1.3.0 // indirect + github.com/imdario/mergo v0.3.6 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/spf13/pflag v1.0.5 // indirect + golang.org/x/net v0.12.0 // indirect + golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect + golang.org/x/sys v0.10.0 // indirect + golang.org/x/term v0.10.0 // indirect + golang.org/x/text v0.11.0 // indirect + golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect + golang.org/x/tools v0.9.3 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/protobuf v1.28.1 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/api v0.27.4 // indirect + k8s.io/klog/v2 v2.90.1 // indirect + k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect + k8s.io/utils v0.0.0-20230209194617-a36077c30491 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect + sigs.k8s.io/yaml v1.3.0 // indirect +) diff --git a/test/e2e/luks/go.sum b/test/e2e/luks/go.sum new file mode 100644 index 000000000..5c982e622 --- /dev/null +++ b/test/e2e/luks/go.sum @@ -0,0 +1,492 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= +github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonreference v0.20.1 h1:FBLnyygC4/IZZr893oiomc9XaghoveYTrLC1F86HID8= +github.com/go-openapi/jsonreference v0.20.1/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= +github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= +github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= +github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= +github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= +github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= +golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b h1:clP8eMhB30EHdc0bd2Twtq6kgU7yl5ub2cQLSdrv1Dg= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c= +golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44= +golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= +golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/api v0.27.4 h1:0pCo/AN9hONazBKlNUdhQymmnfLRbSZjd5H5H3f0bSs= +k8s.io/api v0.27.4/go.mod h1:O3smaaX15NfxjzILfiln1D8Z3+gEYpjEpiNA/1EVK1Y= +k8s.io/apimachinery v0.27.4 h1:CdxflD4AF61yewuid0fLl6bM4a3q04jWel0IlP+aYjs= +k8s.io/apimachinery v0.27.4/go.mod h1:XNfZ6xklnMCOGGFNqXG7bUrQCoR04dh/E7FprV6pb+E= +k8s.io/client-go v0.27.4 h1:vj2YTtSJ6J4KxaC88P4pMPEQECWMY8gqPqsTgUKzvjk= +k8s.io/client-go v0.27.4/go.mod h1:ragcly7lUlN0SRPk5/ZkGnDjPknzb37TICq07WhI6Xc= +k8s.io/klog/v2 v2.90.1 h1:m4bYOKall2MmOiRaR1J+We67Do7vm9KiQVlT96lnHUw= +k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f h1:2kWPakN3i/k81b0gvD5C5FJ2kxm1WrQFanWchyKuqGg= +k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f/go.mod h1:byini6yhqGC14c3ebc/QwanvYwhuMWF6yz2F8uwW8eg= +k8s.io/utils v0.0.0-20230209194617-a36077c30491 h1:r0BAOLElQnnFhE/ApUsg3iHdVYYPBjNSSOMowRZxxsY= +k8s.io/utils v0.0.0-20230209194617-a36077c30491/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/test/e2e/luks/luks_suite_test.go b/test/e2e/luks/luks_suite_test.go new file mode 100644 index 000000000..bf5a83db8 --- /dev/null +++ b/test/e2e/luks/luks_suite_test.go @@ -0,0 +1,13 @@ +package luks_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestLuks(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "LUKS Suite") +} diff --git a/test/e2e/luks/luks_test.go b/test/e2e/luks/luks_test.go new file mode 100644 index 000000000..2e2303ce0 --- /dev/null +++ b/test/e2e/luks/luks_test.go @@ -0,0 +1,242 @@ +package luks_test + +import ( + "context" + "fmt" + "ginkgo/framework" + . "github.com/onsi/ginkgo/v2" + v1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" + "time" +) + +var _ = Describe("LUKS", func() { + f := framework.NewDefaultFramework("luks-e2e-test") + + var ( + client clientset.Interface + podName string + err error + deletePolicy = metav1.DeletePropagationForeground + secretName = "luks-e2e-key" + pvcName = "luks-e2e-pvc" + scName = "luks-e2e-sc" + scReclaimPolicy = v1.PersistentVolumeReclaimDelete + volBindMode = storagev1.VolumeBindingImmediate + ) + + BeforeEach(func() { + client = f.ClientSet + }) + + AfterEach(func() { + podClient := client.CoreV1().Pods(f.Namespace.Name) + + // Delete pod. + if err := podClient.Delete(context.TODO(), podName, metav1.DeleteOptions{ + PropagationPolicy: &deletePolicy, + }); err != nil { + By(fmt.Sprintf("Failed to delete pod, %s", err.Error())) + } else { + By(fmt.Sprintf("Deleted pod: %s", podName)) + } + + // Delete PVC. + if err := client.CoreV1().PersistentVolumeClaims(f.Namespace.Name).Delete(context.TODO(), pvcName, metav1.DeleteOptions{ + PropagationPolicy: &deletePolicy, + }); err != nil { + By(fmt.Sprintf("Failed to delete PVC, %s", err.Error())) + } else { + By(fmt.Sprintf("Deleted PVC: %s", pvcName)) + } + + // Delete secret. + if err := client.CoreV1().Secrets(f.Namespace.Name).Delete(context.TODO(), secretName, metav1.DeleteOptions{ + PropagationPolicy: &deletePolicy, + }); err != nil { + By(fmt.Sprintf("Failed to delete secret, %s", err.Error())) + } else { + By(fmt.Sprintf("Deleted secret: %s", secretName)) + } + + // Delete storage class. + if err := client.StorageV1().StorageClasses().Delete(context.TODO(), scName, metav1.DeleteOptions{ + PropagationPolicy: &deletePolicy, + }); err != nil { + By(fmt.Sprintf("Failed to delete storage class, %s", err.Error())) + } else { + By(fmt.Sprintf("Deleted storage class: %s", scName)) + } + }) + + Describe("LUKS storage", func() { + Context("One pod requesting one LUKS PVC", func() { + It("Creating pod", func() { + // LUKS secret. + secretConfig := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: f.Namespace.Name, + Name: secretName, + }, + Data: map[string][]byte{ + "luksKey": []byte("luks-key"), + }, + Type: v1.SecretTypeOpaque, + } + + _, err = client.CoreV1().Secrets(f.Namespace.Name).Create(context.TODO(), secretConfig, metav1.CreateOptions{}) + By(fmt.Sprintf("Created secret: %s", secretName)) + + // Storage class. + scConfig := &storagev1.StorageClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: scName, + }, + Provisioner: f.DriverData.DriverInfo.Name, + VolumeBindingMode: &volBindMode, + ReclaimPolicy: &scReclaimPolicy, + Parameters: map[string]string{ + "csi.storage.k8s.io/node-stage-secret-name": secretName, + "csi.storage.k8s.io/node-stage-secret-namespace": f.Namespace.Name, + "dobs.csi.digitalocean.com/luks-cipher": "aes-xts-plain64", + "dobs.csi.digitalocean.com/luks-encrypted": "true", + "dobs.csi.digitalocean.com/luks-key-size": "512", + }, + } + + _, err = client.StorageV1().StorageClasses().Create(context.TODO(), scConfig, metav1.CreateOptions{}) + if err != nil { + Fail(err.Error()) + } + + By(fmt.Sprintf("Created storage class: %s", scName)) + + // PVC. + volumeSize := "1Gi" + if f.DriverData.DriverInfo.SupportedSizeRange.Min != "" { + volumeSize = f.DriverData.DriverInfo.SupportedSizeRange.Min + } + + pvcConfig := &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: pvcName, + }, + Spec: v1.PersistentVolumeClaimSpec{ + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceStorage: resource.MustParse(volumeSize), + }, + }, + AccessModes: []v1.PersistentVolumeAccessMode{ + "ReadWriteOnce", + }, + StorageClassName: &scName, + }, + } + + pvc, err := client.CoreV1().PersistentVolumeClaims(f.Namespace.Name).Create(context.TODO(), pvcConfig, metav1.CreateOptions{}) + if err != nil { + Fail(err.Error()) + } + + By(fmt.Sprintf("Created PVC: %s", pvcName)) + + // Wait for PVC 30s. + count := 0 + for pvc.Spec.VolumeName == "" && count < 30 { + By(fmt.Sprintf("Wait for PVC...[%ds]", count)) + time.Sleep(1 * time.Second) + pvc, err = client.CoreV1().PersistentVolumeClaims(f.Namespace.Name).Get(context.TODO(), pvcName, metav1.GetOptions{}) + if err != nil { + Fail(err.Error()) + } + count++ + } + + if pvc.Spec.VolumeName == "" { + Fail(fmt.Sprintf("PVC %s can't be provisioned", pvc.Name)) + } + + // Pod. + // @TODO: leave only one once docker image is ready. + podCmd := "df -T /data | grep /dev/mapper/" + pvc.Spec.VolumeName + //podCmd := "df -T /data | grep /dev/disk/by-id/scsi-0DO_Volume_" + pvc.Spec.VolumeName + + podConfig := &v1.Pod{ + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "pvc-luks-tester-", + Namespace: f.Namespace.Name, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "pod", + Image: "busybox:latest", + Command: []string{"/bin/sh", "-c", podCmd}, + VolumeMounts: []v1.VolumeMount{ + { + MountPath: "/data", + Name: "data", + }, + }, + }, + }, + Volumes: []v1.Volume{ + { + Name: "data", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: pvcName, + ReadOnly: false, + }, + }, + }, + }, + RestartPolicy: v1.RestartPolicyNever, + }, + } + + podClient := client.CoreV1().Pods(f.Namespace.Name) + + pod, err := podClient.Create(context.TODO(), podConfig, metav1.CreateOptions{}) + if err != nil { + Fail(err.Error()) + } + + podName = pod.GetObjectMeta().GetName() + + By(fmt.Sprintf("Created pod: %q", podName)) + + pod, err = podClient.Get(context.TODO(), podName, metav1.GetOptions{}) + if err != nil { + Fail(err.Error()) + } + + // Wait for pod 120s. + count = 0 + for pod.Status.Phase == v1.PodPending && count < 24 { + By(fmt.Sprintf("Wait for pod...[%ds]", count*5)) + time.Sleep(5 * time.Second) + pod, err = podClient.Get(context.TODO(), podName, metav1.GetOptions{}) + if err != nil { + Fail(err.Error()) + } + count++ + } + + if pod.Status.Phase == v1.PodFailed { + Fail(fmt.Sprintf("pod %q failed with status: %+v", podName, pod.Status)) + } + + By("Luks volume was created successfully! Cleaning up.") + }) + }) + }) +}) diff --git a/test/e2e/run-versioned-e2e-tests.sh b/test/e2e/run-versioned-e2e-tests.sh index 9d487600b..63e1272d3 100755 --- a/test/e2e/run-versioned-e2e-tests.sh +++ b/test/e2e/run-versioned-e2e-tests.sh @@ -55,6 +55,9 @@ if [[ "${FOCUS:-}" ]]; then focus=".*${FOCUS}" fi +echo 'Running LUKS tests' +"ginkgo-${KUBE_VER}" -v /e2e.luks.test -- "-storage.testdriver=${TD_FILE}" + if [[ "${SKIP_PARALLEL_TESTS:-}" ]]; then echo 'Skipping parallel tests' else