diff --git a/.github/workflows/unit-quickstart-e2e-test.yaml b/.github/workflows/unit-quickstart-e2e-test.yaml index 84e5cb101..3a30b73b6 100644 --- a/.github/workflows/unit-quickstart-e2e-test.yaml +++ b/.github/workflows/unit-quickstart-e2e-test.yaml @@ -38,6 +38,10 @@ jobs: runs-on: [self-hosted, linux] needs: [unit] steps: + - name: ๐Ÿฉน Safe config for ๐Ÿ’ฉ DNS + run: | + sed 's/options /options use-vc single-request attempts:5 /' < /etc/resolv.conf > /etc/resolv.conf.new + cat /etc/resolv.conf.new > /etc/resolv.conf - name: โฌ‡๏ธ Checkout repository uses: actions/checkout@v4 - name: โฌ‡๏ธ Install kubectl @@ -52,7 +56,8 @@ jobs: kustomize-version: v4.5.7 - name: ๐Ÿ”Ž Check IP id: ip - run: echo "ip=`curl -s https://api.ipify.org`" | tee $GITHUB_OUTPUT + run: | + echo "ip=`curl -s https://api.ipify.org`" | tee $GITHUB_OUTPUT - name: ๐Ÿ” Set ak/sk name based on runner region run: .github/scripts/runneraksk.sh - name: ๐Ÿงน Frieza @@ -87,10 +92,18 @@ jobs: sudo apt install -y docker-buildx-plugin make docker-buildx make docker-push - docker image prune -a -f env: IMG: ${{ vars.REGISTRY }}/outscale/cluster-api-outscale-controllers:${{ github.sha }} DOCKER_BUILDKIT: 1 + - name: ๐Ÿ”Ž Preloader snapshot + id: preloader + uses: outscale/k8s-image-preloader/github_actions/preloader_snapshot@main + with: + KUBECONFIG: ${{ steps.management.outputs.KUBECONFIG }} + OSC_ACCESS_KEY: ${{ secrets[env.OSC_ACCESS_KEY_NAME] }} + OSC_SECRET_KEY: ${{ secrets[env.OSC_SECRET_KEY_NAME] }} + OSC_REGION: ${{ env.OSC_REGION }} + CSI: true - name: ๐Ÿ” Create CAPOSC ns and credentials run: make credential shell: bash @@ -112,10 +125,11 @@ jobs: KUBECONFIG: "${{ github.workspace }}/${{ steps.management.outputs.KUBECONFIG }}" CCM_OSC_ACCESS_KEY: ${{ secrets[env.OSC_ACCESS_KEY_NAME] }} CCM_OSC_SECRET_KEY: ${{ secrets[env.OSC_SECRET_KEY_NAME] }} - CCM_OSC_REGION: $${{ env.OSC_REGION }} + CCM_OSC_REGION: ${{ env.OSC_REGION }} IMG: ${{ vars.REGISTRY }}/outscale/cluster-api-outscale-controllers:${{ github.sha }} IMG_UPGRADE_FROM: ${{ vars.IMG_UPGRADE_FROM }} IMG_UPGRADE_TO: ${{ vars.IMG_UPGRADE_TO }} + PRELOAD_SNAPSHOT_ID: ${{ steps.preloader.outputs.SNAPSHOT_ID }} - name: ๐Ÿ“ Get CAPOSC state/logs run: | echo "**** pod state" diff --git a/api/v1beta1/oscmachine_validation.go b/api/v1beta1/oscmachine_validation.go index e21f556ff..9f48f4053 100644 --- a/api/v1beta1/oscmachine_validation.go +++ b/api/v1beta1/oscmachine_validation.go @@ -76,7 +76,7 @@ func ValidateIops(path *field.Path, iops, size int32) *field.Error { return nil case iops > maxIops || iops < minIops: return field.Invalid(path, iops, fmt.Sprintf("iops must be between %d and %d", minIops, maxIops)) - case iops/size > 300: + case size > 0 && iops/size > 300: return field.Invalid(path, iops, "iops/size should be lower than 300") default: return nil diff --git a/api/v1beta1/types.go b/api/v1beta1/types.go index be6487630..17d51c74b 100644 --- a/api/v1beta1/types.go +++ b/api/v1beta1/types.go @@ -523,14 +523,13 @@ type OscImage struct { type OscVolume struct { // The volume name. Name string `json:"name,omitempty"` - // The volume device (/dev/sdX) + // The volume device (/dev/xvdX) // +kubebuilder:validation:Required Device string `json:"device"` // The volume iops (io1 volumes only) Iops int32 `json:"iops,omitempty"` // The volume size in gibibytes (GiB) - // +kubebuilder:validation:Required - Size int32 `json:"size"` + Size int32 `json:"size,omitempty"` // (unused) SubregionName string `json:"subregionName,omitempty"` // The volume type (io1, gp2 or standard) diff --git a/capm.yaml b/capm.yaml index 2f5a9ce8c..7a30b4179 100644 --- a/capm.yaml +++ b/capm.yaml @@ -10,7 +10,7 @@ kind: CustomResourceDefinition metadata: annotations: cert-manager.io/inject-ca-from: cluster-api-provider-outscale-system/cluster-api-provider-outscale-serving-cert - controller-gen.kubebuilder.io/version: v0.19.1-0.20250924093817-1d67470bef32 + controller-gen.kubebuilder.io/version: v0.19.1-0.20251023132335-bf7d6b742e6a labels: cluster.x-k8s.io/v1alpha3: v1alpha3 cluster.x-k8s.io/v1beta1: v1beta1 @@ -879,7 +879,7 @@ kind: CustomResourceDefinition metadata: annotations: cert-manager.io/inject-ca-from: cluster-api-provider-outscale-system/cluster-api-provider-outscale-serving-cert - controller-gen.kubebuilder.io/version: v0.19.1-0.20250924093817-1d67470bef32 + controller-gen.kubebuilder.io/version: v0.19.1-0.20251023132335-bf7d6b742e6a labels: cluster.x-k8s.io/v1alpha3: v1alpha3 cluster.x-k8s.io/v1beta1: v1beta1 @@ -1588,7 +1588,7 @@ kind: CustomResourceDefinition metadata: annotations: cert-manager.io/inject-ca-from: cluster-api-provider-outscale-system/cluster-api-provider-outscale-serving-cert - controller-gen.kubebuilder.io/version: v0.19.1-0.20250924093817-1d67470bef32 + controller-gen.kubebuilder.io/version: v0.19.1-0.20251023132335-bf7d6b742e6a labels: cluster.x-k8s.io/v1alpha3: v1alpha3 cluster.x-k8s.io/v1beta1: v1beta1 @@ -1789,7 +1789,7 @@ spec: items: properties: device: - description: The volume device (/dev/sdX) + description: The volume device (/dev/xvdX) type: string fromSnapshot: description: The id of a snapshot to use as a volume source. @@ -1816,7 +1816,6 @@ spec: type: string required: - device - - size type: object type: array type: object @@ -1994,7 +1993,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.19.1-0.20250924093817-1d67470bef32 + controller-gen.kubebuilder.io/version: v0.19.1-0.20251023132335-bf7d6b742e6a labels: cluster.x-k8s.io/v1alpha3: v1alpha3 cluster.x-k8s.io/v1beta1: v1beta1 @@ -2226,7 +2225,7 @@ spec: items: properties: device: - description: The volume device (/dev/sdX) + description: The volume device (/dev/xvdX) type: string fromSnapshot: description: The id of a snapshot to use as a volume @@ -2254,7 +2253,6 @@ spec: type: string required: - device - - size type: object type: array type: object diff --git a/cloud/services/compute/vm.go b/cloud/services/compute/vm.go index 0e254dbec..a435eabc3 100644 --- a/cloud/services/compute/vm.go +++ b/cloud/services/compute/vm.go @@ -69,11 +69,13 @@ func (s *Service) CreateVm(ctx context.Context, for _, vol := range volumes { bsuVol := osc.BlockDeviceMappingVmCreation{ Bsu: &osc.BsuToCreate{ - VolumeSize: &vol.Size, VolumeType: &vol.VolumeType, }, DeviceName: &vol.Device, } + if vol.Size > 0 { + bsuVol.Bsu.VolumeSize = &vol.Size + } if vol.VolumeType == "io1" { bsuVol.Bsu.Iops = &vol.Iops } diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_oscclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_oscclusters.yaml index b3a034c1d..aed884f08 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_oscclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_oscclusters.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.19.1-0.20250924093817-1d67470bef32 + controller-gen.kubebuilder.io/version: v0.19.1-0.20251023132335-bf7d6b742e6a name: oscclusters.infrastructure.cluster.x-k8s.io spec: group: infrastructure.cluster.x-k8s.io diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_oscclustertemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_oscclustertemplates.yaml index 832d4a91f..743b42c44 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_oscclustertemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_oscclustertemplates.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.19.1-0.20250924093817-1d67470bef32 + controller-gen.kubebuilder.io/version: v0.19.1-0.20251023132335-bf7d6b742e6a name: oscclustertemplates.infrastructure.cluster.x-k8s.io spec: group: infrastructure.cluster.x-k8s.io diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_oscmachines.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_oscmachines.yaml index 5c2b67f9f..c447604a6 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_oscmachines.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_oscmachines.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.19.1-0.20250924093817-1d67470bef32 + controller-gen.kubebuilder.io/version: v0.19.1-0.20251023132335-bf7d6b742e6a name: oscmachines.infrastructure.cluster.x-k8s.io spec: group: infrastructure.cluster.x-k8s.io @@ -191,7 +191,7 @@ spec: items: properties: device: - description: The volume device (/dev/sdX) + description: The volume device (/dev/xvdX) type: string fromSnapshot: description: The id of a snapshot to use as a volume source. @@ -218,7 +218,6 @@ spec: type: string required: - device - - size type: object type: array type: object diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_oscmachinetemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_oscmachinetemplates.yaml index 8ba0bb0af..199c4fb84 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_oscmachinetemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_oscmachinetemplates.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.19.1-0.20250924093817-1d67470bef32 + controller-gen.kubebuilder.io/version: v0.19.1-0.20251023132335-bf7d6b742e6a name: oscmachinetemplates.infrastructure.cluster.x-k8s.io spec: group: infrastructure.cluster.x-k8s.io @@ -232,7 +232,7 @@ spec: items: properties: device: - description: The volume device (/dev/sdX) + description: The volume device (/dev/xvdX) type: string fromSnapshot: description: The id of a snapshot to use as a volume @@ -260,7 +260,6 @@ spec: type: string required: - device - - size type: object type: array type: object diff --git a/controllers/osccluster_netpeering.go b/controllers/osccluster_netpeering.go index 23598a6f8..41edaa11c 100644 --- a/controllers/osccluster_netpeering.go +++ b/controllers/osccluster_netpeering.go @@ -74,7 +74,7 @@ func (r *OscClusterReconciler) reconcileNetPeering(ctx context.Context, clusterS return reconcile.Result{}, fmt.Errorf("cannot get mgmt credentials: %w", err) } mgmtSvc := r.Cloud.NetPeering(mgmt) - log.V(2).Info("Accepting netPeering") + log.V(2).Info("Accepting netPeering", "netPeeringId", np.GetNetPeeringId()) err = mgmtSvc.AcceptNetPeering(ctx, np.GetNetPeeringId()) if err != nil { return reconcile.Result{}, fmt.Errorf("cannot accept netPeering: %w", err) @@ -109,7 +109,7 @@ func (r *OscClusterReconciler) reconcileDeleteNetPeering(ctx context.Context, cl if np.State.GetName() != "pending-acceptance" && np.State.GetName() != "active" { continue } - log.V(2).Info("Deleting netPeering", "subnetId", np.GetNetPeeringId()) + log.V(2).Info("Deleting netPeering", "netPeeringId", np.GetNetPeeringId()) err = svc.DeleteNetPeering(ctx, np.GetNetPeeringId()) if err != nil { return reconcile.Result{}, fmt.Errorf("cannot delete netPeering: %w", err) diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index 7311c2316..051af7150 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -8,7 +8,7 @@ - [Multi AZ clusters](./topics/config-multiaz.md) - [Securing cluster access](./topics/config-security.md) - [Air-gapped clusters](./topics/config-airgap.md) - - [Omi](./topics/omi.md) + - [Preloading images](./topics/preload.md) - [Troubleshooting](./topics/troubleshooting.md) - [Cluster-Autoscaler](./topics/cluster-autoscaler.md) - [How to upgrade](./topics/upgrade-cluster.md) diff --git a/docs/src/topics/config-airgap.md b/docs/src/topics/config-airgap.md index 5e5068886..89e96990a 100644 --- a/docs/src/topics/config-airgap.md +++ b/docs/src/topics/config-airgap.md @@ -65,9 +65,9 @@ network: TBD -## Pre-loaded images +## Preloaded images -TBD +See [Preloading images](preload.md). ## Outscale API access diff --git a/docs/src/topics/config-nodes.md b/docs/src/topics/config-nodes.md index 795b62c7b..57f5c3bef 100644 --- a/docs/src/topics/config-nodes.md +++ b/docs/src/topics/config-nodes.md @@ -28,28 +28,40 @@ In your `OscMachineTemplate` spec, add the list of required volumes: vm: [...] volumes: - name: data - device: /dev/sdb + device: /dev/xvdb iops: 500 size: 50 volumeType: io1 - name: logs - device: /dev/sdc + device: /dev/xvdc iops: 500 size: 10 volumeType: io1 [...] ``` +> New volumes are unformatted. You will need to partition, format and mount them during cloud-init. + A snapshot can be used as a volume source: ```yaml - name: images - device: /dev/sdd - size: 5 - volumeType: gp2 + device: /dev/xvdd fromSnapshot: snap-xxx ``` -> Volumes not created from a snapshot are unformatted. You will need to format newly created volumes during cloud-init. +> By default, the size of the snapshot is used for the new volume. + +You will need to mount snapshot-based volumes during cloud-init. In `KubeadmConfigTemplate`/`KubeadmControlPlane` resources: +```yaml + spec: + joinConfiguration: + [...] + mounts: + - - xvdd + - /mnt/example + - ext4 + - auto,exec,ro +``` ## OscMachineTemplate configuration @@ -88,13 +100,14 @@ Outscale Open-Source images are published on the `eu-west-2`, `us-east-2` and `c `volumes` is a list of additional volumes. -| Name | Required | Description -| --- | --- | --- -| `name` | false | The volume name -| `device` | true | The volume device (`/dev/sdX`) -| `size` | true | The volume size -| `volumeType` | true | The volume type (`io1`, `gp2` or `standard`) -| `iops` | false | The volume iops (only for the `io1` type) +| Name | Default | Required | Description +| --- | --- | --- | --- +| `name` | n/a | false | The volume name +| `device` | n/a | true | The volume device (`/dev/xvdX`) +| `size` | n/a | false | The volume size (required unless the source is a snapshot; if not set, the snapshot size is used) +| `volumeType` | `standard` | false | The volume type (`io1`, `gp2` or `standard`) +| `iops` | n/a | false | The volume iops (only for the `io1` type) +| `fromSnapshot` | n/a | false | The ID of the source snapshot ### Subnet & security group selection diff --git a/docs/src/topics/preload.md b/docs/src/topics/preload.md new file mode 100644 index 000000000..fb4b4846c --- /dev/null +++ b/docs/src/topics/preload.md @@ -0,0 +1,68 @@ +# Preloading images + +Preloading images lets you add container images to the local containerd cache at boot. + +It enables: +* faster startup times (no need to download images from the internet), +* building clusters without internet access. + +## Creating a snapshot + +You need a seed cluster with access to the internet and all apps required on the target cluster installed. + +The seed cluster must have CSI installed, with snapshotting enabled. + +You can use the [following example](k8s-image-preloader-example) based on [k8s-image-preloader](8s-image-preloader) to create the snapshot. + +You may want to adapt the spec if you need more than 2GB of storage for your images or if you want to change the `deletionPolicy` of the `VolumeSnapshotClass`. + +[k8s-image-preloader](8s-image-preloader): +* mounts a `PersistentVolumeClaim` (PVC), +* searches for images in the local containerd cache, +* searches `Pods` and `CronJobs`, +* pulls images from the remote registry (even if already present in the cache, as containerd may prune some layers from cached images), +* exports the images to the PVC, +* creates a `VolumeSnapshot` from the PVC. + +You can get the handle (IaaS snapshot ID) by querying the `VolumeSnapshotContent` status: +```bash +kubectl get vsc -o custom-columns=NAME:.metadata.name,SNAPSHOT:.spec.volumeSnapshotRef.name,NAMESPACE:.spec.volumeSnapshotRef.namespace,HANDLE:.status.snapshotHandle +``` + +## Importing images + +You will need to mount a volume based on the previously generated snapshot on target nodes: + +In `OscMachineTemplate` resources: +```yaml + vm: + [...] + volumes: + - device: /dev/xvdb + fromSnapshot: snap-xxx +``` + +Then import all images during cloud-init: + +In `KubeadmConfigTemplate`/`KubeadmControlPlane` resources: +```yaml + spec: + joinConfiguration: + [...] + mounts: + - - xvdb + - /preload + - ext4 + - auto,exec,ro + preKubeadmCommands: + - /preload/restore.sh +``` + +## Notes + +k8s-image-preloader only works with containerd, and uses the `ctr` tool installed on the node, but a similar process can be built for CRI-O using [skopeo](skopeo). + + +[k8s-image-preloader-example]: https://raw.githubusercontent.com/outscale/k8s-image-preloader/refs/heads/main/examples/example.yaml +[skopeo]: https://github.com/containers/skopeo +[k8s-image-preloader]: https://github.com/outscale/k8s-image-preloader diff --git a/github_actions/deploy_cluster/cleanup.sh b/github_actions/deploy_cluster/cleanup.sh index 49903fbfb..2e1b025e4 100755 --- a/github_actions/deploy_cluster/cleanup.sh +++ b/github_actions/deploy_cluster/cleanup.sh @@ -16,4 +16,5 @@ kubeconfig=`/.venv/bin/oks-cli cluster kubeconfig --cluster-name $cluster_name - export KUBECONFIG=$kubeconfig # do not wait if stuck, frieza will purge everything -kubectl delete ns $OSC_CLUSTER_NAME --timeout 2m || /bin/true \ No newline at end of file +# as freeza cannot delete netpeerings, be careful +kubectl delete ns $OSC_CLUSTER_NAME --timeout 10m || /bin/true \ No newline at end of file diff --git a/github_actions/deploy_cluster/main.sh b/github_actions/deploy_cluster/main.sh index 6f78a811f..92fc81a61 100755 --- a/github_actions/deploy_cluster/main.sh +++ b/github_actions/deploy_cluster/main.sh @@ -26,6 +26,9 @@ export OSC_REGION=`echo $OSC_AKSK|cut -d% -f 3` export OSC_IMAGE_NAME=`echo $OSC_IMAGE_NAME_ACCOUNT_ID|cut -d% -f 1` export OSC_IMAGE_ACCOUNT_ID=`echo $OSC_IMAGE_NAME_ACCOUNT_ID|cut -d% -f 2` +# F***g DNS +echo "options use-vc single-request attempts:5" >> /etc/resolv.conf + cluster_name=`echo $RUNNER_NAME|tr '[:upper:]' '[:lower:]'|sed -r 's/-[a-z0-9]+$//'|cut -c1-40|sed -r 's/[^a-z0-9-]+/-/g'` # OKS /.venv/bin/oks-cli profile add --profile-name "default" --access-key $OKS_ACCESS_KEY --secret-key $OKS_SECRET_KEY --region $OKS_REGION diff --git a/test/e2e/config/outscale.yaml b/test/e2e/config/outscale.yaml index 32a6bb08f..2aee0832f 100644 --- a/test/e2e/config/outscale.yaml +++ b/test/e2e/config/outscale.yaml @@ -5,8 +5,8 @@ providers: - name: cluster-api type: CoreProvider versions: - - name: v1.8.10 - value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.8.10/core-components.yaml + - name: v1.10.7 + value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.10.7/core-components.yaml type: "url" contract: v1beta1 files: @@ -19,8 +19,8 @@ providers: - name: kubeadm type: BootstrapProvider versions: - - name: v1.8.10 - value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.8.10/bootstrap-components.yaml + - name: v1.10.7 + value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.10.7/bootstrap-components.yaml type: "url" contract: v1beta1 files: @@ -33,8 +33,8 @@ providers: - name: kubeadm type: ControlPlaneProvider versions: - - name: v1.8.10 - value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.8.10/control-plane-components.yaml + - name: v1.10.7 + value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.10.7/control-plane-components.yaml type: "url" contract: v1beta1 files: @@ -107,6 +107,7 @@ variables: CONFORMANCE_CONTROL_PLANE_MACHINE_COUNT: 3 KUBETEST_CONFIGURATION: "${KUBETEST_CONF_PATH:=./data/kubetest/conformance.yaml}" NODE_DRAIN_TIMEOUT: "60s" + PRELOAD_SNAPSHOT_ID: "" intervals: default/wait-cluster: ["15m", "10s"] default/wait-control-plane: ["30m", "10s"] diff --git a/test/e2e/data/ccm/ccm.yaml.template b/test/e2e/data/ccm/ccm.yaml.template index 55ffd6620..d5ae0ea55 100644 --- a/test/e2e/data/ccm/ccm.yaml.template +++ b/test/e2e/data/ccm/ccm.yaml.template @@ -5,14 +5,8 @@ metadata: name: osc-secret namespace: kube-system stringData: - key_id: ${CCM_OSC_ACCESS_KEY} - access_key: ${CCM_OSC_SECRET_KEY} - aws_default_region: ${CCM_OSC_REGION} - aws_availability_zones: AWS_AVAILABILITY_ZONES - osc_account_id: OSC_ACCOUNT_ID - osc_account_iam: OSC_ACCOUNT_IAM - osc_user_id: OSC_USER_ID - osc_arn: OSC_ARN + access_key: ${CCM_OSC_ACCESS_KEY} + secret_key: ${CCM_OSC_SECRET_KEY} --- apiVersion: v1 kind: ServiceAccount @@ -155,7 +149,7 @@ spec: serviceAccountName: cloud-controller-manager containers: - name: osc-cloud-controller-manager - image: outscale/cloud-provider-osc:v0.2.7 + image: outscale/cloud-provider-osc:v1.0.1 imagePullPolicy: IfNotPresent command: - /bin/osc-cloud-controller-manager @@ -163,65 +157,17 @@ spec: - --cloud-provider=osc - -v=5 env: - - name: OSC_ACCOUNT_ID - valueFrom: - secretKeyRef: - name: "osc-secret" - key: osc_account_id - optional: false - - name: OSC_ACCOUNT_IAM - valueFrom: - secretKeyRef: - name: "osc-secret" - key: osc_account_iam - optional: false - - name: OSC_USER_ID - valueFrom: - secretKeyRef: - name: "osc-secret" - key: osc_user_id - optional: false - - name: OSC_ARN - valueFrom: - secretKeyRef: - name: "osc-secret" - key: osc_arn - optional: false - - name: AWS_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - name: "osc-secret" - key: key_id - optional: false - - name: AWS_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - name: "osc-secret" - key: access_key - optional: false - - name: AWS_DEFAULT_REGION - valueFrom: - secretKeyRef: - name: "osc-secret" - key: aws_default_region - optional: false - - name: AWS_AVAILABILITY_ZONES - valueFrom: - secretKeyRef: - name: "osc-secret" - key: aws_availability_zones - optional: false - name: OSC_SECRET_KEY valueFrom: secretKeyRef: name: "osc-secret" - key: access_key + key: secret_key optional: false - name: OSC_ACCESS_KEY valueFrom: secretKeyRef: name: "osc-secret" - key: key_id + key: access_key optional: false hostNetwork: true tolerations: diff --git a/test/e2e/data/infrastructure-outscale/cluster-template-airgap.yaml b/test/e2e/data/infrastructure-outscale/cluster-template-airgap.yaml index ec5010c2b..980a41df7 100644 --- a/test/e2e/data/infrastructure-outscale/cluster-template-airgap.yaml +++ b/test/e2e/data/infrastructure-outscale/cluster-template-airgap.yaml @@ -75,6 +75,9 @@ spec: keypairName: "${OSC_KEYPAIR_NAME}" subregionName: ${OSC_REGION}a vmType: "${OSC_VM_TYPE}" + volumes: + - device: /dev/xvdb + fromSnapshot: ${PRELOAD_SNAPSHOT_ID} --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: OscMachineTemplate @@ -96,6 +99,9 @@ spec: keypairName: "${OSC_KEYPAIR_NAME}" role: controlplane vmType: "${OSC_VM_TYPE}" + volumes: + - device: /dev/xvdb + fromSnapshot: ${PRELOAD_SNAPSHOT_ID} --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KubeadmConfigTemplate @@ -104,12 +110,20 @@ metadata: spec: template: spec: + verbosity: 5 joinConfiguration: nodeRegistration: name: "{{ ds.meta_data.local_hostname }}" kubeletExtraArgs: cloud-provider: external provider-id: aws:///'{{ ds.meta_data.placement.availability_zone }}'/'{{ ds.meta_data.instance_id }}' + mounts: + - - xvdb + - /preload + - ext4 + - auto,exec,ro + preKubeadmCommands: + - /preload/restore.sh --- kind: KubeadmControlPlane apiVersion: controlplane.cluster.x-k8s.io/v1beta1 @@ -123,18 +137,25 @@ spec: apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 name: "${CLUSTER_NAME}-control-plane" kubeadmConfigSpec: + verbosity: 5 initConfiguration: nodeRegistration: kubeletExtraArgs: cloud-provider: external provider-id: aws:///'{{ ds.meta_data.placement.availability_zone }}'/'{{ ds.meta_data.instance_id }}' name: '{{ ds.meta_data.local_hostname }}' - files: joinConfiguration: nodeRegistration: kubeletExtraArgs: cloud-provider: external provider-id: aws:///'{{ ds.meta_data.placement.availability_zone }}'/'{{ ds.meta_data.instance_id }}' + mounts: + - - xvdb + - /preload + - ext4 + - "auto,exec,ro,nodev" + preKubeadmCommands: + - /preload/restore.sh version: "${KUBERNETES_VERSION}" --- apiVersion: v1 diff --git a/test/e2e/data/infrastructure-outscale/infrastructure-components.yaml b/test/e2e/data/infrastructure-outscale/infrastructure-components.yaml deleted file mode 100644 index 02ddd66aa..000000000 --- a/test/e2e/data/infrastructure-outscale/infrastructure-components.yaml +++ /dev/null @@ -1,2179 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - labels: - control-plane: controller-manager - name: cluster-api-provider-outscale-system ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - cert-manager.io/inject-ca-from: cluster-api-provider-outscale-system/cluster-api-provider-outscale-serving-cert - controller-gen.kubebuilder.io/version: v0.17.3-0.20250416153108-23dd517d714e - labels: - cluster.x-k8s.io/v1alpha3: v1alpha3 - cluster.x-k8s.io/v1beta1: v1beta1 - name: oscclusters.infrastructure.cluster.x-k8s.io -spec: - group: infrastructure.cluster.x-k8s.io - names: - categories: - - cluster-api - kind: OscCluster - listKind: OscClusterList - plural: oscclusters - singular: osccluster - scope: Namespaced - versions: - - name: v1beta1 - schema: - openAPIV3Schema: - description: OscCluster is the Schema for the oscclusters API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: OscClusterSpec defines the desired state of OscCluster - properties: - controlPlaneEndpoint: - description: APIEndpoint represents a reachable Kubernetes API endpoint. - properties: - host: - description: The hostname on which the API server is serving. - type: string - port: - description: The port on which the API server is serving. - format: int32 - type: integer - required: - - host - - port - type: object - network: - properties: - allowFromIPRanges: - description: The list of IP ranges (in CIDR notation) to restrict - bastion/Kubernetes API access to. - items: - type: string - type: array - bastion: - description: The bastion configuration - properties: - PublicIpId: - description: The ID of an existing public IP to use for this - VM. - type: string - clusterName: - type: string - deviceName: - type: string - enable: - type: boolean - imageAccountId: - type: string - imageId: - type: string - imageName: - type: string - keypairName: - type: string - name: - type: string - privateIps: - items: - properties: - name: - type: string - privateIp: - type: string - type: object - type: array - publicIpName: - description: unused - type: string - resourceId: - type: string - rootDisk: - properties: - rootDiskIops: - format: int32 - type: integer - rootDiskSize: - format: int32 - type: integer - rootDiskType: - type: string - type: object - securityGroupNames: - items: - properties: - name: - type: string - type: object - type: array - subnetName: - type: string - subregionName: - type: string - vmType: - type: string - type: object - clusterName: - description: The name of the cluster (unused) - type: string - controlPlaneSubnets: - description: List of subnet to spread controlPlane nodes - items: - type: string - type: array - extraSecurityGroupRule: - description: Add SecurityGroup Rule after the cluster is created - (unused) - type: boolean - image: - description: The image configuration (unused) - properties: - accountId: - type: string - name: - type: string - resourceId: - type: string - type: object - internetService: - description: The Internet Service configuration - properties: - clusterName: - description: the name of the cluster (unused) - type: string - name: - description: The name of the Internet service - type: string - resourceId: - description: the Internet Service resource id (unused) - type: string - type: object - loadBalancer: - description: The Load Balancer configuration - properties: - clusterName: - description: unused - type: string - healthCheck: - description: The healthCheck configuration of the Load Balancer - properties: - checkinterval: - description: the time in second between two pings - format: int32 - type: integer - healthythreshold: - description: the consecutive number of pings which are - successful to consider the vm healthy - format: int32 - type: integer - port: - description: the HealthCheck port number - format: int32 - type: integer - protocol: - description: The HealthCheck protocol ('HTTP'|'TCP') - type: string - timeout: - description: the Timeout to consider VM unhealthy - format: int32 - type: integer - unhealthythreshold: - description: the consecutive number of pings which are - failed to consider the vm unhealthy - format: int32 - type: integer - type: object - listener: - description: The Listener configuration of the loadBalancer - properties: - backendport: - description: The port on which the backend VMs will listen - format: int32 - type: integer - backendprotocol: - description: The protocol ('HTTP'|'TCP') to route the - traffic to the backend vm - type: string - loadbalancerport: - description: The port on which the loadbalancer will listen - format: int32 - type: integer - loadbalancerprotocol: - description: the routing protocol ('HTTP'|'TCP') - type: string - type: object - loadbalancername: - description: The Load Balancer unique name - type: string - loadbalancertype: - description: The Load Balancer type (internet-facing or internal) - type: string - securitygroupname: - description: The security group name for the load-balancer - type: string - subnetname: - description: The subnet name where to add the load balancer. - type: string - type: object - natService: - description: The Nat Service configuration - properties: - clusterName: - description: The name of the cluster (unused) - type: string - name: - description: The name of the Nat Service - type: string - publicipname: - description: The Public Ip name (unused) - type: string - resourceId: - description: The resource id (unused) - type: string - subnetname: - description: The name of the Subnet to which the Nat Service - will be attached - type: string - subregionName: - description: The name of the Subregion to which the Nat Service - will be attached, unless a subnet has been defined (unused) - type: string - type: object - natServices: - description: The Nat Services configuration - items: - properties: - clusterName: - description: The name of the cluster (unused) - type: string - name: - description: The name of the Nat Service - type: string - publicipname: - description: The Public Ip name (unused) - type: string - resourceId: - description: The resource id (unused) - type: string - subnetname: - description: The name of the Subnet to which the Nat Service - will be attached - type: string - subregionName: - description: The name of the Subregion to which the Nat - Service will be attached, unless a subnet has been defined - (unused) - type: string - type: object - type: array - net: - description: The Net configuration - properties: - clusterName: - description: the name of the cluster (unused) - type: string - ipRange: - description: the ip range in CIDR notation of the Net - type: string - name: - description: the network name - type: string - resourceId: - description: The Id of the Net to reuise (if useExisting is - set) - type: string - useExisting: - description: Reuse an existing network defined by resourceId - ? - type: boolean - type: object - publicIps: - description: The Public Ip configuration - items: - properties: - clusterName: - type: string - name: - description: The tag name associate with the Public Ip - type: string - resourceId: - description: The Public Ip Id response - type: string - type: object - type: array - routeTables: - description: The Route Table configuration - items: - properties: - name: - description: The tag name associate with the Route Table - type: string - resourceId: - description: The Route Table Id response - type: string - role: - description: The role for this route table - type: string - routes: - description: The Route configuration - items: - properties: - destination: - description: the destination match Ip range with CIDR - notation - type: string - name: - description: The tag name associate with the Route - type: string - resourceId: - description: The Route Id response - type: string - targetName: - description: The tag name associate with the target - resource type - type: string - targetType: - description: The target resource type which can be - Internet Service (gateway) or Nat Service (nat-service) - type: string - type: object - type: array - subnets: - description: The subnet tag name associate with a Subnet - items: - type: string - type: array - subregionName: - description: The subregion for this route table - type: string - type: object - type: array - securityGroups: - items: - properties: - description: - description: The description of the security group - type: string - name: - description: The name of the security group - type: string - resourceId: - description: When reusing network, the id of an existing - securityGroup to use. - type: string - roles: - description: The roles the securityGroup applies to. - items: - type: string - type: array - securityGroupRules: - description: The list of rules for this securityGroup. - items: - properties: - flow: - description: The flow of the security group (inbound - or outbound) - type: string - fromPortRange: - description: The beginning of the port range - format: int32 - type: integer - ipProtocol: - description: The ip protocol name (tcp, udp, icmp - or -1) - type: string - ipRange: - description: The ip range of the security group rule - (deprecated) - type: string - ipRanges: - description: The list of ip ranges of the security - group rule - items: - type: string - type: array - name: - description: The tag name associate with the security - group - type: string - resourceId: - description: The security group rule id - type: string - toPortRange: - description: The end of the port range - format: int32 - type: integer - type: object - type: array - tag: - type: string - type: object - type: array - subnets: - description: The Subnet configuration - items: - properties: - ipSubnetRange: - description: the Ip range in CIDR notation of the Subnet - type: string - name: - description: The name of the Subnet - type: string - resourceId: - description: The id of the Subnet to reuse (if net.useExisting - is set) - type: string - roles: - description: The role of the Subnet (controlplane, worker, - loadbalancer, bastion or nat) - items: - type: string - type: array - subregionName: - description: The subregion name of the Subnet - type: string - type: object - type: array - subregionName: - description: The default subregion name - type: string - type: object - type: object - status: - description: OscClusterStatus defines the observed state of OscCluster - properties: - conditions: - description: Conditions provide observations of the operational state - of a Cluster API resource. - items: - description: Condition defines an observation of a Cluster API resource - operational state. - properties: - lastTransitionTime: - description: |- - Last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when - the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - A human readable message indicating details about the transition. - This field may be empty. - type: string - reason: - description: |- - The reason for the condition's last transition in CamelCase. - The specific API may choose whether or not this field is considered a guaranteed API. - This field may not be empty. - type: string - severity: - description: |- - Severity provides an explicit classification of Reason code, so the users or machines can immediately - understand the current situation and act accordingly. - The Severity field MUST be set only when Status=False. - type: string - status: - description: Status of the condition, one of True, False, Unknown. - type: string - type: - description: |- - Type of condition in CamelCase or in foo.example.com/CamelCase. - Many .condition.type values are consistent across resources like Available, but because arbitrary conditions - can be useful (see .node.status.conditions), the ability to deconflict is important. - type: string - required: - - lastTransitionTime - - status - - type - type: object - type: array - failureDomains: - additionalProperties: - description: |- - FailureDomainSpec is the Schema for Cluster API failure domains. - It allows controllers to understand how many failure domains a cluster can optionally span across. - properties: - attributes: - additionalProperties: - type: string - description: Attributes is a free form map of attributes an - infrastructure provider might use or require. - type: object - controlPlane: - description: ControlPlane determines if this failure domain - is suitable for use by control plane machines. - type: boolean - type: object - description: FailureDomains is a slice of FailureDomains. - type: object - network: - properties: - LoadbalancerRef: - description: Map between LoadbalancerId and LoadbalancerName - (Load Balancer tag Name with cluster UID) - properties: - resourceMap: - additionalProperties: - type: string - type: object - type: object - bastionref: - description: Map between InstanceId and BastionName (Bastion - tag Name with cluster UID) - properties: - resourceMap: - additionalProperties: - type: string - type: object - type: object - internetserviceref: - description: Map between InternetServiceId and InternetServiceName - (Internet Service tag Name with cluster UID) - properties: - resourceMap: - additionalProperties: - type: string - type: object - type: object - linkPublicIpRef: - description: Map between LinkPublicIpId and PublicIpName (Public - IP tag Name with cluster UID) - properties: - resourceMap: - additionalProperties: - type: string - type: object - type: object - linkroutetableref: - additionalProperties: - items: - type: string - type: array - description: Map between LinkRouteTableId and RouteTablesName - (Route Table tag Name with cluster UID) - type: object - natref: - description: Map between NatServiceId and NatServiceName (Nat - Service tag Name with cluster UID) - properties: - resourceMap: - additionalProperties: - type: string - type: object - type: object - netref: - description: Map between NetId and NetName (Net tag Name with - cluster UID) - properties: - resourceMap: - additionalProperties: - type: string - type: object - type: object - publicipref: - description: Map between PublicIpId and PublicIpName (Public - IP tag Name with cluster UID) - properties: - resourceMap: - additionalProperties: - type: string - type: object - type: object - routeref: - description: Map between RouteId and RouteName (Route tag Name - with cluster UID) - properties: - resourceMap: - additionalProperties: - type: string - type: object - type: object - routetableref: - description: Map between RouteTablesId and RouteTablesName (Route - Tables tag Name with cluster UID) - properties: - resourceMap: - additionalProperties: - type: string - type: object - type: object - securitygroupref: - description: Map between SecurityGroupId and SecurityGroupName - (Security Group tag Name with cluster UID) - properties: - resourceMap: - additionalProperties: - type: string - type: object - type: object - securitygroupruleref: - description: Map between SecurityGroupRuleId and SecurityGroupName - (Security Group Rule tag Name with cluster UID) - properties: - resourceMap: - additionalProperties: - type: string - type: object - type: object - subnetref: - description: Map between SubnetId and SubnetName (Subnet tag - Name with cluster UID) - properties: - resourceMap: - additionalProperties: - type: string - type: object - type: object - type: object - ready: - type: boolean - reconcilerGeneration: - additionalProperties: - format: int64 - type: integer - type: object - resources: - properties: - bastion: - additionalProperties: - type: string - type: object - internetService: - additionalProperties: - type: string - type: object - natService: - additionalProperties: - type: string - type: object - net: - additionalProperties: - type: string - type: object - publicIps: - additionalProperties: - type: string - type: object - securityGroup: - additionalProperties: - type: string - type: object - subnet: - additionalProperties: - type: string - type: object - type: object - vmState: - type: string - type: object - type: object - served: true - storage: true - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - cert-manager.io/inject-ca-from: cluster-api-provider-outscale-system/cluster-api-provider-outscale-serving-cert - controller-gen.kubebuilder.io/version: v0.17.3-0.20250416153108-23dd517d714e - labels: - cluster.x-k8s.io/v1alpha3: v1alpha3 - cluster.x-k8s.io/v1beta1: v1beta1 - name: oscclustertemplates.infrastructure.cluster.x-k8s.io -spec: - group: infrastructure.cluster.x-k8s.io - names: - categories: - - cluster-api - kind: OscClusterTemplate - listKind: OscClusterTemplateList - plural: oscclustertemplates - singular: oscclustertemplate - scope: Namespaced - versions: - - name: v1beta1 - schema: - openAPIV3Schema: - description: OscClusterTemplate is the Schema for the oscclustertemplates - API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: OscClusterTemplateSpec defines the desired state of OscClusterTemplate - properties: - template: - properties: - metadata: - description: |- - ObjectMeta is metadata that all persisted resources must have, which includes all objects - users must create. This is a copy of customizable fields from metav1.ObjectMeta. - - ObjectMeta is embedded in `Machine.Spec`, `MachineDeployment.Template` and `MachineSet.Template`, - which are not top-level Kubernetes objects. Given that metav1.ObjectMeta has lots of special cases - and read-only fields which end up in the generated CRD validation, having it as a subset simplifies - the API and some issues that can impact user experience. - - During the [upgrade to controller-tools@v2](https://github.com/kubernetes-sigs/cluster-api/pull/1054) - for v1alpha2, we noticed a failure would occur running Cluster API test suite against the new CRDs, - specifically `spec.metadata.creationTimestamp in body must be of type string: "null"`. - The investigation showed that `controller-tools@v2` behaves differently than its previous version - when handling types from [metav1](k8s.io/apimachinery/pkg/apis/meta/v1) package. - - In more details, we found that embedded (non-top level) types that embedded `metav1.ObjectMeta` - had validation properties, including for `creationTimestamp` (metav1.Time). - The `metav1.Time` type specifies a custom json marshaller that, when IsZero() is true, returns `null` - which breaks validation because the field isn't marked as nullable. - - In future versions, controller-tools@v2 might allow overriding the type and validation for embedded - types. When that happens, this hack should be revisited. - properties: - annotations: - additionalProperties: - type: string - description: |- - Annotations is an unstructured key value map stored with a resource that may be - set by external tools to store and retrieve arbitrary metadata. They are not - queryable and should be preserved when modifying objects. - More info: http://kubernetes.io/docs/user-guide/annotations - type: object - labels: - additionalProperties: - type: string - description: |- - Map of string keys and values that can be used to organize and categorize - (scope and select) objects. May match selectors of replication controllers - and services. - More info: http://kubernetes.io/docs/user-guide/labels - type: object - type: object - spec: - description: OscClusterSpec defines the desired state of OscCluster - properties: - controlPlaneEndpoint: - description: APIEndpoint represents a reachable Kubernetes - API endpoint. - properties: - host: - description: The hostname on which the API server is serving. - type: string - port: - description: The port on which the API server is serving. - format: int32 - type: integer - required: - - host - - port - type: object - network: - properties: - allowFromIPRanges: - description: The list of IP ranges (in CIDR notation) - to restrict bastion/Kubernetes API access to. - items: - type: string - type: array - bastion: - description: The bastion configuration - properties: - PublicIpId: - description: The ID of an existing public IP to use - for this VM. - type: string - clusterName: - type: string - deviceName: - type: string - enable: - type: boolean - imageAccountId: - type: string - imageId: - type: string - imageName: - type: string - keypairName: - type: string - name: - type: string - privateIps: - items: - properties: - name: - type: string - privateIp: - type: string - type: object - type: array - publicIpName: - description: unused - type: string - resourceId: - type: string - rootDisk: - properties: - rootDiskIops: - format: int32 - type: integer - rootDiskSize: - format: int32 - type: integer - rootDiskType: - type: string - type: object - securityGroupNames: - items: - properties: - name: - type: string - type: object - type: array - subnetName: - type: string - subregionName: - type: string - vmType: - type: string - type: object - clusterName: - description: The name of the cluster (unused) - type: string - controlPlaneSubnets: - description: List of subnet to spread controlPlane nodes - items: - type: string - type: array - extraSecurityGroupRule: - description: Add SecurityGroup Rule after the cluster - is created (unused) - type: boolean - image: - description: The image configuration (unused) - properties: - accountId: - type: string - name: - type: string - resourceId: - type: string - type: object - internetService: - description: The Internet Service configuration - properties: - clusterName: - description: the name of the cluster (unused) - type: string - name: - description: The name of the Internet service - type: string - resourceId: - description: the Internet Service resource id (unused) - type: string - type: object - loadBalancer: - description: The Load Balancer configuration - properties: - clusterName: - description: unused - type: string - healthCheck: - description: The healthCheck configuration of the - Load Balancer - properties: - checkinterval: - description: the time in second between two pings - format: int32 - type: integer - healthythreshold: - description: the consecutive number of pings which - are successful to consider the vm healthy - format: int32 - type: integer - port: - description: the HealthCheck port number - format: int32 - type: integer - protocol: - description: The HealthCheck protocol ('HTTP'|'TCP') - type: string - timeout: - description: the Timeout to consider VM unhealthy - format: int32 - type: integer - unhealthythreshold: - description: the consecutive number of pings which - are failed to consider the vm unhealthy - format: int32 - type: integer - type: object - listener: - description: The Listener configuration of the loadBalancer - properties: - backendport: - description: The port on which the backend VMs - will listen - format: int32 - type: integer - backendprotocol: - description: The protocol ('HTTP'|'TCP') to route - the traffic to the backend vm - type: string - loadbalancerport: - description: The port on which the loadbalancer - will listen - format: int32 - type: integer - loadbalancerprotocol: - description: the routing protocol ('HTTP'|'TCP') - type: string - type: object - loadbalancername: - description: The Load Balancer unique name - type: string - loadbalancertype: - description: The Load Balancer type (internet-facing - or internal) - type: string - securitygroupname: - description: The security group name for the load-balancer - type: string - subnetname: - description: The subnet name where to add the load - balancer. - type: string - type: object - natService: - description: The Nat Service configuration - properties: - clusterName: - description: The name of the cluster (unused) - type: string - name: - description: The name of the Nat Service - type: string - publicipname: - description: The Public Ip name (unused) - type: string - resourceId: - description: The resource id (unused) - type: string - subnetname: - description: The name of the Subnet to which the Nat - Service will be attached - type: string - subregionName: - description: The name of the Subregion to which the - Nat Service will be attached, unless a subnet has - been defined (unused) - type: string - type: object - natServices: - description: The Nat Services configuration - items: - properties: - clusterName: - description: The name of the cluster (unused) - type: string - name: - description: The name of the Nat Service - type: string - publicipname: - description: The Public Ip name (unused) - type: string - resourceId: - description: The resource id (unused) - type: string - subnetname: - description: The name of the Subnet to which the - Nat Service will be attached - type: string - subregionName: - description: The name of the Subregion to which - the Nat Service will be attached, unless a subnet - has been defined (unused) - type: string - type: object - type: array - net: - description: The Net configuration - properties: - clusterName: - description: the name of the cluster (unused) - type: string - ipRange: - description: the ip range in CIDR notation of the - Net - type: string - name: - description: the network name - type: string - resourceId: - description: The Id of the Net to reuise (if useExisting - is set) - type: string - useExisting: - description: Reuse an existing network defined by - resourceId ? - type: boolean - type: object - publicIps: - description: The Public Ip configuration - items: - properties: - clusterName: - type: string - name: - description: The tag name associate with the Public - Ip - type: string - resourceId: - description: The Public Ip Id response - type: string - type: object - type: array - routeTables: - description: The Route Table configuration - items: - properties: - name: - description: The tag name associate with the Route - Table - type: string - resourceId: - description: The Route Table Id response - type: string - role: - description: The role for this route table - type: string - routes: - description: The Route configuration - items: - properties: - destination: - description: the destination match Ip range - with CIDR notation - type: string - name: - description: The tag name associate with the - Route - type: string - resourceId: - description: The Route Id response - type: string - targetName: - description: The tag name associate with the - target resource type - type: string - targetType: - description: The target resource type which - can be Internet Service (gateway) or Nat - Service (nat-service) - type: string - type: object - type: array - subnets: - description: The subnet tag name associate with - a Subnet - items: - type: string - type: array - subregionName: - description: The subregion for this route table - type: string - type: object - type: array - securityGroups: - items: - properties: - description: - description: The description of the security group - type: string - name: - description: The name of the security group - type: string - resourceId: - description: When reusing network, the id of an - existing securityGroup to use. - type: string - roles: - description: The roles the securityGroup applies - to. - items: - type: string - type: array - securityGroupRules: - description: The list of rules for this securityGroup. - items: - properties: - flow: - description: The flow of the security group - (inbound or outbound) - type: string - fromPortRange: - description: The beginning of the port range - format: int32 - type: integer - ipProtocol: - description: The ip protocol name (tcp, udp, - icmp or -1) - type: string - ipRange: - description: The ip range of the security - group rule (deprecated) - type: string - ipRanges: - description: The list of ip ranges of the - security group rule - items: - type: string - type: array - name: - description: The tag name associate with the - security group - type: string - resourceId: - description: The security group rule id - type: string - toPortRange: - description: The end of the port range - format: int32 - type: integer - type: object - type: array - tag: - type: string - type: object - type: array - subnets: - description: The Subnet configuration - items: - properties: - ipSubnetRange: - description: the Ip range in CIDR notation of the - Subnet - type: string - name: - description: The name of the Subnet - type: string - resourceId: - description: The id of the Subnet to reuse (if net.useExisting - is set) - type: string - roles: - description: The role of the Subnet (controlplane, - worker, loadbalancer, bastion or nat) - items: - type: string - type: array - subregionName: - description: The subregion name of the Subnet - type: string - type: object - type: array - subregionName: - description: The default subregion name - type: string - type: object - type: object - required: - - spec - type: object - required: - - template - type: object - type: object - served: true - storage: true ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - cert-manager.io/inject-ca-from: cluster-api-provider-outscale-system/cluster-api-provider-outscale-serving-cert - controller-gen.kubebuilder.io/version: v0.17.3-0.20250416153108-23dd517d714e - labels: - cluster.x-k8s.io/v1alpha3: v1alpha3 - cluster.x-k8s.io/v1beta1: v1beta1 - name: oscmachines.infrastructure.cluster.x-k8s.io -spec: - group: infrastructure.cluster.x-k8s.io - names: - categories: - - cluster-api - kind: OscMachine - listKind: OscMachineList - plural: oscmachines - singular: oscmachine - scope: Namespaced - versions: - - additionalPrinterColumns: - - jsonPath: .spec.node.vm.resourceId - name: VM - type: string - - jsonPath: .spec.node.vm.vmType - name: VM Type - type: string - - jsonPath: .spec.node.vm.subregionName - name: SubRegion - type: string - - jsonPath: .status.vmState - name: State - type: string - name: v1beta1 - schema: - openAPIV3Schema: - description: OscMachine is the Schema for the oscmachines API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: OscMachineSpec defines the desired state of OscMachine - properties: - node: - properties: - clusterName: - type: string - image: - properties: - accountId: - type: string - name: - type: string - resourceId: - type: string - type: object - keypair: - properties: - clusterName: - description: Deprecated - type: string - deleteKeypair: - description: Deprecated - type: boolean - name: - description: Deprecated - type: string - publicKey: - description: Deprecated - type: string - resourceId: - description: Deprecated - type: string - type: object - vm: - properties: - clusterName: - type: string - deviceName: - type: string - imageId: - type: string - keypairName: - type: string - loadBalancerName: - type: string - name: - type: string - privateIps: - items: - properties: - name: - type: string - privateIp: - type: string - type: object - type: array - publicIp: - type: boolean - publicIpName: - type: string - replica: - format: int32 - type: integer - resourceId: - type: string - role: - type: string - rootDisk: - properties: - rootDiskIops: - format: int32 - type: integer - rootDiskSize: - format: int32 - type: integer - rootDiskType: - type: string - type: object - securityGroupNames: - items: - properties: - name: - type: string - type: object - type: array - subnetName: - type: string - subregionName: - type: string - tags: - additionalProperties: - type: string - type: object - vmType: - type: string - volumeDeviceName: - type: string - volumeName: - type: string - required: - - keypairName - type: object - volumes: - items: - properties: - device: - type: string - iops: - format: int32 - type: integer - name: - type: string - resourceId: - type: string - size: - format: int32 - type: integer - subregionName: - description: Deprecated - type: string - volumeType: - type: string - required: - - device - - size - type: object - type: array - type: object - providerID: - type: string - type: object - status: - description: OscMachineStatus defines the observed state of OscMachine - properties: - addresses: - items: - description: NodeAddress contains information for the node's address. - properties: - address: - description: The node address. - type: string - type: - description: Node address type, one of Hostname, ExternalIP - or InternalIP. - type: string - required: - - address - - type - type: object - type: array - conditions: - description: Conditions provide observations of the operational state - of a Cluster API resource. - items: - description: Condition defines an observation of a Cluster API resource - operational state. - properties: - lastTransitionTime: - description: |- - Last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when - the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - A human readable message indicating details about the transition. - This field may be empty. - type: string - reason: - description: |- - The reason for the condition's last transition in CamelCase. - The specific API may choose whether or not this field is considered a guaranteed API. - This field may not be empty. - type: string - severity: - description: |- - Severity provides an explicit classification of Reason code, so the users or machines can immediately - understand the current situation and act accordingly. - The Severity field MUST be set only when Status=False. - type: string - status: - description: Status of the condition, one of True, False, Unknown. - type: string - type: - description: |- - Type of condition in CamelCase or in foo.example.com/CamelCase. - Many .condition.type values are consistent across resources like Available, but because arbitrary conditions - can be useful (see .node.status.conditions), the ability to deconflict is important. - type: string - required: - - lastTransitionTime - - status - - type - type: object - type: array - failureDomain: - type: string - failureMessage: - type: string - failureReason: - description: MachineStatusError defines errors states for Machine - objects. - type: string - node: - properties: - imageRef: - description: Map between resourceId and resourceName (tag Name - with cluster UID) - properties: - resourceMap: - additionalProperties: - type: string - type: object - type: object - keypairRef: - description: Map between resourceId and resourceName (tag Name - with cluster UID) - properties: - resourceMap: - additionalProperties: - type: string - type: object - type: object - linkPublicIpRef: - description: Map between resourceId and resourceName (tag Name - with cluster UID) - properties: - resourceMap: - additionalProperties: - type: string - type: object - type: object - publicIpIdRef: - description: Map between resourceId and resourceName (tag Name - with cluster UID) - properties: - resourceMap: - additionalProperties: - type: string - type: object - type: object - vmRef: - description: Map between resourceId and resourceName (tag Name - with cluster UID) - properties: - resourceMap: - additionalProperties: - type: string - type: object - type: object - volumeRef: - description: Map between resourceId and resourceName (tag Name - with cluster UID) - properties: - resourceMap: - additionalProperties: - type: string - type: object - type: object - type: object - ready: - type: boolean - reconcilerGeneration: - additionalProperties: - format: int64 - type: integer - type: object - resources: - properties: - image: - additionalProperties: - type: string - type: object - vm: - additionalProperties: - type: string - type: object - volumes: - additionalProperties: - type: string - type: object - type: object - vmState: - type: string - type: object - type: object - served: true - storage: true - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.17.3-0.20250416153108-23dd517d714e - labels: - cluster.x-k8s.io/v1alpha3: v1alpha3 - cluster.x-k8s.io/v1beta1: v1beta1 - name: oscmachinetemplates.infrastructure.cluster.x-k8s.io -spec: - group: infrastructure.cluster.x-k8s.io - names: - categories: - - cluster-api - kind: OscMachineTemplate - listKind: OscMachineTemplateList - plural: oscmachinetemplates - singular: oscmachinetemplate - scope: Namespaced - versions: - - name: v1beta1 - schema: - openAPIV3Schema: - description: OscMachineTemplate is the Schema for the OscMachineTemplate API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: OscMachineTemplateSpec define oscMachine template - properties: - template: - description: OscMachineTemplateResource is the Schema for the OscMachineTemplate - api - properties: - metadata: - description: |- - ObjectMeta is metadata that all persisted resources must have, which includes all objects - users must create. This is a copy of customizable fields from metav1.ObjectMeta. - - ObjectMeta is embedded in `Machine.Spec`, `MachineDeployment.Template` and `MachineSet.Template`, - which are not top-level Kubernetes objects. Given that metav1.ObjectMeta has lots of special cases - and read-only fields which end up in the generated CRD validation, having it as a subset simplifies - the API and some issues that can impact user experience. - - During the [upgrade to controller-tools@v2](https://github.com/kubernetes-sigs/cluster-api/pull/1054) - for v1alpha2, we noticed a failure would occur running Cluster API test suite against the new CRDs, - specifically `spec.metadata.creationTimestamp in body must be of type string: "null"`. - The investigation showed that `controller-tools@v2` behaves differently than its previous version - when handling types from [metav1](k8s.io/apimachinery/pkg/apis/meta/v1) package. - - In more details, we found that embedded (non-top level) types that embedded `metav1.ObjectMeta` - had validation properties, including for `creationTimestamp` (metav1.Time). - The `metav1.Time` type specifies a custom json marshaller that, when IsZero() is true, returns `null` - which breaks validation because the field isn't marked as nullable. - - In future versions, controller-tools@v2 might allow overriding the type and validation for embedded - types. When that happens, this hack should be revisited. - properties: - annotations: - additionalProperties: - type: string - description: |- - Annotations is an unstructured key value map stored with a resource that may be - set by external tools to store and retrieve arbitrary metadata. They are not - queryable and should be preserved when modifying objects. - More info: http://kubernetes.io/docs/user-guide/annotations - type: object - labels: - additionalProperties: - type: string - description: |- - Map of string keys and values that can be used to organize and categorize - (scope and select) objects. May match selectors of replication controllers - and services. - More info: http://kubernetes.io/docs/user-guide/labels - type: object - type: object - spec: - description: OscMachineSpec defines the desired state of OscMachine - properties: - node: - properties: - clusterName: - type: string - image: - properties: - accountId: - type: string - name: - type: string - resourceId: - type: string - type: object - keypair: - properties: - clusterName: - description: Deprecated - type: string - deleteKeypair: - description: Deprecated - type: boolean - name: - description: Deprecated - type: string - publicKey: - description: Deprecated - type: string - resourceId: - description: Deprecated - type: string - type: object - vm: - properties: - clusterName: - type: string - deviceName: - type: string - imageId: - type: string - keypairName: - type: string - loadBalancerName: - type: string - name: - type: string - privateIps: - items: - properties: - name: - type: string - privateIp: - type: string - type: object - type: array - publicIp: - type: boolean - publicIpName: - type: string - replica: - format: int32 - type: integer - resourceId: - type: string - role: - type: string - rootDisk: - properties: - rootDiskIops: - format: int32 - type: integer - rootDiskSize: - format: int32 - type: integer - rootDiskType: - type: string - type: object - securityGroupNames: - items: - properties: - name: - type: string - type: object - type: array - subnetName: - type: string - subregionName: - type: string - tags: - additionalProperties: - type: string - type: object - vmType: - type: string - volumeDeviceName: - type: string - volumeName: - type: string - required: - - keypairName - type: object - volumes: - items: - properties: - device: - type: string - iops: - format: int32 - type: integer - name: - type: string - resourceId: - type: string - size: - format: int32 - type: integer - subregionName: - description: Deprecated - type: string - volumeType: - type: string - required: - - device - - size - type: object - type: array - type: object - providerID: - type: string - type: object - required: - - spec - type: object - required: - - template - type: object - status: - properties: - capacity: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: ResourceList is a set of (resource name, quantity) pairs. - type: object - conditions: - description: Conditions provide observations of the operational state - of a Cluster API resource. - items: - description: Condition defines an observation of a Cluster API resource - operational state. - properties: - lastTransitionTime: - description: |- - Last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when - the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - A human readable message indicating details about the transition. - This field may be empty. - type: string - reason: - description: |- - The reason for the condition's last transition in CamelCase. - The specific API may choose whether or not this field is considered a guaranteed API. - This field may not be empty. - type: string - severity: - description: |- - Severity provides an explicit classification of Reason code, so the users or machines can immediately - understand the current situation and act accordingly. - The Severity field MUST be set only when Status=False. - type: string - status: - description: Status of the condition, one of True, False, Unknown. - type: string - type: - description: |- - Type of condition in CamelCase or in foo.example.com/CamelCase. - Many .condition.type values are consistent across resources like Available, but because arbitrary conditions - can be useful (see .node.status.conditions), the ability to deconflict is important. - type: string - required: - - lastTransitionTime - - status - - type - type: object - type: array - type: object - type: object - served: true - storage: true - subresources: - status: {} ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: cluster-api-provider-outscale-controller-manager - namespace: cluster-api-provider-outscale-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: cluster-api-provider-outscale-leader-election-role - namespace: cluster-api-provider-outscale-system -rules: - - apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch - - create - - update - - patch - - delete - - apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - get - - list - - watch - - create - - update - - patch - - delete - - apiGroups: - - "" - resources: - - events - verbs: - - create - - patch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - creationTimestamp: null - name: cluster-api-provider-outscale-manager-role -rules: - - apiGroups: - - "" - resources: - - events - verbs: - - create - - get - - list - - patch - - update - - watch - - apiGroups: - - "" - resources: - - secrets - verbs: - - get - - list - - watch - - apiGroups: - - cluster.x-k8s.io - resources: - - clusters - verbs: - - get - - list - - watch - - apiGroups: - - cluster.x-k8s.io - resources: - - clusters/status - verbs: - - get - - list - - watch - - apiGroups: - - infrastructure.cluster.x-k8s.io - resources: - - oscclusters - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - - apiGroups: - - infrastructure.cluster.x-k8s.io - resources: - - oscclusters/finalizers - verbs: - - update - - apiGroups: - - infrastructure.cluster.x-k8s.io - resources: - - oscclusters/status - verbs: - - get - - patch - - update - - apiGroups: - - infrastructure.cluster.x-k8s.io - resources: - - oscmachines - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - - apiGroups: - - infrastructure.cluster.x-k8s.io - resources: - - oscmachines/finalizers - verbs: - - update - - apiGroups: - - infrastructure.cluster.x-k8s.io - resources: - - oscmachines/status - verbs: - - get - - patch - - update ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: cluster-api-provider-outscale-metrics-reader -rules: - - nonResourceURLs: - - /metrics - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: cluster-api-provider-outscale-proxy-role -rules: - - apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create - - apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: cluster-api-provider-outscale-leader-election-rolebinding - namespace: cluster-api-provider-outscale-system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: cluster-api-provider-outscale-leader-election-role -subjects: - - kind: ServiceAccount - name: cluster-api-provider-outscale-controller-manager - namespace: cluster-api-provider-outscale-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: cluster-api-provider-outscale-manager-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cluster-api-provider-outscale-manager-role -subjects: - - kind: ServiceAccount - name: cluster-api-provider-outscale-controller-manager - namespace: cluster-api-provider-outscale-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: cluster-api-provider-outscale-proxy-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cluster-api-provider-outscale-proxy-role -subjects: - - kind: ServiceAccount - name: cluster-api-provider-outscale-controller-manager - namespace: cluster-api-provider-outscale-system ---- -apiVersion: v1 -data: - controller_manager_config.yaml: | - apiVersion: controller-runtime.sigs.k8s.io/v1alpha1 - kind: ControllerManagerConfig - health: - healthProbeBindAddress: :8081 - metrics: - bindAddress: 127.0.0.1:8080 - webhook: - port: 9443 - leaderElection: - leaderElect: true - resourceName: 4b982d66.cluster.x-k8s.io -kind: ConfigMap -metadata: - name: cluster-api-provider-outscale-manager-config - namespace: cluster-api-provider-outscale-system ---- -apiVersion: v1 -kind: Service -metadata: - labels: - control-plane: controller-manager - name: cluster-api-provider-outscale-controller-manager-service - namespace: cluster-api-provider-outscale-system -spec: - ports: - - name: https - port: 8443 - protocol: TCP - targetPort: https - selector: - control-plane: controller-manager ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - control-plane: controller-manager - name: cluster-api-provider-outscale-controller-manager - namespace: cluster-api-provider-outscale-system -spec: - replicas: 1 - selector: - matchLabels: - control-plane: controller-manager - template: - metadata: - annotations: - kubectl.kubernetes.io/default-container: manager - labels: - control-plane: controller-manager - spec: - containers: - - args: - - --health-probe-bind-address=:8081 - - --metrics-bind-address=127.0.0.1:8080 - - --leader-elect - command: - - /manager - env: - - name: OSC_ACCESS_KEY - valueFrom: - secretKeyRef: - key: access_key - name: cluster-api-provider-outscale - optional: true - - name: OSC_SECRET_KEY - valueFrom: - secretKeyRef: - key: secret_key - name: cluster-api-provider-outscale - optional: true - image: outscale/cluster-api-osc-controller:v0.1.0-alpha.1 - livenessProbe: - httpGet: - path: /healthz - port: 8081 - initialDelaySeconds: 15 - periodSeconds: 20 - name: manager - readinessProbe: - httpGet: - path: /readyz - port: 8081 - initialDelaySeconds: 5 - periodSeconds: 10 - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 10m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - imagePullSecrets: - - name: regcred - securityContext: - runAsNonRoot: true - serviceAccountName: cluster-api-provider-outscale-controller-manager - terminationGracePeriodSeconds: 10 diff --git a/test/e2e/data/shared/v1beta1/metadata.yaml b/test/e2e/data/shared/v1beta1/metadata.yaml index bb79f3a59..4e3ff0f86 100644 --- a/test/e2e/data/shared/v1beta1/metadata.yaml +++ b/test/e2e/data/shared/v1beta1/metadata.yaml @@ -1,6 +1,12 @@ apiVersion: clusterctl.cluster.x-k8s.io/v1alpha3 kind: Metadata releaseSeries: + - major: 1 + minor: 10 + contract: v1beta1 + - major: 1 + minor: 9 + contract: v1beta1 - major: 1 minor: 8 contract: v1beta1 @@ -27,7 +33,4 @@ releaseSeries: contract: v1beta1 - major: 1 minor: 0 - contract: v1beta1 - - major: 0 - minor: 1 contract: v1beta1 \ No newline at end of file diff --git a/test/e2e/quickstart_test.go b/test/e2e/quickstart_test.go index fac284969..4b6f24ff2 100644 --- a/test/e2e/quickstart_test.go +++ b/test/e2e/quickstart_test.go @@ -6,7 +6,6 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/utils/ptr" - capi_e2e "sigs.k8s.io/cluster-api/test/e2e" )