✨ feat(airgap): add image preloading

jfbus · jfbus · commit 4e37ed7a5786 · 2025-10-23T12:04:09.000Z
diff --git a/.github/workflows/unit-func-e2e-test.yaml b/.github/workflows/unit-func-e2e-test.yaml
@@ -40,6 +40,10 @@ jobs:
     runs-on: [self-hosted, linux]
     needs: [unit_test]
     steps:
+    - name: 🩹 Safe config for 💩 DNS
+      run: |
+        sed 's/options /options use-vc single-request attempts:5 /' < /etc/resolv.conf > /etc/resolv.conf.new
+        cat /etc/resolv.conf.new > /etc/resolv.conf
     - name: ⬇️ Checkout repository
       uses: actions/checkout@v4
     - name: ⬇️ Install kubectl
@@ -54,7 +58,8 @@ jobs:
         kustomize-version: v4.5.7
     - name: 🔎 Check IP
       id: ip
-      run: echo "ip=`curl -s https://api.ipify.org`" | tee $GITHUB_OUTPUT
+      run: |
+        echo "ip=`curl -s https://api.ipify.org`" | tee $GITHUB_OUTPUT
     - name: 🔐 Set ak/sk name based on runner region
       run: .github/scripts/runneraksk.sh
     - name: 🧹 Frieza
@@ -89,10 +94,18 @@ jobs:
         sudo apt install -y docker-buildx-plugin 
         make docker-buildx
         make docker-push
-        docker image prune -a -f
       env:
         IMG: ${{ vars.REGISTRY }}/outscale/cluster-api-outscale-controllers:${{ github.sha }}
         DOCKER_BUILDKIT: 1
+    - name: 🔎 Preloader snapshot
+      id: preloader
+      uses: outscale/k8s-image-preloader/github_actions/preloader_snapshot@main
+      with:
+        KUBECONFIG: ${{ steps.management.outputs.KUBECONFIG }}
+        OSC_ACCESS_KEY: ${{ secrets[env.OSC_ACCESS_KEY_NAME] }}
+        OSC_SECRET_KEY: ${{ secrets[env.OSC_SECRET_KEY_NAME] }}
+        OSC_REGION: ${{ env.OSC_REGION }}
+        CSI: true
     - name: 🔐 Create CAPOSC ns and credentials
       run: make credential
       shell: bash
@@ -114,10 +127,11 @@ jobs:
         KUBECONFIG: "${{ github.workspace }}/${{ steps.management.outputs.KUBECONFIG }}"
         CCM_OSC_ACCESS_KEY: ${{ secrets[env.OSC_ACCESS_KEY_NAME] }}
         CCM_OSC_SECRET_KEY: ${{ secrets[env.OSC_SECRET_KEY_NAME] }}
-        CCM_OSC_REGION: $${{ env.OSC_REGION }}
+        CCM_OSC_REGION: ${{ env.OSC_REGION }}
         IMG: ${{ vars.REGISTRY }}/outscale/cluster-api-outscale-controllers:${{ github.sha }}
         IMG_UPGRADE_FROM: ${{ vars.IMG_UPGRADE_FROM }}
         IMG_UPGRADE_TO: ${{ vars.IMG_UPGRADE_TO }}
+        PRELOAD_SNAPSHOT_ID: ${{ steps.preloader.outputs.SNAPSHOT_ID }}
     - name: 📝 Get CAPOSC state/logs
       run: |
         echo "**** pod state"
diff --git a/api/v1beta1/types.go b/api/v1beta1/types.go
@@ -523,7 +523,7 @@ type OscImage struct {
 type OscVolume struct {
 	// The volume name.
 	Name string `json:"name,omitempty"`
-	// The volume device (/dev/sdX)
+	// The volume device (/dev/xvdX)
 	// +kubebuilder:validation:Required
 	Device string `json:"device"`
 	// The volume iops (io1 volumes only)
diff --git a/capm.yaml b/capm.yaml
@@ -10,7 +10,7 @@ kind: CustomResourceDefinition
 metadata:
   annotations:
     cert-manager.io/inject-ca-from: cluster-api-provider-outscale-system/cluster-api-provider-outscale-serving-cert
-    controller-gen.kubebuilder.io/version: v0.19.1-0.20250924093817-1d67470bef32
+    controller-gen.kubebuilder.io/version: v0.19.1-0.20251013055054-8ef4174c3d65
   labels:
     cluster.x-k8s.io/v1alpha3: v1alpha3
     cluster.x-k8s.io/v1beta1: v1beta1
@@ -879,7 +879,7 @@ kind: CustomResourceDefinition
 metadata:
   annotations:
     cert-manager.io/inject-ca-from: cluster-api-provider-outscale-system/cluster-api-provider-outscale-serving-cert
-    controller-gen.kubebuilder.io/version: v0.19.1-0.20250924093817-1d67470bef32
+    controller-gen.kubebuilder.io/version: v0.19.1-0.20251013055054-8ef4174c3d65
   labels:
     cluster.x-k8s.io/v1alpha3: v1alpha3
     cluster.x-k8s.io/v1beta1: v1beta1
@@ -1588,7 +1588,7 @@ kind: CustomResourceDefinition
 metadata:
   annotations:
     cert-manager.io/inject-ca-from: cluster-api-provider-outscale-system/cluster-api-provider-outscale-serving-cert
-    controller-gen.kubebuilder.io/version: v0.19.1-0.20250924093817-1d67470bef32
+    controller-gen.kubebuilder.io/version: v0.19.1-0.20251013055054-8ef4174c3d65
   labels:
     cluster.x-k8s.io/v1alpha3: v1alpha3
     cluster.x-k8s.io/v1beta1: v1beta1
@@ -1994,7 +1994,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.19.1-0.20250924093817-1d67470bef32
+    controller-gen.kubebuilder.io/version: v0.19.1-0.20251013055054-8ef4174c3d65
   labels:
     cluster.x-k8s.io/v1alpha3: v1alpha3
     cluster.x-k8s.io/v1beta1: v1beta1
diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_oscclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_oscclusters.yaml
@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.19.1-0.20250924093817-1d67470bef32
+    controller-gen.kubebuilder.io/version: v0.19.1-0.20251013055054-8ef4174c3d65
   name: oscclusters.infrastructure.cluster.x-k8s.io
 spec:
   group: infrastructure.cluster.x-k8s.io
diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_oscclustertemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_oscclustertemplates.yaml
@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.19.1-0.20250924093817-1d67470bef32
+    controller-gen.kubebuilder.io/version: v0.19.1-0.20251013055054-8ef4174c3d65
   name: oscclustertemplates.infrastructure.cluster.x-k8s.io
 spec:
   group: infrastructure.cluster.x-k8s.io
diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_oscmachines.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_oscmachines.yaml
@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.19.1-0.20250924093817-1d67470bef32
+    controller-gen.kubebuilder.io/version: v0.19.1-0.20251013055054-8ef4174c3d65
   name: oscmachines.infrastructure.cluster.x-k8s.io
 spec:
   group: infrastructure.cluster.x-k8s.io
diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_oscmachinetemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_oscmachinetemplates.yaml
@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.19.1-0.20250924093817-1d67470bef32
+    controller-gen.kubebuilder.io/version: v0.19.1-0.20251013055054-8ef4174c3d65
   name: oscmachinetemplates.infrastructure.cluster.x-k8s.io
 spec:
   group: infrastructure.cluster.x-k8s.io
diff --git a/controllers/osccluster_netpeering.go b/controllers/osccluster_netpeering.go
@@ -74,7 +74,7 @@ func (r *OscClusterReconciler) reconcileNetPeering(ctx context.Context, clusterS
 			return reconcile.Result{}, fmt.Errorf("cannot get mgmt credentials: %w", err)
 		}
 		mgmtSvc := r.Cloud.NetPeering(mgmt)
-		log.V(2).Info("Accepting netPeering")
+		log.V(2).Info("Accepting netPeering", "netPeeringId", np.GetNetPeeringId())
 		err = mgmtSvc.AcceptNetPeering(ctx, np.GetNetPeeringId())
 		if err != nil {
 			return reconcile.Result{}, fmt.Errorf("cannot accept netPeering: %w", err)
@@ -109,7 +109,7 @@ func (r *OscClusterReconciler) reconcileDeleteNetPeering(ctx context.Context, cl
 		if np.State.GetName() != "pending-acceptance" && np.State.GetName() != "active" {
 			continue
 		}
-		log.V(2).Info("Deleting netPeering", "subnetId", np.GetNetPeeringId())
+		log.V(2).Info("Deleting netPeering", "netPeeringId", np.GetNetPeeringId())
 		err = svc.DeleteNetPeering(ctx, np.GetNetPeeringId())
 		if err != nil {
 			return reconcile.Result{}, fmt.Errorf("cannot delete netPeering: %w", err)
diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md
@@ -8,7 +8,7 @@
     - [Multi AZ clusters](./topics/config-multiaz.md)
     - [Securing cluster access](./topics/config-security.md)
     - [Air-gapped clusters](./topics/config-airgap.md)
-    - [Omi](./topics/omi.md)
+    - [Preloading images](./topics/preload.md)
     - [Troubleshooting](./topics/troubleshooting.md)
     - [Cluster-Autoscaler](./topics/cluster-autoscaler.md)
     - [How to upgrade](./topics/upgrade-cluster.md)
diff --git a/docs/src/topics/config-airgap.md b/docs/src/topics/config-airgap.md
@@ -65,9 +65,9 @@ network:
 
 TBD
 
-## Pre-loaded images
+## Preloaded images
 
-TBD
+See [Preloading images](preload.md).
 
 ## Outscale API access
 
diff --git a/docs/src/topics/preload.md b/docs/src/topics/preload.md
@@ -0,0 +1,70 @@
+# Preloading images
+
+Preloading images allows you to add container images to the local containerd cache during boot.
+
+It allows:
+* faster startup times (no need to download images from the internet),
+* building clusters without internet access.
+
+## Creating a snapshot
+
+You need to have a seed cluster that has access to the internet and all apps required on the target cluster installed.
+
+The seed cluster needs to have CSI installed, with snapshotting enabled.
+
+You can use the [following example](k8s-image-preloader-example) based on [k8s-image-preloader](8s-image-preloader) to create the snapshot.
+
+You may want to adapt the spec if you need more than 2GB of storage for your images or if you want to change the VolumeSnapshotClass deletionPolicy.
+
+[k8s-image-preloader](8s-image-preloader):
+* mounts a PersistentVolumeClaim,
+* searches images in the local containerd cache,
+* searches pods and cronjobs,
+* pulls images from the remote registry (even if already present in the cache, as containerd prunes some layers from cached images),
+* exports the images to the PVC,
+* creates a VolumeSnapshot from the PVC.
+
+You can have the handle (IaaS snapshot ID) by querying the VolumeSnapshotContent:
+```bash
+kubectl get vsc -o custom-columns=NAME:.metadata.name,SNAPSHOT:.spec.volumeSnapshotRef.name,NAMESPACE:.spec.volumeSnapshotRef.namespace,HANDLE:.status.snapshotHandle
+```
+
+## Importing images
+
+You will need to mount a volume based on the previously generated snapshot on target nodes:
+
+In `OscMachineTemplate` resources:
+```yaml
+        vm:
+          [...]
+        volumes:
+        - device: /dev/xvdb
+          size: 2
+          type: gp2
+          fromSnapshot: snap-xxx
+```
+
+And import all images during cloud-init:
+
+In `KubeadmConfigTemplate`/`KubeadmControlPlane` resources:
+```yaml
+  spec:
+    joinConfiguration:
+      [...]
+    mounts:
+      - - xvdb
+        - /preload
+        - ext4
+        - auto,exec,ro
+    preKubeadmCommands:
+      - /preload/restore.sh
+```
+
+## Notes
+
+k8s-image-preloader only works with containerd, and uses the `ctr` tool present on the node, but a similar process can be built on cri-o using [skopeo](skopeo).
+
+<!-- References -->
+[k8s-image-preloader-example]: https://raw.githubusercontent.com/outscale/k8s-image-preloader/refs/heads/main/examples/example.yaml
+[skopeo]: https://github.com/containers/skopeo
+[k8s-image-preloader]: https://github.com/outscale/k8s-image-preloader
diff --git a/github_actions/deploy_cluster/cleanup.sh b/github_actions/deploy_cluster/cleanup.sh
@@ -16,4 +16,5 @@ kubeconfig=`/.venv/bin/oks-cli cluster kubeconfig --cluster-name $cluster_name -
 export KUBECONFIG=$kubeconfig
 
 # do not wait if stuck, frieza will purge everything
-kubectl delete ns $OSC_CLUSTER_NAME --timeout 2m || /bin/true
+# as freeza cannot delete netpeerings, be careful
+kubectl delete ns $OSC_CLUSTER_NAME --timeout 10m || /bin/true
diff --git a/github_actions/deploy_cluster/main.sh b/github_actions/deploy_cluster/main.sh
@@ -26,6 +26,9 @@ export OSC_REGION=`echo $OSC_AKSK|cut -d% -f 3`
 export OSC_IMAGE_NAME=`echo $OSC_IMAGE_NAME_ACCOUNT_ID|cut -d% -f 1`
 export OSC_IMAGE_ACCOUNT_ID=`echo $OSC_IMAGE_NAME_ACCOUNT_ID|cut -d% -f 2`
 
+# F***g DNS
+echo "options use-vc single-request attempts:5" >> /etc/resolv.conf
+
 if [ -z "$OSC_IMAGE_ACCOUNT_ID" ]; then
   export OSC_IMAGE_ACCOUNT_ID=`curl --retry 5 --retry-all-errors \
     -X POST https://api.$OSC_REGION.outscale.com/api/v1/ReadAccounts \
diff --git a/test/e2e/config/outscale.yaml b/test/e2e/config/outscale.yaml
@@ -5,8 +5,8 @@ providers:
   - name: cluster-api
     type: CoreProvider
     versions:
-      - name: v1.8.10
-        value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.8.10/core-components.yaml
+      - name: v1.10.7
+        value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.10.7/core-components.yaml
         type: "url"
         contract: v1beta1
         files:
@@ -19,8 +19,8 @@ providers:
   - name: kubeadm
     type: BootstrapProvider
     versions:
-      - name: v1.8.10
-        value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.8.10/bootstrap-components.yaml
+      - name: v1.10.7
+        value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.10.7/bootstrap-components.yaml
         type: "url"
         contract: v1beta1
         files:
@@ -33,8 +33,8 @@ providers:
   - name: kubeadm
     type: ControlPlaneProvider
     versions:
-      - name: v1.8.10
-        value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.8.10/control-plane-components.yaml
+      - name: v1.10.7
+        value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.10.7/control-plane-components.yaml
         type: "url"
         contract: v1beta1
         files:
@@ -107,6 +107,7 @@ variables:
   CONFORMANCE_CONTROL_PLANE_MACHINE_COUNT: 3
   KUBETEST_CONFIGURATION: "${KUBETEST_CONF_PATH:=./data/kubetest/conformance.yaml}"
   NODE_DRAIN_TIMEOUT: "60s"
+  PRELOAD_SNAPSHOT_ID: ""
 intervals:
   default/wait-cluster: ["15m", "10s"]
   default/wait-control-plane: ["30m", "10s"]
diff --git a/test/e2e/data/infrastructure-outscale/cluster-template-airgap.yaml b/test/e2e/data/infrastructure-outscale/cluster-template-airgap.yaml
@@ -75,6 +75,11 @@ spec:
           keypairName: "${OSC_KEYPAIR_NAME}"
           subregionName: ${OSC_REGION}a
           vmType: "${OSC_VM_TYPE}"
+        volumes:
+        - device: /dev/xvdb
+          size: 4
+          volumeType: gp2
+          fromSnapshot: ${PRELOAD_SNAPSHOT_ID}
 ---
 apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
 kind: OscMachineTemplate
@@ -96,6 +101,11 @@ spec:
           keypairName: "${OSC_KEYPAIR_NAME}"
           role: controlplane
           vmType: "${OSC_VM_TYPE}"
+        volumes:
+        - device: /dev/xvdb
+          size: 4
+          volumeType: gp2
+          fromSnapshot: ${PRELOAD_SNAPSHOT_ID}
 ---
 apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
 kind: KubeadmConfigTemplate
@@ -104,12 +114,20 @@ metadata:
 spec:
   template:
     spec:
+      verbosity: 5
       joinConfiguration:
         nodeRegistration:
           name: "{{ ds.meta_data.local_hostname }}"
           kubeletExtraArgs:
             cloud-provider: external
             provider-id: aws:///'{{ ds.meta_data.placement.availability_zone }}'/'{{ ds.meta_data.instance_id }}'
+      mounts:
+        - - xvdb
+          - /preload
+          - ext4
+          - auto,exec,ro
+      preKubeadmCommands:
+        - /preload/restore.sh
 ---
 kind: KubeadmControlPlane
 apiVersion: controlplane.cluster.x-k8s.io/v1beta1
@@ -123,18 +141,29 @@ spec:
       apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
       name: "${CLUSTER_NAME}-control-plane"
   kubeadmConfigSpec:
+    verbosity: 5
     initConfiguration:
       nodeRegistration:
         kubeletExtraArgs:
           cloud-provider: external
           provider-id: aws:///'{{ ds.meta_data.placement.availability_zone }}'/'{{ ds.meta_data.instance_id }}'
         name: '{{ ds.meta_data.local_hostname }}'
-    files:
     joinConfiguration:
       nodeRegistration:
         kubeletExtraArgs:
           cloud-provider: external
           provider-id: aws:///'{{ ds.meta_data.placement.availability_zone }}'/'{{ ds.meta_data.instance_id }}'
+    mounts:
+      - - xvdb
+        - /preload
+        - ext4
+        - "auto,exec,ro,nodev"
+    preKubeadmCommands:
+      - lsblk
+      - mount -t ext4 /dev/xvdb /preload
+      - mount
+      - ls -al /preload
+      - /preload/restore.sh
   version: "${KUBERNETES_VERSION}"
 ---
 apiVersion: v1
diff --git a/test/e2e/data/infrastructure-outscale/infrastructure-components.yaml b/test/e2e/data/infrastructure-outscale/infrastructure-components.yaml
diff --git a/test/e2e/data/shared/v1beta1/metadata.yaml b/test/e2e/data/shared/v1beta1/metadata.yaml
