podvm: add scratch-space logic

This adds the configuration for an encrypted scratch space on an mkosi
image. At bootup a /dev/sda4 partition will be created and encrypted
with LUKS using an ephemeral key.

The partition will use the space available on the image volume. By
default the qcow2 image has 100mb allocated for this space. This amount
of space will only work for very small images, hence we do not mount the
scratch space to `/run/kata-container` by default.

If the kata-agent service units encounters a `/run/peerpod/mount-scratch` file
it will mount the encrypted partition `/dev/sda4` to `/run/kata-containers`.

This file is provisioned by `process-user-data`, configured by the CAA
daemonset.

Signed-off-by: Magnus Kulke <[email protected]>

src/cloud-api-adaptor/cmd/cloud-api-adaptor/main.go

@@ -125,7 +125,7 @@ func (cfg *daemonConfig) Setup() (cmd.Starter, error) {
 		flags.StringVar(&cfg.serverConfig.Initdata, "initdata", "", "Default initdata for all Pods")
 		flags.BoolVar(&cfg.serverConfig.EnableCloudConfigVerify, "cloud-config-verify", false, "Enable cloud config verify - should use it for production")
 		flags.IntVar(&cfg.serverConfig.PeerPodsLimitPerNode, "peerpods-limit-per-node", 10, "peer pods limit per node (default=10)")
-		flags.Uint64Var(&cfg.serverConfig.DiskSize, "disk-size", 0, "Disk size in GB. Default is 0, which implies the default image disk size")
+		flags.Uint64Var(&cfg.serverConfig.RootVolumeSize, "root-volume-size", 0, "Root volume size in GB. Default is 0, which implies the default image disk size")
 
 		cloud.ParseCmd(flags)
 	})

src/cloud-api-adaptor/entrypoint.sh

@@ -30,7 +30,6 @@ optionals+=""
 [[ "${SECURE_COMMS_PP_OUTBOUNDS}" ]] && optionals+="-secure-comms-pp-outbounds ${SECURE_COMMS_PP_OUTBOUNDS} "
 [[ "${SECURE_COMMS_KBS_ADDR}" ]] && optionals+="-secure-comms-kbs ${SECURE_COMMS_KBS_ADDR} "
 [[ "${PEERPODS_LIMIT_PER_NODE}" ]] && optionals+="-peerpods-limit-per-node ${PEERPODS_LIMIT_PER_NODE} "
-[[ "${DISK_SIZE}" ]] && optionals+="-disk-size ${DISK_SIZE} "
 
 test_vars() {
     for i in "$@"; do
@@ -78,6 +77,7 @@ azure() {
     [[ "${TAGS}" ]] && optionals+="-tags ${TAGS} " # Custom tags applied to pod vm
     [[ "${ENABLE_SECURE_BOOT}" == "true" ]] && optionals+="-enable-secure-boot "
     [[ "${USE_PUBLIC_IP}" == "true" ]] && optionals+="-use-public-ip "
+    [[ "${ROOT_VOLUME_SIZE}" ]] && optionals+="-root-volume-size ${ROOT_VOLUME_SIZE} " # OS disk size in GB
 
     set -x
     exec cloud-api-adaptor azure \

src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go

@@ -64,8 +64,7 @@ type ServerConfig struct {
 	SecureCommsPpOutbounds  string
 	SecureCommsKbsAddress   string
 	PeerPodsLimitPerNode    int
-	UseEncryptedDisk        bool
-	DiskSize                uint64
+	RootVolumeSize          uint64
 }
 
 var logger = log.New(log.Writer(), "[adaptor/cloud] ", log.LstdFlags|log.Lmsgprefix)
@@ -227,8 +226,8 @@ func (s *cloudService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (r
 	resources := util.GetPodVMResourcesFromAnnotation(req.Annotations)
 
 	// Add disk size to resources if set
-	if s.serverConfig.DiskSize > 0 {
-		resources.Storage = int64(s.serverConfig.DiskSize)
+	if s.serverConfig.RootVolumeSize > 0 {
+		resources.Storage = int64(s.serverConfig.RootVolumeSize)
 	}
 
 	// Get Pod VM image from annotations
@@ -337,8 +336,8 @@ func (s *cloudService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (r
 		},
 	}
 
-	if s.serverConfig.DiskSize > 0 {
-		// Write an empty file to indicate that we want to use  available space as sandbox storage
+	if s.serverConfig.RootVolumeSize > 0 {
+		// Write an empty file to indicate that we want to use available space as sandbox storage
 		cloudConfig.WriteFiles = append(cloudConfig.WriteFiles, cloudinit.WriteFile{
 			Path:    UseScratchPath,
 			Content: "",

src/cloud-api-adaptor/podvm-mkosi/Makefile

@@ -76,6 +76,7 @@ else
 	touch resources/buildBootableImage
 	nix develop ..#podvm-mkosi --command mkosi --environment=VARIANT_ID=production
 	qemu-img convert -f raw -O qcow2 build/system.raw build/podvm-$(PODVM_DISTRO)-$(ARCH).qcow2
+	qemu-img resize build/podvm-$(PODVM_DISTRO)-$(ARCH).qcow2 +100M
 endif
 
 PHONY: image-debug
@@ -97,6 +98,7 @@ else
 	touch resources/buildBootableImage
 	nix develop ..#podvm-mkosi --command mkosi --environment=VARIANT_ID=debug
 	qemu-img convert -f raw -O qcow2 build/system.raw build/podvm-$(PODVM_DISTRO)-$(ARCH).qcow2
+	qemu-img resize build/podvm-$(PODVM_DISTRO)-$(ARCH).qcow2 +100M
 endif
 
 PHONY: image-container

src/cloud-api-adaptor/podvm-mkosi/mkosi.presets/system/mkosi.conf.d/fedora.conf

@@ -7,7 +7,7 @@ Release=40
 
 [Content]
 CleanPackageMetadata=true
-SkeletonTrees=../../resources/binaries-tree
+SkeletonTrees=../../mkosi.skeleton-rootfs,../../resources/binaries-tree,
 Packages=
     kernel
     kernel-core
@@ -23,6 +23,7 @@ Packages=
     iptables
     afterburn
     neofetch
+    e2fsprogs
 
 RemoveFiles=/etc/issue
 RemoveFiles=/etc/issue.net

src/cloud-api-adaptor/podvm-mkosi/mkosi.skeleton-rootfs/etc/crypttab

@@ -0,0 +1 @@
+scratch /dev/disk/by-label/scratch - try-empty-password

src/cloud-api-adaptor/podvm-mkosi/mkosi.skeleton-rootfs/usr/lib/systemd/system/kata-agent.service.d/10-override.conf

@@ -0,0 +1,2 @@
+[Service]
+ExecStartPre=sh -c '[[ ! -f /run/peerpod/mount-scratch ]] || mount /dev/mapper/scratch /run/kata-containers'

src/cloud-api-adaptor/podvm-mkosi/mkosi.skeleton/usr/lib/repart.d/30-scratch.conf

@@ -0,0 +1,5 @@
+[Partition]
+Type=linux-generic
+Label=scratch
+Encrypt=key-file
+Format=ext4