Integrated microk8s as a target cluster

cmd/installer/main.go

@@ -32,11 +32,15 @@ import (
 )
 
 var (
-	criuImage      = flag.String("criu-image", "ghcr.io/ctrox/zeropod-criu:v4.2", "criu image to use.")
-	runtime        = flag.String("runtime", "containerd", "specifies which runtime to configure. containerd/k3s/rke2")
-	hostOptPath    = flag.String("host-opt-path", defaultOptPath, "path where zeropod binaries are stored on the host")
-	uninstall      = flag.Bool("uninstall", false, "uninstalls zeropod by cleaning up all the files the installer created")
-	installTimeout = flag.Duration("timeout", time.Minute, "duration the installer waits for the installation to complete")
+	criuImage           = flag.String("criu-image", "ghcr.io/ctrox/zeropod-criu:v4.2", "criu image to use.")
+	runtime             = flag.String("runtime", "containerd", "specifies which runtime to configure. containerd/k3s/rke2")
+	hostOptPath         = flag.String("host-opt-path", defaultOptPath, "path where zeropod binaries are stored on the host")
+	uninstall           = flag.Bool("uninstall", false, "uninstalls zeropod by cleaning up all the files the installer created")
+	installTimeout      = flag.Duration("timeout", time.Minute, "duration the installer waits for the installation to complete")
+	containerdSocket    = flag.String("containerd-socket", defaultContainerdSock, "path to the containerd socket")
+	containerdConfig    = flag.String("containerd-config", defaultContainerdConfigPath, "path to the containerd config file")
+	containerdNamespace = flag.String("containerd-namespace", defaultContainerdNamespace, "containerd namespace")
+	systemdCgroup       = flag.Bool("systemd-cgroup", true, "enbles systemd cgroup support")
 )
 
 type containerRuntime string
@@ -45,14 +49,16 @@ const (
 	runtimeContainerd containerRuntime = "containerd"
 	runtimeRKE2       containerRuntime = "rke2"
 	runtimeK3S        containerRuntime = "k3s"
+	runtimeMicroK8s   containerRuntime = "microk8s"
 
 	hostRoot                    = "/host"
 	binPath                     = "bin/"
 	criuConfigFile              = "/etc/criu/default.conf"
 	shimBinaryName              = "containerd-shim-zeropod-v2"
 	runtimePath                 = "/build/" + shimBinaryName
 	defaultContainerdConfigPath = "/etc/containerd/config.toml"
-	containerdSock              = "/run/containerd/containerd.sock"
+	defaultContainerdSock       = "/run/containerd/containerd.sock"
+	defaultContainerdNamespace  = "k8s"
 	configBackupSuffix          = ".original"
 	templateSuffix              = ".tmpl"
 	caSecretName                = "ca-cert"
@@ -92,7 +98,7 @@ network-lock skip
 
   [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.zeropod.options]
     # use systemd cgroup by default
-    SystemdCgroup = true
+    SystemdCgroup = %t
 `
 	configVersion2 = "version = 2"
 	runtimeConfig  = `
@@ -115,7 +121,7 @@ network-lock skip
 
   [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.zeropod.options]
     # use systemd cgroup by default
-    SystemdCgroup = true
+    SystemdCgroup = %t
 `
 )
 
@@ -167,7 +173,7 @@ func main() {
 }
 
 func installCriu(ctx context.Context) error {
-	client, err := containerd.New(containerdSock, containerd.WithDefaultNamespace("k8s"))
+	client, err := containerd.New(*containerdSocket, containerd.WithDefaultNamespace(*containerdNamespace))
 	if err != nil {
 		return err
 	}
@@ -233,7 +239,7 @@ func installRuntime(ctx context.Context, runtime containerRuntime) error {
 
 	restartRequired, err := configureContainerd(ctx, runtime)
 	if err != nil {
-		if restoreErr := restoreContainerdConfig(runtime, defaultContainerdConfigPath); restoreErr != nil {
+		if restoreErr := restoreContainerdConfig(runtime, *containerdConfig); restoreErr != nil {
 			return fmt.Errorf("unable to configure and restore containerd config: %w: %w", restoreErr, err)
 		}
 		return fmt.Errorf("unable to configure containerd: %w", err)
@@ -267,6 +273,8 @@ func installRuntime(ctx context.Context, runtime containerRuntime) error {
 		}
 
 		return nil
+	case runtimeMicroK8s:
+		return restartUnit(ctx, conn, "snap.microk8s.daemon-containerd.service")
 	}
 
 	return nil
@@ -283,7 +291,7 @@ func restartUnit(ctx context.Context, conn *dbus.Conn, service string) error {
 }
 
 func configureContainerd(ctx context.Context, runtime containerRuntime) (restartRequired bool, err error) {
-	client, err := containerd.New(containerdSock, containerd.WithDefaultNamespace("k8s"))
+	client, err := containerd.New(*containerdSocket, containerd.WithDefaultNamespace(*containerdNamespace))
 	if err != nil {
 		return false, fmt.Errorf("creating containerd client: %w", err)
 	}
@@ -294,9 +302,9 @@ func configureContainerd(ctx context.Context, runtime containerRuntime) (restart
 	}
 	log.Printf("configuring containerd %s", v.Version)
 	if strings.HasPrefix(v.Version, "1") || strings.HasPrefix(v.Version, "v1") {
-		return configureContainerdv1(ctx, runtime, defaultContainerdConfigPath)
+		return configureContainerdv1(ctx, runtime, *containerdConfig)
 	}
-	return configureContainerdv2(ctx, runtime, defaultContainerdConfigPath)
+	return configureContainerdv2(ctx, runtime, *containerdConfig)
 }
 
 func configureContainerdv2(ctx context.Context, runtime containerRuntime, containerdConfig string) (bool, error) {
@@ -394,7 +402,7 @@ func configureContainerdv1(ctx context.Context, runtime containerRuntime, contai
 		optPath = containerdOptPath
 	}
 
-	if _, err := fmt.Fprintf(cfg, runtimeConfig, strings.TrimSuffix(optPath, "/")); err != nil {
+	if _, err := fmt.Fprintf(cfg, runtimeConfig, strings.TrimSuffix(optPath, "/"), *systemdCgroup); err != nil {
 		return false, err
 	}
 
@@ -492,7 +500,7 @@ func writeZeropodRuntimeConfig(containerdConfig, optPath string, existingOpt boo
 	if version == 3 {
 		zeropodRuntimeConfig = runtimeConfigV3
 	}
-	zeropodRuntimeConfig = fmt.Sprintf(zeropodRuntimeConfig, strings.TrimSuffix(optPath, "/"))
+	zeropodRuntimeConfig = fmt.Sprintf(zeropodRuntimeConfig, strings.TrimSuffix(optPath, "/"), *systemdCgroup)
 	if !existingOpt {
 		zeropodRuntimeConfig = zeropodRuntimeConfig + fmt.Sprintf(optPlugin, optPath)
 	}
@@ -588,7 +596,7 @@ func optConfigured(ctx context.Context, containerdConfig string) (bool, string,
 }
 
 func optPath(ctx context.Context, runtime containerRuntime) string {
-	ok, path, err := optConfigured(ctx, containerdConfigFile(runtime, defaultContainerdConfigPath))
+	ok, path, err := optConfigured(ctx, containerdConfigFile(runtime, *containerdConfig))
 	if err != nil {
 		return defaultOptPath
 	}
@@ -643,7 +651,7 @@ func runUninstall(ctx context.Context, client kubernetes.Interface, runtime cont
 		return fmt.Errorf("removing opt path: %w", err)
 	}
 
-	if err := restoreContainerdConfig(runtime, defaultContainerdConfigPath); err != nil {
+	if err := restoreContainerdConfig(runtime, *containerdConfig); err != nil {
 		return err
 	}
 

cmd/installer/main_test.go

@@ -90,7 +90,28 @@ imports = [
   "runtime_zeropod.toml",
 ]
 `
-	containerdv1AlreadyConfigured = fullContainerdConfigV2 + runtimeConfig + `
+	containerdv1AlreadyConfigured = fullContainerdConfigV2 + `
+[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.zeropod]
+  runtime_type = "io.containerd.runc.v2"
+  runtime_path = "/opt/zeropod/bin/containerd-shim-zeropod-v2"
+  pod_annotations = [
+    "zeropod.ctrox.dev/ports-map",
+    "zeropod.ctrox.dev/container-names",
+    "zeropod.ctrox.dev/scaledown-duration",
+    "zeropod.ctrox.dev/disable-checkpointing",
+    "zeropod.ctrox.dev/pre-dump",
+    "zeropod.ctrox.dev/migrate",
+    "zeropod.ctrox.dev/live-migrate",
+    "zeropod.ctrox.dev/disable-probe-detection",
+    "zeropod.ctrox.dev/probe-buffer-size",
+    "zeropod.ctrox.dev/disable-migrate-data",
+    "io.containerd.runc.v2.group"
+  ]
+
+  [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.zeropod.options]
+    # use systemd cgroup by default
+    SystemdCgroup = true
+` + `
 [plugins."io.containerd.internal.v1.opt"]
   path = "/opt/zeropod"
 `
@@ -104,6 +125,7 @@ type testConfig struct {
 	newConfigSuffix        string
 	expectedOptPath        string
 	containerdv1           bool
+	systemdCgroup          *bool
 }
 
 func TestConfigureContainerd(t *testing.T) {
@@ -158,6 +180,11 @@ func TestConfigureContainerd(t *testing.T) {
 			expectedRestart:  true,
 			newConfigSuffix:  templateSuffix,
 		},
+		"microk8s config": {
+			containerdConfig: fullContainerdConfigV2,
+			runtime:          runtimeMicroK8s,
+			expectedRestart:  true,
+		},
 		"full config v1": {
 			containerdConfig: fullContainerdConfigV2,
 			runtime:          runtimeContainerd,
@@ -171,11 +198,22 @@ func TestConfigureContainerd(t *testing.T) {
 			expectedRestart:        false,
 			containerdv1:           true,
 		},
+		"systemd cgroup disabled": {
+			containerdConfig: fullContainerdConfigV2,
+			runtime:          runtimeContainerd,
+			expectedRestart:  true,
+			systemdCgroup:    &[]bool{false}[0],
+		},
 	} {
 		t.Run(name, func(t *testing.T) {
 			if tc.expectedOptPath == "" {
 				tc.expectedOptPath = defaultOptPath
 			}
+			if tc.systemdCgroup != nil {
+				original := *systemdCgroup
+				*systemdCgroup = *tc.systemdCgroup
+				defer func() { *systemdCgroup = original }()
+			}
 			assert := assert.New(t)
 			require := require.New(t)
 			configName := setupTestConfig(t, tc)

config/microk8s/kustomization.yaml

@@ -0,0 +1,28 @@
+resources:
+  - ../production
+patches:
+  - path: microk8s.yaml
+  - patch: |-
+      - op: add
+        path: /spec/template/spec/initContainers/0/args
+        value: []
+      - op: add
+        path: /spec/template/spec/initContainers/0/args/-
+        value: -runtime=microk8s
+      - op: add
+        path: /spec/template/spec/initContainers/0/args/-
+        value: -systemd-cgroup=false
+      - op: add
+        path: /spec/template/spec/initContainers/0/args/-
+        value: -containerd-socket=/run/containerd/containerd.sock
+      - op: add
+        path: /spec/template/spec/initContainers/0/args/-
+        value: -containerd-config=/etc/containerd/containerd-template.toml
+      - op: add
+        path: /spec/template/spec/initContainers/0/args/-
+        value: -containerd-namespace=k8s.io
+      - op: add
+        path: /spec/template/spec/containers/0/args/-
+        value: -probe-binary-name=kubelite
+    target:
+      kind: DaemonSet

config/microk8s/microk8s.yaml

@@ -0,0 +1,15 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: zeropod-node
+  namespace: zeropod-system
+spec:
+  template:
+    spec:
+      volumes:
+        - name: containerd-run
+          hostPath:
+            path: /var/snap/microk8s/common/run
+        - name: containerd-etc
+          hostPath:
+            path: /var/snap/microk8s/current/args

docs/development.md

@@ -33,3 +33,125 @@ tests can also be run but it's a bit more involved than just running `go test`
 since that requires `GOOS=linux`. You can use `make docker-test-e2e` to run
 the e2e tests within a docker container, so everything will be run on the
 linux podman VM.
+
+## Developing on MicroK8s
+
+To test changes from a local clone on MicroK8s:
+
+### 1. Build and Import Images
+
+```bash
+make build-installer build-manager TAG=dev
+docker save ghcr.io/ctrox/zeropod-installer:dev | microk8s ctr image import -
+docker save ghcr.io/ctrox/zeropod-manager:dev | microk8s ctr image import -
+```
+
+### 2. Update Kustomization
+
+Temporarily update `config/microk8s/kustomization.yaml` for local development:
+
+```yaml
+resources:
+  - ../production
+images:
+- name: ghcr.io/ctrox/zeropod-installer
+  newTag: dev
+- name: ghcr.io/ctrox/zeropod-manager
+  newTag: dev
+patches:
+  - path: microk8s.yaml
+  - patch: |-
+      - op: add
+        path: /spec/template/spec/initContainers/0/args
+        value: []
+      - op: add
+        path: /spec/template/spec/initContainers/0/args/-
+        value: -runtime=microk8s
+      - op: add
+        path: /spec/template/spec/initContainers/0/args/-
+        value: -systemd-cgroup=false
+      - op: add
+        path: /spec/template/spec/initContainers/0/args/-
+        value: -containerd-socket=/run/containerd/containerd.sock
+      - op: add
+        path: /spec/template/spec/initContainers/0/args/-
+        value: -containerd-config=/etc/containerd/containerd-template.toml
+      - op: add
+        path: /spec/template/spec/initContainers/0/args/-
+        value: -containerd-namespace=k8s.io
+      - op: add
+        path: /spec/template/spec/containers/0/args/-
+        value: -probe-binary-name=kubelite
+    target:
+      kind: DaemonSet
+  - patch: |-
+      - op: add
+        path: /spec/template/spec/initContainers/0/imagePullPolicy
+        value: IfNotPresent
+      - op: add
+        path: /spec/template/spec/containers/0/imagePullPolicy
+        value: IfNotPresent
+    target:
+      kind: DaemonSet
+```
+
+### 3. Deploy
+
+```bash
+kubectl apply -k config/microk8s
+```
+
+## Developing on K3s
+
+To test changes from a local clone on K3s:
+
+### 1. Build and Import Images
+
+```bash
+make build-installer build-manager TAG=dev
+docker save ghcr.io/ctrox/zeropod-installer:dev | sudo k3s ctr images import -
+docker save ghcr.io/ctrox/zeropod-manager:dev | sudo k3s ctr images import -
+```
+
+### 2. Update Kustomization
+
+Temporarily update `config/k3s/kustomization.yaml` for local development:
+
+```yaml
+resources:
+  - ../production
+images:
+- name: ghcr.io/ctrox/zeropod-installer
+  newTag: dev
+- name: ghcr.io/ctrox/zeropod-manager
+  newTag: dev
+patches:
+  - path: k3s.yaml
+  - patch: |-
+      - op: add
+        path: /spec/template/spec/initContainers/0/args/-
+        value: -runtime=k3s
+    target:
+      kind: DaemonSet
+  - patch: |-
+      - op: add
+        path: /spec/template/spec/containers/0/args/-
+        value: -probe-binary-name=k3s
+    target:
+      kind: DaemonSet
+  - patch: |-
+      - op: add
+        path: /spec/template/spec/initContainers/0/imagePullPolicy
+        value: IfNotPresent
+      - op: add
+        path: /spec/template/spec/containers/0/imagePullPolicy
+        value: IfNotPresent
+    target:
+      kind: DaemonSet
+```
+
+### 3. Deploy
+
+```bash
+sudo k3s kubectl apply -k config/k3s
+```