diff --git a/cmd/installer/main.go b/cmd/installer/main.go index 76314dd..a42faa7 100644 --- a/cmd/installer/main.go +++ b/cmd/installer/main.go @@ -32,11 +32,15 @@ import ( ) var ( - criuImage = flag.String("criu-image", "ghcr.io/ctrox/zeropod-criu:v4.2", "criu image to use.") - runtime = flag.String("runtime", "containerd", "specifies which runtime to configure. containerd/k3s/rke2") - hostOptPath = flag.String("host-opt-path", defaultOptPath, "path where zeropod binaries are stored on the host") - uninstall = flag.Bool("uninstall", false, "uninstalls zeropod by cleaning up all the files the installer created") - installTimeout = flag.Duration("timeout", time.Minute, "duration the installer waits for the installation to complete") + criuImage = flag.String("criu-image", "ghcr.io/ctrox/zeropod-criu:v4.2", "criu image to use.") + runtime = flag.String("runtime", "containerd", "specifies which runtime to configure. containerd/k3s/rke2") + hostOptPath = flag.String("host-opt-path", defaultOptPath, "path where zeropod binaries are stored on the host") + uninstall = flag.Bool("uninstall", false, "uninstalls zeropod by cleaning up all the files the installer created") + installTimeout = flag.Duration("timeout", time.Minute, "duration the installer waits for the installation to complete") + containerdSocket = flag.String("containerd-socket", defaultContainerdSock, "path to the containerd socket") + containerdConfig = flag.String("containerd-config", defaultContainerdConfigPath, "path to the containerd config file") + containerdNamespace = flag.String("containerd-namespace", defaultContainerdNamespace, "containerd namespace") + systemdCgroup = flag.Bool("systemd-cgroup", true, "enbles systemd cgroup support") ) type containerRuntime string @@ -45,6 +49,7 @@ const ( runtimeContainerd containerRuntime = "containerd" runtimeRKE2 containerRuntime = "rke2" runtimeK3S containerRuntime = "k3s" + runtimeMicroK8s containerRuntime = "microk8s" hostRoot = "/host" binPath = "bin/" @@ -52,7 +57,8 @@ const ( shimBinaryName = "containerd-shim-zeropod-v2" runtimePath = "/build/" + shimBinaryName defaultContainerdConfigPath = "/etc/containerd/config.toml" - containerdSock = "/run/containerd/containerd.sock" + defaultContainerdSock = "/run/containerd/containerd.sock" + defaultContainerdNamespace = "k8s" configBackupSuffix = ".original" templateSuffix = ".tmpl" caSecretName = "ca-cert" @@ -92,7 +98,7 @@ network-lock skip [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.zeropod.options] # use systemd cgroup by default - SystemdCgroup = true + SystemdCgroup = %t ` configVersion2 = "version = 2" runtimeConfig = ` @@ -115,7 +121,7 @@ network-lock skip [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.zeropod.options] # use systemd cgroup by default - SystemdCgroup = true + SystemdCgroup = %t ` ) @@ -167,7 +173,7 @@ func main() { } func installCriu(ctx context.Context) error { - client, err := containerd.New(containerdSock, containerd.WithDefaultNamespace("k8s")) + client, err := containerd.New(*containerdSocket, containerd.WithDefaultNamespace(*containerdNamespace)) if err != nil { return err } @@ -233,7 +239,7 @@ func installRuntime(ctx context.Context, runtime containerRuntime) error { restartRequired, err := configureContainerd(ctx, runtime) if err != nil { - if restoreErr := restoreContainerdConfig(runtime, defaultContainerdConfigPath); restoreErr != nil { + if restoreErr := restoreContainerdConfig(runtime, *containerdConfig); restoreErr != nil { return fmt.Errorf("unable to configure and restore containerd config: %w: %w", restoreErr, err) } return fmt.Errorf("unable to configure containerd: %w", err) @@ -267,6 +273,8 @@ func installRuntime(ctx context.Context, runtime containerRuntime) error { } return nil + case runtimeMicroK8s: + return restartUnit(ctx, conn, "snap.microk8s.daemon-containerd.service") } return nil @@ -283,7 +291,7 @@ func restartUnit(ctx context.Context, conn *dbus.Conn, service string) error { } func configureContainerd(ctx context.Context, runtime containerRuntime) (restartRequired bool, err error) { - client, err := containerd.New(containerdSock, containerd.WithDefaultNamespace("k8s")) + client, err := containerd.New(*containerdSocket, containerd.WithDefaultNamespace(*containerdNamespace)) if err != nil { return false, fmt.Errorf("creating containerd client: %w", err) } @@ -294,9 +302,9 @@ func configureContainerd(ctx context.Context, runtime containerRuntime) (restart } log.Printf("configuring containerd %s", v.Version) if strings.HasPrefix(v.Version, "1") || strings.HasPrefix(v.Version, "v1") { - return configureContainerdv1(ctx, runtime, defaultContainerdConfigPath) + return configureContainerdv1(ctx, runtime, *containerdConfig) } - return configureContainerdv2(ctx, runtime, defaultContainerdConfigPath) + return configureContainerdv2(ctx, runtime, *containerdConfig) } func configureContainerdv2(ctx context.Context, runtime containerRuntime, containerdConfig string) (bool, error) { @@ -394,7 +402,7 @@ func configureContainerdv1(ctx context.Context, runtime containerRuntime, contai optPath = containerdOptPath } - if _, err := fmt.Fprintf(cfg, runtimeConfig, strings.TrimSuffix(optPath, "/")); err != nil { + if _, err := fmt.Fprintf(cfg, runtimeConfig, strings.TrimSuffix(optPath, "/"), *systemdCgroup); err != nil { return false, err } @@ -492,7 +500,7 @@ func writeZeropodRuntimeConfig(containerdConfig, optPath string, existingOpt boo if version == 3 { zeropodRuntimeConfig = runtimeConfigV3 } - zeropodRuntimeConfig = fmt.Sprintf(zeropodRuntimeConfig, strings.TrimSuffix(optPath, "/")) + zeropodRuntimeConfig = fmt.Sprintf(zeropodRuntimeConfig, strings.TrimSuffix(optPath, "/"), *systemdCgroup) if !existingOpt { zeropodRuntimeConfig = zeropodRuntimeConfig + fmt.Sprintf(optPlugin, optPath) } @@ -588,7 +596,7 @@ func optConfigured(ctx context.Context, containerdConfig string) (bool, string, } func optPath(ctx context.Context, runtime containerRuntime) string { - ok, path, err := optConfigured(ctx, containerdConfigFile(runtime, defaultContainerdConfigPath)) + ok, path, err := optConfigured(ctx, containerdConfigFile(runtime, *containerdConfig)) if err != nil { return defaultOptPath } @@ -643,7 +651,7 @@ func runUninstall(ctx context.Context, client kubernetes.Interface, runtime cont return fmt.Errorf("removing opt path: %w", err) } - if err := restoreContainerdConfig(runtime, defaultContainerdConfigPath); err != nil { + if err := restoreContainerdConfig(runtime, *containerdConfig); err != nil { return err } diff --git a/cmd/installer/main_test.go b/cmd/installer/main_test.go index 1442150..5236f3e 100644 --- a/cmd/installer/main_test.go +++ b/cmd/installer/main_test.go @@ -90,7 +90,28 @@ imports = [ "runtime_zeropod.toml", ] ` - containerdv1AlreadyConfigured = fullContainerdConfigV2 + runtimeConfig + ` + containerdv1AlreadyConfigured = fullContainerdConfigV2 + ` +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.zeropod] + runtime_type = "io.containerd.runc.v2" + runtime_path = "/opt/zeropod/bin/containerd-shim-zeropod-v2" + pod_annotations = [ + "zeropod.ctrox.dev/ports-map", + "zeropod.ctrox.dev/container-names", + "zeropod.ctrox.dev/scaledown-duration", + "zeropod.ctrox.dev/disable-checkpointing", + "zeropod.ctrox.dev/pre-dump", + "zeropod.ctrox.dev/migrate", + "zeropod.ctrox.dev/live-migrate", + "zeropod.ctrox.dev/disable-probe-detection", + "zeropod.ctrox.dev/probe-buffer-size", + "zeropod.ctrox.dev/disable-migrate-data", + "io.containerd.runc.v2.group" + ] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.zeropod.options] + # use systemd cgroup by default + SystemdCgroup = true +` + ` [plugins."io.containerd.internal.v1.opt"] path = "/opt/zeropod" ` @@ -104,6 +125,7 @@ type testConfig struct { newConfigSuffix string expectedOptPath string containerdv1 bool + systemdCgroup *bool } func TestConfigureContainerd(t *testing.T) { @@ -158,6 +180,11 @@ func TestConfigureContainerd(t *testing.T) { expectedRestart: true, newConfigSuffix: templateSuffix, }, + "microk8s config": { + containerdConfig: fullContainerdConfigV2, + runtime: runtimeMicroK8s, + expectedRestart: true, + }, "full config v1": { containerdConfig: fullContainerdConfigV2, runtime: runtimeContainerd, @@ -171,11 +198,22 @@ func TestConfigureContainerd(t *testing.T) { expectedRestart: false, containerdv1: true, }, + "systemd cgroup disabled": { + containerdConfig: fullContainerdConfigV2, + runtime: runtimeContainerd, + expectedRestart: true, + systemdCgroup: &[]bool{false}[0], + }, } { t.Run(name, func(t *testing.T) { if tc.expectedOptPath == "" { tc.expectedOptPath = defaultOptPath } + if tc.systemdCgroup != nil { + original := *systemdCgroup + *systemdCgroup = *tc.systemdCgroup + defer func() { *systemdCgroup = original }() + } assert := assert.New(t) require := require.New(t) configName := setupTestConfig(t, tc) diff --git a/config/microk8s/kustomization.yaml b/config/microk8s/kustomization.yaml new file mode 100644 index 0000000..75b6521 --- /dev/null +++ b/config/microk8s/kustomization.yaml @@ -0,0 +1,28 @@ +resources: + - ../production +patches: + - path: microk8s.yaml + - patch: |- + - op: add + path: /spec/template/spec/initContainers/0/args + value: [] + - op: add + path: /spec/template/spec/initContainers/0/args/- + value: -runtime=microk8s + - op: add + path: /spec/template/spec/initContainers/0/args/- + value: -systemd-cgroup=false + - op: add + path: /spec/template/spec/initContainers/0/args/- + value: -containerd-socket=/run/containerd/containerd.sock + - op: add + path: /spec/template/spec/initContainers/0/args/- + value: -containerd-config=/etc/containerd/containerd-template.toml + - op: add + path: /spec/template/spec/initContainers/0/args/- + value: -containerd-namespace=k8s.io + - op: add + path: /spec/template/spec/containers/0/args/- + value: -probe-binary-name=kubelite + target: + kind: DaemonSet diff --git a/config/microk8s/microk8s.yaml b/config/microk8s/microk8s.yaml new file mode 100644 index 0000000..aab418c --- /dev/null +++ b/config/microk8s/microk8s.yaml @@ -0,0 +1,15 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: zeropod-node + namespace: zeropod-system +spec: + template: + spec: + volumes: + - name: containerd-run + hostPath: + path: /var/snap/microk8s/common/run + - name: containerd-etc + hostPath: + path: /var/snap/microk8s/current/args diff --git a/docs/development.md b/docs/development.md index a34de49..319a52a 100644 --- a/docs/development.md +++ b/docs/development.md @@ -33,3 +33,125 @@ tests can also be run but it's a bit more involved than just running `go test` since that requires `GOOS=linux`. You can use `make docker-test-e2e` to run the e2e tests within a docker container, so everything will be run on the linux podman VM. + +## Developing on MicroK8s + +To test changes from a local clone on MicroK8s: + +### 1. Build and Import Images + +```bash +make build-installer build-manager TAG=dev +docker save ghcr.io/ctrox/zeropod-installer:dev | microk8s ctr image import - +docker save ghcr.io/ctrox/zeropod-manager:dev | microk8s ctr image import - +``` + +### 2. Update Kustomization + +Temporarily update `config/microk8s/kustomization.yaml` for local development: + +```yaml +resources: + - ../production +images: +- name: ghcr.io/ctrox/zeropod-installer + newTag: dev +- name: ghcr.io/ctrox/zeropod-manager + newTag: dev +patches: + - path: microk8s.yaml + - patch: |- + - op: add + path: /spec/template/spec/initContainers/0/args + value: [] + - op: add + path: /spec/template/spec/initContainers/0/args/- + value: -runtime=microk8s + - op: add + path: /spec/template/spec/initContainers/0/args/- + value: -systemd-cgroup=false + - op: add + path: /spec/template/spec/initContainers/0/args/- + value: -containerd-socket=/run/containerd/containerd.sock + - op: add + path: /spec/template/spec/initContainers/0/args/- + value: -containerd-config=/etc/containerd/containerd-template.toml + - op: add + path: /spec/template/spec/initContainers/0/args/- + value: -containerd-namespace=k8s.io + - op: add + path: /spec/template/spec/containers/0/args/- + value: -probe-binary-name=kubelite + target: + kind: DaemonSet + - patch: |- + - op: add + path: /spec/template/spec/initContainers/0/imagePullPolicy + value: IfNotPresent + - op: add + path: /spec/template/spec/containers/0/imagePullPolicy + value: IfNotPresent + target: + kind: DaemonSet +``` + +### 3. Deploy + +```bash +kubectl apply -k config/microk8s +``` + +## Developing on K3s + +To test changes from a local clone on K3s: + +### 1. Build and Import Images + +```bash +make build-installer build-manager TAG=dev +docker save ghcr.io/ctrox/zeropod-installer:dev | sudo k3s ctr images import - +docker save ghcr.io/ctrox/zeropod-manager:dev | sudo k3s ctr images import - +``` + +### 2. Update Kustomization + +Temporarily update `config/k3s/kustomization.yaml` for local development: + +```yaml +resources: + - ../production +images: +- name: ghcr.io/ctrox/zeropod-installer + newTag: dev +- name: ghcr.io/ctrox/zeropod-manager + newTag: dev +patches: + - path: k3s.yaml + - patch: |- + - op: add + path: /spec/template/spec/initContainers/0/args/- + value: -runtime=k3s + target: + kind: DaemonSet + - patch: |- + - op: add + path: /spec/template/spec/containers/0/args/- + value: -probe-binary-name=k3s + target: + kind: DaemonSet + - patch: |- + - op: add + path: /spec/template/spec/initContainers/0/imagePullPolicy + value: IfNotPresent + - op: add + path: /spec/template/spec/containers/0/imagePullPolicy + value: IfNotPresent + target: + kind: DaemonSet +``` + +### 3. Deploy + +```bash +sudo k3s kubectl apply -k config/k3s +``` diff --git a/docs/getting_started.md b/docs/getting_started.md index 33103aa..69e7727 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -34,10 +34,10 @@ kubectl apply -k https://github.com/ctrox/zeropod/config/production kubectl apply -k https://github.com/ctrox/zeropod/config/gke ``` -> ⚠️⚠️⚠️ For k3s and rke2, the initial installation needs to restart the -> k3s/k3s-agent or rke2-server/rke2-agent services, since it's not possible to +> ⚠️⚠️⚠️ For k3s, rke2 and microk8s, the initial installation needs to restart the +> k3s/k3s-agent, rke2-server/rke2-agent or snap services, since it's not possible to > just restart Containerd. This might lead to restarts of other workloads on -> each targeted node depending on the k3s/rke2 version. +> each targeted node depending on the k3s/rke2/microk8s version. ```bash # k3s: @@ -45,6 +45,9 @@ kubectl apply -k https://github.com/ctrox/zeropod/config/k3s # rke2: kubectl apply -k https://github.com/ctrox/zeropod/config/rke2 + +# microk8s: +kubectl apply -k https://github.com/ctrox/zeropod/config/microk8s ``` By default, zeropod will only be installed on nodes with the label @@ -67,7 +70,38 @@ Now you can create workloads which make use of zeropod. ```bash # create an example pod which makes use of zeropod -kubectl apply -f https://github.com/ctrox/zeropod/config/examples/nginx.yaml +kubectl apply -f https://raw.githubusercontent.com/ctrox/zeropod/refs/heads/main/config/examples/nginx.yaml +``` + +### Verifying Checkpoint and Restore + +The nginx example has a `scaledown-duration` of 10 seconds. After no external +traffic for 10 seconds, the container will be checkpointed (frozen) to disk. + +**1. Watch the manager logs:** +```bash +kubectl logs -f -n zeropod-system -l app.kubernetes.io/name=zeropod-node -c manager +``` + +Look for status events: +- `"phase":1` means RUNNING +- `"phase":0` means SCALED_DOWN (checkpointed) + +**2. Trigger a restore:** + +Once you see `phase:0`, the container is checkpointed. Send a request to wake it: +```bash +POD_IP=$(kubectl get pod -l app=nginx -o jsonpath='{.items[0].status.podIP}') +curl $POD_IP +``` + +The manager logs should show `phase:1` again, indicating the container was +restored from checkpoint. The curl request will succeed as soon as the +container is ready. + +**3. Verify checkpoint data on disk:** +```bash +sudo ls -la /var/lib/zeropod/i/ ``` Depending on your cluster setup, none of the predefined configs might not