diff --git a/cmd/installer/main.go b/cmd/installer/main.go index 76314dd..9c505d9 100644 --- a/cmd/installer/main.go +++ b/cmd/installer/main.go @@ -45,6 +45,7 @@ const ( runtimeContainerd containerRuntime = "containerd" runtimeRKE2 containerRuntime = "rke2" runtimeK3S containerRuntime = "k3s" + runtimeK0S containerRuntime = "k0s" hostRoot = "/host" binPath = "bin/" @@ -53,6 +54,11 @@ const ( runtimePath = "/build/" + shimBinaryName defaultContainerdConfigPath = "/etc/containerd/config.toml" containerdSock = "/run/containerd/containerd.sock" + k0sContainerdDir = "/etc/k0s" + k0sContainerdConfigPath = k0sContainerdDir + "/containerd.toml" + k0sDropInDirName = "containerd.d" + k0sContainerdSock = "/run/k0s/containerd.sock" + k0sManagedSentinel = "k0s_managed=true" configBackupSuffix = ".original" templateSuffix = ".tmpl" caSecretName = "ca-cert" @@ -92,7 +98,7 @@ network-lock skip [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.zeropod.options] # use systemd cgroup by default - SystemdCgroup = true + SystemdCgroup = %t ` configVersion2 = "version = 2" runtimeConfig = ` @@ -115,7 +121,7 @@ network-lock skip [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.zeropod.options] # use systemd cgroup by default - SystemdCgroup = true + SystemdCgroup = %t ` ) @@ -139,7 +145,7 @@ func main() { os.Exit(0) } - if err := installCriu(ctx); err != nil { + if err := installCriu(ctx, containerRuntime(*runtime)); err != nil { log.Fatalf("error installing criu: %s", err) } @@ -166,8 +172,8 @@ func main() { log.Println("installer completed") } -func installCriu(ctx context.Context) error { - client, err := containerd.New(containerdSock, containerd.WithDefaultNamespace("k8s")) +func installCriu(ctx context.Context, runtime containerRuntime) error { + client, err := containerd.New(containerdSocketPath(runtime), containerd.WithDefaultNamespace("k8s")) if err != nil { return err } @@ -180,7 +186,7 @@ func installCriu(ctx context.Context) error { if err := client.Install( ctx, image, containerd.WithInstallLibs, containerd.WithInstallReplace, - containerd.WithInstallPath(optPath(ctx, containerRuntime(*runtime))), + containerd.WithInstallPath(optPath(ctx, runtime)), ); err != nil { return err } @@ -233,7 +239,8 @@ func installRuntime(ctx context.Context, runtime containerRuntime) error { restartRequired, err := configureContainerd(ctx, runtime) if err != nil { - if restoreErr := restoreContainerdConfig(runtime, defaultContainerdConfigPath); restoreErr != nil { + configPath := containerdConfigPathForRuntime(runtime) + if restoreErr := restoreContainerdConfig(runtime, configPath); restoreErr != nil { return fmt.Errorf("unable to configure and restore containerd config: %w: %w", restoreErr, err) } return fmt.Errorf("unable to configure containerd: %w", err) @@ -283,7 +290,11 @@ func restartUnit(ctx context.Context, conn *dbus.Conn, service string) error { } func configureContainerd(ctx context.Context, runtime containerRuntime) (restartRequired bool, err error) { - client, err := containerd.New(containerdSock, containerd.WithDefaultNamespace("k8s")) + if runtime == runtimeK0S { + return configureContainerdK0s(ctx, containerdConfigPathForRuntime(runtime)) + } + + client, err := containerd.New(containerdSocketPath(runtime), containerd.WithDefaultNamespace("k8s")) if err != nil { return false, fmt.Errorf("creating containerd client: %w", err) } @@ -293,10 +304,45 @@ func configureContainerd(ctx context.Context, runtime containerRuntime) (restart return false, fmt.Errorf("getting containerd version: %w", err) } log.Printf("configuring containerd %s", v.Version) + configPath := containerdConfigPathForRuntime(runtime) if strings.HasPrefix(v.Version, "1") || strings.HasPrefix(v.Version, "v1") { - return configureContainerdv1(ctx, runtime, defaultContainerdConfigPath) + return configureContainerdv1(ctx, runtime, configPath) + } + return configureContainerdv2(ctx, runtime, configPath) +} + +func configureContainerdK0s(ctx context.Context, containerdConfig string) (bool, error) { + if err := ensureK0sManaged(containerdConfig); err != nil { + return false, err + } + + dropInPath := zeropodRuntimeConfigPath(runtimeK0S, containerdConfig) + if _, err := os.Stat(dropInPath); err == nil { + log.Println("runtime already configured, refreshing zeropod drop-in") + } else if !errors.Is(err, os.ErrNotExist) { + return false, fmt.Errorf("checking zeropod drop-in: %w", err) + } + + conf := &config.Config{} + if err := config.LoadConfig(ctx, containerdConfig, conf); err != nil { + return false, fmt.Errorf("loading containerd config: %w", err) + } + + existingOpt, containerdOptPath, err := optConfigured(ctx, containerdConfig) + if err != nil { + return false, fmt.Errorf("could not check opt configuration: %w", err) + } + + optPath := *hostOptPath + if existingOpt { + optPath = containerdOptPath } - return configureContainerdv2(ctx, runtime, defaultContainerdConfigPath) + + if err := writeZeropodRuntimeConfig(runtimeK0S, containerdConfig, optPath, existingOpt, conf.Version, useSystemdCgroup(runtimeK0S)); err != nil { + return false, err + } + + return false, nil } func configureContainerdv2(ctx context.Context, runtime containerRuntime, containerdConfig string) (bool, error) { @@ -342,7 +388,7 @@ func configureContainerdv2(ctx context.Context, runtime containerRuntime, contai optPath = containerdOptPath } - if err := writeZeropodRuntimeConfig(containerdConfig, optPath, existingOpt, conf.Version); err != nil { + if err := writeZeropodRuntimeConfig(runtime, containerdConfig, optPath, existingOpt, conf.Version, useSystemdCgroup(runtime)); err != nil { return false, err } @@ -394,7 +440,8 @@ func configureContainerdv1(ctx context.Context, runtime containerRuntime, contai optPath = containerdOptPath } - if _, err := fmt.Fprintf(cfg, runtimeConfig, strings.TrimSuffix(optPath, "/")); err != nil { + configSnippet := fmt.Sprintf(runtimeConfig, strings.TrimSuffix(optPath, "/"), useSystemdCgroup(runtime)) + if _, err := fmt.Fprint(cfg, configSnippet); err != nil { return false, err } @@ -479,29 +526,44 @@ func zeropodImportConfigured(imports []string) bool { return false } -func zeropodRuntimeConfigPath(containerdConfig string) string { - return filepath.Join(filepath.Dir(containerdConfig), zeropodTomlName) +func zeropodRuntimeConfigPath(runtime containerRuntime, containerdConfig string) string { + dir := filepath.Dir(containerdConfig) + if runtime == runtimeK0S { + dir = filepath.Join(dir, k0sDropInDirName) + } + return filepath.Join(dir, zeropodTomlName) } func backupContainerdConfig(containerdConfig string) error { return copyConfig(containerdConfig, containerdConfig+configBackupSuffix) } -func writeZeropodRuntimeConfig(containerdConfig, optPath string, existingOpt bool, version int) error { - zeropodRuntimeConfig := fmt.Sprintf("%s\n%s", configVersion2, runtimeConfig) - if version == 3 { - zeropodRuntimeConfig = runtimeConfigV3 - } - zeropodRuntimeConfig = fmt.Sprintf(zeropodRuntimeConfig, strings.TrimSuffix(optPath, "/")) - if !existingOpt { - zeropodRuntimeConfig = zeropodRuntimeConfig + fmt.Sprintf(optPlugin, optPath) +func writeZeropodRuntimeConfig(runtime containerRuntime, containerdConfig, optPath string, existingOpt bool, version int, systemdCgroup bool) error { + zeropodRuntimeConfig := renderZeropodRuntimeConfig(optPath, existingOpt, version, systemdCgroup) + dest := zeropodRuntimeConfigPath(runtime, containerdConfig) + if runtime == runtimeK0S { + if err := os.MkdirAll(filepath.Dir(dest), 0755); err != nil { + return fmt.Errorf("ensuring k0s drop-in directory: %w", err) + } } - if err := os.WriteFile(zeropodRuntimeConfigPath(containerdConfig), []byte(zeropodRuntimeConfig), 0644); err != nil { + if err := os.WriteFile(dest, []byte(zeropodRuntimeConfig), 0644); err != nil { return fmt.Errorf("writing zeropod runtime config: %w", err) } return nil } +func renderZeropodRuntimeConfig(optPath string, existingOpt bool, version int, systemdCgroup bool) string { + trimmedOpt := strings.TrimSuffix(optPath, "/") + zeropodRuntimeConfig := fmt.Sprintf("%s\n%s", configVersion2, fmt.Sprintf(runtimeConfig, trimmedOpt, systemdCgroup)) + if version == 3 { + zeropodRuntimeConfig = fmt.Sprintf(runtimeConfigV3, trimmedOpt, systemdCgroup) + } + if !existingOpt { + zeropodRuntimeConfig += fmt.Sprintf(optPlugin, optPath) + } + return zeropodRuntimeConfig +} + func restoreContainerdConfig(runtime containerRuntime, containerdConfigPath string) error { if _, err := os.Stat(containerdConfigPath + configBackupSuffix); err != nil { if errors.Is(err, os.ErrNotExist) { @@ -549,6 +611,39 @@ func containerdConfigFile(runtime containerRuntime, containerdConfigPath string) return containerdConfigPath } +func containerdConfigPathForRuntime(runtime containerRuntime) string { + switch runtime { + case runtimeK0S: + return k0sContainerdConfigPath + default: + return defaultContainerdConfigPath + } +} + +func containerdSocketPath(runtime containerRuntime) string { + switch runtime { + case runtimeK0S: + return k0sContainerdSock + default: + return containerdSock + } +} + +func useSystemdCgroup(runtime containerRuntime) bool { + return runtime != runtimeK0S +} + +func ensureK0sManaged(containerdConfig string) error { + data, err := os.ReadFile(containerdConfig) + if err != nil { + return fmt.Errorf("reading containerd config: %w", err) + } + if !bytes.Contains(data, []byte(k0sManagedSentinel)) { + return fmt.Errorf("containerd config %s is not k0s managed", containerdConfig) + } + return nil +} + func copyConfig(from, to string) error { info, err := os.Stat(from) if err != nil { @@ -588,7 +683,7 @@ func optConfigured(ctx context.Context, containerdConfig string) (bool, string, } func optPath(ctx context.Context, runtime containerRuntime) string { - ok, path, err := optConfigured(ctx, containerdConfigFile(runtime, defaultContainerdConfigPath)) + ok, path, err := optConfigured(ctx, containerdConfigFile(runtime, containerdConfigPathForRuntime(runtime))) if err != nil { return defaultOptPath } @@ -643,7 +738,12 @@ func runUninstall(ctx context.Context, client kubernetes.Interface, runtime cont return fmt.Errorf("removing opt path: %w", err) } - if err := restoreContainerdConfig(runtime, defaultContainerdConfigPath); err != nil { + runtimeConfigPath := zeropodRuntimeConfigPath(runtime, containerdConfigPathForRuntime(runtime)) + if err := os.Remove(runtimeConfigPath); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("removing runtime config %s: %w", runtimeConfigPath, err) + } + + if err := restoreContainerdConfig(runtime, containerdConfigPathForRuntime(runtime)); err != nil { return err } diff --git a/cmd/installer/main_test.go b/cmd/installer/main_test.go index 1442150..be8a992 100644 --- a/cmd/installer/main_test.go +++ b/cmd/installer/main_test.go @@ -2,7 +2,10 @@ package main import ( "context" + "fmt" "os" + "path/filepath" + "strings" "testing" "github.com/containerd/containerd/v2/cmd/containerd/server/config" @@ -90,11 +93,12 @@ imports = [ "runtime_zeropod.toml", ] ` - containerdv1AlreadyConfigured = fullContainerdConfigV2 + runtimeConfig + ` +) + +var containerdv1AlreadyConfigured = fullContainerdConfigV2 + fmt.Sprintf(runtimeConfig, strings.TrimSuffix(defaultOptPath, "/"), true) + ` [plugins."io.containerd.internal.v1.opt"] path = "/opt/zeropod" ` -) type testConfig struct { containerdConfig string @@ -197,7 +201,7 @@ func TestConfigureContainerd(t *testing.T) { assert.NoError(config.LoadConfig(ctx, configName+tc.newConfigSuffix, conf)) if !tc.containerdv1 { - zeropodConfig, err := os.ReadFile(zeropodRuntimeConfigPath(configName)) + zeropodConfig, err := os.ReadFile(zeropodRuntimeConfigPath(tc.runtime, configName)) require.NoError(err) assert.NotEmpty(zeropodConfig) t.Log(string(zeropodConfig)) @@ -218,6 +222,34 @@ func TestConfigureContainerd(t *testing.T) { } } +func TestConfigureContainerdK0s(t *testing.T) { + t.Run("writes drop-in", func(t *testing.T) { + temp := t.TempDir() + cfg := filepath.Join(temp, "containerd.toml") + managedConfig := []byte("# " + k0sManagedSentinel + "\nversion = 2\n") + require.NoError(t, os.WriteFile(cfg, managedConfig, 0644)) + + restart, err := configureContainerdK0s(context.Background(), cfg) + require.NoError(t, err) + assert.False(t, restart) + + dropIn := filepath.Join(filepath.Dir(cfg), k0sDropInDirName, zeropodTomlName) + data, err := os.ReadFile(dropIn) + require.NoError(t, err) + assert.Contains(t, string(data), "runtime_type = \"io.containerd.runc.v2\"") + }) + + t.Run("requires managed config", func(t *testing.T) { + temp := t.TempDir() + cfg := filepath.Join(temp, "containerd.toml") + require.NoError(t, os.WriteFile(cfg, []byte("version = 2\n"), 0644)) + + _, err := configureContainerdK0s(context.Background(), cfg) + require.Error(t, err) + assert.Contains(t, err.Error(), "not k0s managed") + }) +} + func setupTestConfig(t *testing.T, tc testConfig) string { temp := t.TempDir() configFile, err := os.CreateTemp(temp, "containerd-config-*.toml") @@ -226,7 +258,7 @@ func setupTestConfig(t *testing.T, tc testConfig) string { if tc.preCreateZeropodConfig { if !tc.containerdv1 { - require.NoError(t, writeZeropodRuntimeConfig(configFile.Name(), tc.expectedOptPath, tc.expectedOptPath == "", 2)) + require.NoError(t, writeZeropodRuntimeConfig(tc.runtime, configFile.Name(), tc.expectedOptPath, tc.expectedOptPath == "", 2, useSystemdCgroup(tc.runtime))) } require.NoError(t, backupContainerdConfig(configFile.Name())) } diff --git a/config/k0s/k0s.yaml b/config/k0s/k0s.yaml new file mode 100644 index 0000000..c53d970 --- /dev/null +++ b/config/k0s/k0s.yaml @@ -0,0 +1,29 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: zeropod-node + namespace: zeropod-system +spec: + template: + spec: + volumes: + - name: containerd-etc + hostPath: + path: /etc/k0s + - name: containerd-run + hostPath: + path: /run/k0s + initContainers: + - name: installer + volumeMounts: + - name: containerd-etc + mountPath: /etc/k0s + - name: containerd-run + mountPath: /run/k0s + containers: + - name: manager + volumeMounts: + - name: containerd-etc + mountPath: /etc/k0s + - name: containerd-run + mountPath: /run/k0s diff --git a/config/k0s/kustomization.yaml b/config/k0s/kustomization.yaml new file mode 100644 index 0000000..b317585 --- /dev/null +++ b/config/k0s/kustomization.yaml @@ -0,0 +1,10 @@ +resources: + - ../production +patches: + - path: k0s.yaml + - patch: |- + - op: add + path: /spec/template/spec/initContainers/0/args/- + value: -runtime=k0s + target: + kind: DaemonSet