Refactor module options and support rootless k3s & nix-snapshotter

.github/workflows/ci.yml

@@ -42,6 +42,9 @@ jobs:
         test:
           - snapshotter
           - kubernetes
+          - k3s
+          - k3s-external
+          - k3s-rootless
     runs-on: nix-snapshotter-runner
     needs: [lint, build]
     if: contains(github.event.pull_request.labels.*.name, 'ok-to-test')

README.md

@@ -71,15 +71,25 @@ redis-cli -p 30000 ping
 Or you can try running in rootless mode:
 
 ```sh
-nix run ".#vm"
+nix run "github:pdtpartners/nix-snapshotter#vm-rootless"
 nixos login: rootless # (Ctrl-a then x to quit)
 Password: rootless
 
-# Running pkgs.hello image with nix-snapshotter
-nerdctl run ghcr.io/pdtpartners/hello
+# `nerdctl run` with rootless k3s containerd currently not supported yet
+# See: https://github.com/containerd/nerdctl/issues/2831
+#
+# If rootless kubernetes not needed, `nerdctl run` does work with rootless
+# containerd + nix-snapshotter.
+
+# Running `pkgs.redis` image with kubernetes & nix-snapshotter
+kubectl apply -f /etc/kubernetes/redis/
+
+# Wait a few seconds... 
+watch kubectl get pods
 
-# Rootless kubernetes not supported yet.
-# See: https://github.com/k3s-io/k3s/pull/8279
+# And a kubernetes service will be ready to forward port 30000 to the redis
+# pod, so you can test it out with a `ping` command
+redis-cli -p 30000 ping
 ```
 
 ## Installation
@@ -128,15 +138,18 @@ easy installation.
           }
           ({ pkgs, ... }: {
             # (1) Import home-manager module.
-            imports = [ nix-snapshotter.homeModules.nix-snapshotter-rootless ];
+            imports = [ nix-snapshotter.homeModules.default ];
 
             # (2) Add overlay.
             nixpkgs.overlays = [ nix-snapshotter.overlays.default ];
 
             # (3) Enable service.
+            virtualisation.containerd.rootless = {
+              enable = true;
+              nixSnapshotterIntegration = true;
+            };
             services.nix-snapshotter.rootless = {
               enable = true;
-              setContainerdSnapshotter = true;
             };
 
             # (4) Add a containerd CLI like nerdctl.
@@ -161,22 +174,24 @@ easy installation.
 
   in {
     imports = [
-      ./hardware-configuration.nix
       # (1) Import home-manager module.
-      nix-snapshotter.nixosModules.default
+      nix-snapshotter.homeModules.default
     ];
 
-    # (2) Add overlay.
+    // # (2) Add overlay.
     nixpkgs.overlays = [ nix-snapshotter.overlays.default ];
 
     # (3) Enable service.
-    services.nix-snapshotter = {
+    virtualisation.containerd.rootless = {
+      enable = true;
+      nixSnapshotterIntegration = true;
+    };
+    services.nix-snapshotter.rootless = {
       enable = true;
-      setContainerdSnapshotter = true;
     };
 
     # (4) Add a containerd CLI like nerdctl.
-    environment.systemPackages = [ pkgs.nerdctl ];
+    home.packages = [ pkgs.nerdctl ];
   }
   ```
   </details>
@@ -209,9 +224,12 @@ easy installation.
             nixpkgs.overlays = [ nix-snapshotter.overlays.default ];
 
             # (3) Enable service.
+            virtualisation.containerd = {
+              enable = true;
+              nixSnapshotterIntegration = true;
+            };
             services.nix-snapshotter = {
               enable = true;
-              setContainerdSnapshotter = true;
             };
 
             # (4) Add a containerd CLI like nerdctl.
@@ -245,9 +263,12 @@ easy installation.
     nixpkgs.overlays = [ nix-snapshotter.overlays.default ];
 
     # (3) Enable service.
+    virtualisation.containerd = {
+      enable = true;
+      nixSnapshotterIntegration = true;
+    };
     services.nix-snapshotter = {
       enable = true;
-      setContainerdSnapshotter = true;
     };
 
     # (4) Add a containerd CLI like nerdctl.

modules/common/containerd-rootless.nix

@@ -8,6 +8,8 @@ let
 
   cfg = config.virtualisation.containerd.rootless;
 
+  ctrd-lib = config.virtualisation.containerd.lib;
+
   settingsFormat = pkgs.formats.toml {};
 
   configFile = settingsFormat.generate "containerd.toml" cfg.settings;
@@ -40,8 +42,10 @@ let
       exec nsenter \
         --no-fork \
         --preserve-credentials \
-        -m -n -U \
-        -t "$pid" \
+        --mount \
+        --net \
+        --user \
+        --target "$pid" \
         -- "$@"
     '';
   };
@@ -54,7 +58,8 @@ let
 
   mkRootlessContainerdService = cfg:
     let
-      containerdArgs = lib.concatStringsSep " " (lib.cli.toGNUCommandLine {} cfg.args);
+      containerdArgs =
+        lib.concatStringsSep " " (lib.cli.toGNUCommandLine {} cfg.args);
 
       containerd-rootless = makeProg {
         name = "containerd-rootless";
@@ -148,7 +153,18 @@ let
     };
 
 in {
+  imports = [
+    ./containerd.nix
+  ];
+
   options.virtualisation.containerd.rootless = {
+    inherit (ctrd-lib.options)
+      nixSnapshotterIntegration
+      setAddress
+      setNamespace
+      setSnapshotter
+    ;
+
     settings = lib.mkOption {
       type = settingsFormat.type;
       default = {};
@@ -175,15 +191,6 @@ in {
 
     package = mkPackageOptionMD pkgs "containerd" { };
 
-    setSocketVariable = mkOption {
-      type = types.bool;
-      default = false;
-      description = lib.mdDoc ''
-        Point {command}`CONTAINERD_ADDRESS` to rootless containerd for normal
-        users by default.
-      '';
-    };
-
     bindMounts = lib.mkOption {
       type = types.attrsOf (types.submodule bindMountOpts);
       example = lib.literalExpression ''
@@ -208,36 +215,52 @@ in {
     };
 
     lib = mkOption {
+      type = types.attrs;
       description = lib.mdDoc "Common functions for the containerd modules.";
       default = {
-        inherit mkRootlessContainerdService;
+        inherit
+          mkRootlessContainerdService
+        ;
       };
-      type = types.attrs;
       internal = true;
     };
   };
 
-  config = lib.mkIf cfg.enable {
-    virtualisation.containerd.rootless = {
-      inherit nsenter;
+  config = lib.mkIf cfg.enable (lib.mkMerge [
+    {
+      virtualisation.containerd.rootless = {
+        inherit nsenter;
 
-      args = {
-        config = toString containerdConfigChecked;
-      };
+        args.config = toString containerdConfigChecked;
+
+        setAddress = lib.mkDefault "$XDG_RUNTIME_DIR/containerd/containerd.sock";
+
+        settings = {
+          version = 2;
+          plugins."io.containerd.grpc.v1.cri" = {
+            cni.bin_dir = lib.mkOptionDefault "${pkgs.cni-plugins}/bin";
+          };
+        };
 
-      settings = {
-        version = 2;
-        plugins."io.containerd.grpc.v1.cri" = {
-         cni.bin_dir = lib.mkOptionDefault "${pkgs.cni-plugins}/bin";
+        bindMounts = {
+          "$XDG_RUNTIME_DIR/containerd".mountPoint = "/run/containerd";
+          "$XDG_DATA_HOME/containerd".mountPoint = "/var/lib/containerd";
+          "$XDG_DATA_HOME/cni".mountPoint = "/var/lib/cni";
+          "$XDG_CONFIG_HOME/cni".mountPoint = "/etc/cni";
         };
       };
+    }
+    (lib.mkIf cfg.nixSnapshotterIntegration {
+      virtualisation.containerd.rootless = {
+        setSnapshotter = lib.mkDefault "nix";
+
+        settings = ctrd-lib.mkNixSnapshotterSettings;
 
-      bindMounts = {
-        "$XDG_RUNTIME_DIR/containerd".mountPoint = "/run/containerd";
-        "$XDG_DATA_HOME/containerd".mountPoint = "/var/lib/containerd";
-        "$XDG_DATA_HOME/cni".mountPoint = "/var/lib/cni";
-        "$XDG_CONFIG_HOME/cni".mountPoint = "/etc/cni";
+        bindMounts = {
+          "$XDG_RUNTIME_DIR/nix-snapshotter".mountPoint = "/run/nix-snapshotter";
+          "$XDG_DATA_HOME/nix-snapshotter".mountPoint = "/var/lib/containerd/io.containerd.snapshotter.v1.nix";
+        };
       };
-    };
-  };
+    })
+  ]);
 }

modules/common/containerd.nix

@@ -0,0 +1,94 @@
+{ config, pkgs, lib, ... }:
+let
+  inherit (lib)
+    mkEnableOption
+    mkOption
+    types
+  ;
+
+  inherit (pkgs.go)
+    GOOS
+    GOARCH
+  ;
+
+  cfg = config.virtualisation.containerd;
+
+  options = {
+    k3sIntegration = mkEnableOption "K3s integration";
+
+    nixSnapshotterIntegration = mkEnableOption "Nix snapshotter integration";
+
+    setAddress = mkOption {
+      type = types.str;
+      default = "/run/containerd/containerd.sock";
+      description = lib.mdDoc ''
+        Set the default containerd address via environment variable
+        `CONTAINERD_ADDRESS`.
+      '';
+    };
+
+    setNamespace = mkOption {
+      type = types.str;
+      default = "default";
+      description = lib.mdDoc ''
+        Set the default containerd namespace via environment variable
+        `CONTAINERD_NAMESPACE`.
+      '';
+    };
+
+    setSnapshotter = mkOption {
+      type = types.str;
+      default = "";
+      description = lib.mdDoc ''
+        Set the default containerd snapshotter via environment variable
+        `CONTAINERD_SNAPSHOTTER`.
+      '';
+    };
+  };
+
+  mkNixSnapshotterSettings = {
+    plugins."io.containerd.grpc.v1.cri".containerd = {
+      snapshotter = "nix";
+    };
+
+    plugins."io.containerd.transfer.v1.local".unpack_config = [{
+      platform = "${GOOS}/${GOARCH}";
+      snapshotter = "nix";
+    }];
+
+    proxy_plugins.nix = {
+      type = "snapshot";
+      address = "/run/nix-snapshotter/nix-snapshotter.sock";
+    };
+  };
+
+in {
+  options.virtualisation.containerd = {
+    inherit (options)
+      k3sIntegration
+      nixSnapshotterIntegration
+      setAddress
+      setNamespace
+      setSnapshotter
+    ;
+
+    lib = mkOption {
+      type = types.attrs;
+      description = lib.mdDoc "Common functions for containerd.";
+      default = {
+        inherit
+          options
+          mkNixSnapshotterSettings
+        ;
+      };
+      internal = true;
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    virtualisation.containerd = lib.mkIf cfg.nixSnapshotterIntegration {
+      setSnapshotter = lib.mkDefault "nix";
+      settings = mkNixSnapshotterSettings;
+    };
+  };
+}