Skip to content

Commit

Permalink
Refactor module options and support rootless k3s & nix-snapshotter
Browse files Browse the repository at this point in the history
- Bumps version to 0.2.0
- Separate `nix run .#vm` and `nix run .#vm-rootless`
- Add integration tests for k3s, k3s-external, k3s-rootless
- Separate preload-container service into independent modules
- Pin k3s to v1.27.9+k3s1 with patches to enable embedded nix-snapshotter
- Add k8sResources flake perSystem output and plumb into specialArgs

- Services `preload-containerd` & `preload-containerd.rootless`:

  ```nix
  config.services.preload-containerd = {
    enable = true;
    targets = [{
      archives = [ pkgs.nix-snapshotter.buildImage { /* ... */ } ];
      namespace = "k8s.io";
      address = "/run/k3s/containerd/containerd.sock";
    }];
  };
  ```

- New options for `k3s` & new service `k3s.rootless`:

  ```nix
  config.services.k3s = {
    enable = true;
    # Sets the snapshotter for embedded containerd.
    snapshotter = "nix";
    # Sets KUBECONFIG env var to point to k3s.
    setKubeConfig = true;
    # Sets CONTAINERD_* env vars to point to k3s embedded containerd.
    setEmbeddedContainerd = true;
  }
  ```

- New options for `containerd` & `containerd.rootless`:

  ```nix
  config.virtualisation.containerd = {
    enable = true;
    # Enable integration with nix-snapshotter.
    nixSnapshotterIntegration = true;
    # Set the CONTAINERD_* env vars, but also set automatically by
    # `nixSnapshotterIntegration` or by `services.k3s.setEmbeddedContainerd`.
    setAddress = "/run/containerd/containerd.sock";
    setNamespace = "default";
    setSnapshotter = "nix";
  }
  ```

- New option only for NixOS module `containerd`:

  ```nix
  config.virtualisation.containerd = {
    enable = true;
    # Enable integration with k3s. This is mutually exclusive with setting
    # `services.k3s.snapshotter` and `services.k3s.setEmbeddedContainerd`.
    k3sIntegration = true;
  };
  ```

- Removed `options.services.nix-snapshotter.setContainerdSnapshotter`

  ```nix
  # v0.1.x
  services.nix-snapshotter = {
    enable = true;
    setContainerdSnapshotter = true;
  };

  # v0.2.0 (same for rootless)
  virtualisation.containerd = {
    enable = true;
    nixSnapshotterIntegration = true;
  };
  services.nix-snapshotter = {
    enable = true;
  };
  ```

- Removed `options.services.nix-snapshotter.preloadContainerdImages`

  ```nix
  # v0.1.x
  services.nix-snapshotter = {
    enable = true;
    preloadContainerdImages = [ pkgs.nix-snapshotter.buildImage { /* ... */ } ];
  };

  # v0.2.0 (same for rootless)
  virtualisation.containerd = {
    enable = true;
    nixSnapshotterIntegration = true;
  }
  services.nix-snapshotter = {
    enable = true;
  };
  services.preload-containerd = {
    targets = [{
      archives = [ pkgs.nix-snapshotter.buildImage { /* ... */ } ];
    }];
  };
  ```
  • Loading branch information
elpdt852 committed Feb 16, 2024
1 parent 8d2197e commit d91481f
Show file tree
Hide file tree
Showing 46 changed files with 1,616 additions and 736 deletions.
3 changes: 3 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,9 @@ jobs:
test:
- snapshotter
- kubernetes
- k3s
- k3s-external
- k3s-rootless
runs-on: nix-snapshotter-runner
needs: [lint, build]
if: contains(github.event.pull_request.labels.*.name, 'ok-to-test')
Expand Down
51 changes: 36 additions & 15 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -71,15 +71,25 @@ redis-cli -p 30000 ping
Or you can try running in rootless mode:

```sh
nix run ".#vm"
nix run "github:pdtpartners/nix-snapshotter#vm-rootless"
nixos login: rootless # (Ctrl-a then x to quit)
Password: rootless

# Running pkgs.hello image with nix-snapshotter
nerdctl run ghcr.io/pdtpartners/hello
# `nerdctl run` with rootless k3s containerd currently not supported yet
# See: https://github.com/containerd/nerdctl/issues/2831
#
# If rootless kubernetes not needed, `nerdctl run` does work with rootless
# containerd + nix-snapshotter.

# Running `pkgs.redis` image with kubernetes & nix-snapshotter
kubectl apply -f /etc/kubernetes/redis/

# Wait a few seconds...
watch kubectl get pods

# Rootless kubernetes not supported yet.
# See: https://github.com/k3s-io/k3s/pull/8279
# And a kubernetes service will be ready to forward port 30000 to the redis
# pod, so you can test it out with a `ping` command
redis-cli -p 30000 ping
```

## Installation
Expand Down Expand Up @@ -128,15 +138,18 @@ easy installation.
}
({ pkgs, ... }: {
# (1) Import home-manager module.
imports = [ nix-snapshotter.homeModules.nix-snapshotter-rootless ];
imports = [ nix-snapshotter.homeModules.default ];
# (2) Add overlay.
nixpkgs.overlays = [ nix-snapshotter.overlays.default ];
# (3) Enable service.
virtualisation.containerd.rootless = {
enable = true;
nixSnapshotterIntegration = true;
};
services.nix-snapshotter.rootless = {
enable = true;
setContainerdSnapshotter = true;
};
# (4) Add a containerd CLI like nerdctl.
Expand All @@ -161,22 +174,24 @@ easy installation.
in {
imports = [
./hardware-configuration.nix
# (1) Import home-manager module.
nix-snapshotter.nixosModules.default
nix-snapshotter.homeModules.default
];
# (2) Add overlay.
// # (2) Add overlay.
nixpkgs.overlays = [ nix-snapshotter.overlays.default ];
# (3) Enable service.
services.nix-snapshotter = {
virtualisation.containerd.rootless = {
enable = true;
nixSnapshotterIntegration = true;
};
services.nix-snapshotter.rootless = {
enable = true;
setContainerdSnapshotter = true;
};
# (4) Add a containerd CLI like nerdctl.
environment.systemPackages = [ pkgs.nerdctl ];
home.packages = [ pkgs.nerdctl ];
}
```
</details>
Expand Down Expand Up @@ -209,9 +224,12 @@ easy installation.
nixpkgs.overlays = [ nix-snapshotter.overlays.default ];
# (3) Enable service.
virtualisation.containerd = {
enable = true;
nixSnapshotterIntegration = true;
};
services.nix-snapshotter = {
enable = true;
setContainerdSnapshotter = true;
};
# (4) Add a containerd CLI like nerdctl.
Expand Down Expand Up @@ -245,9 +263,12 @@ easy installation.
nixpkgs.overlays = [ nix-snapshotter.overlays.default ];
# (3) Enable service.
virtualisation.containerd = {
enable = true;
nixSnapshotterIntegration = true;
};
services.nix-snapshotter = {
enable = true;
setContainerdSnapshotter = true;
};
# (4) Add a containerd CLI like nerdctl.
Expand Down
85 changes: 54 additions & 31 deletions modules/common/containerd-rootless.nix
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ let

cfg = config.virtualisation.containerd.rootless;

ctrd-lib = config.virtualisation.containerd.lib;

settingsFormat = pkgs.formats.toml {};

configFile = settingsFormat.generate "containerd.toml" cfg.settings;
Expand Down Expand Up @@ -40,8 +42,10 @@ let
exec nsenter \
--no-fork \
--preserve-credentials \
-m -n -U \
-t "$pid" \
--mount \
--net \
--user \
--target "$pid" \
-- "$@"
'';
};
Expand All @@ -54,7 +58,8 @@ let

mkRootlessContainerdService = cfg:
let
containerdArgs = lib.concatStringsSep " " (lib.cli.toGNUCommandLine {} cfg.args);
containerdArgs =
lib.concatStringsSep " " (lib.cli.toGNUCommandLine {} cfg.args);

containerd-rootless = makeProg {
name = "containerd-rootless";
Expand Down Expand Up @@ -148,7 +153,18 @@ let
};

in {
imports = [
./containerd.nix
];

options.virtualisation.containerd.rootless = {
inherit (ctrd-lib.options)
nixSnapshotterIntegration
setAddress
setNamespace
setSnapshotter
;

settings = lib.mkOption {
type = settingsFormat.type;
default = {};
Expand All @@ -175,15 +191,6 @@ in {

package = mkPackageOptionMD pkgs "containerd" { };

setSocketVariable = mkOption {
type = types.bool;
default = false;
description = lib.mdDoc ''
Point {command}`CONTAINERD_ADDRESS` to rootless containerd for normal
users by default.
'';
};

bindMounts = lib.mkOption {
type = types.attrsOf (types.submodule bindMountOpts);
example = lib.literalExpression ''
Expand All @@ -208,36 +215,52 @@ in {
};

lib = mkOption {
type = types.attrs;
description = lib.mdDoc "Common functions for the containerd modules.";
default = {
inherit mkRootlessContainerdService;
inherit
mkRootlessContainerdService
;
};
type = types.attrs;
internal = true;
};
};

config = lib.mkIf cfg.enable {
virtualisation.containerd.rootless = {
inherit nsenter;
config = lib.mkIf cfg.enable (lib.mkMerge [
{
virtualisation.containerd.rootless = {
inherit nsenter;

args = {
config = toString containerdConfigChecked;
};
args.config = toString containerdConfigChecked;

setAddress = lib.mkDefault "$XDG_RUNTIME_DIR/containerd/containerd.sock";

settings = {
version = 2;
plugins."io.containerd.grpc.v1.cri" = {
cni.bin_dir = lib.mkOptionDefault "${pkgs.cni-plugins}/bin";
};
};

settings = {
version = 2;
plugins."io.containerd.grpc.v1.cri" = {
cni.bin_dir = lib.mkOptionDefault "${pkgs.cni-plugins}/bin";
bindMounts = {
"$XDG_RUNTIME_DIR/containerd".mountPoint = "/run/containerd";
"$XDG_DATA_HOME/containerd".mountPoint = "/var/lib/containerd";
"$XDG_DATA_HOME/cni".mountPoint = "/var/lib/cni";
"$XDG_CONFIG_HOME/cni".mountPoint = "/etc/cni";
};
};
}
(lib.mkIf cfg.nixSnapshotterIntegration {
virtualisation.containerd.rootless = {
setSnapshotter = lib.mkDefault "nix";

settings = ctrd-lib.mkNixSnapshotterSettings;

bindMounts = {
"$XDG_RUNTIME_DIR/containerd".mountPoint = "/run/containerd";
"$XDG_DATA_HOME/containerd".mountPoint = "/var/lib/containerd";
"$XDG_DATA_HOME/cni".mountPoint = "/var/lib/cni";
"$XDG_CONFIG_HOME/cni".mountPoint = "/etc/cni";
bindMounts = {
"$XDG_RUNTIME_DIR/nix-snapshotter".mountPoint = "/run/nix-snapshotter";
"$XDG_DATA_HOME/nix-snapshotter".mountPoint = "/var/lib/containerd/io.containerd.snapshotter.v1.nix";
};
};
};
};
})
]);
}
94 changes: 94 additions & 0 deletions modules/common/containerd.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
{ config, pkgs, lib, ... }:
let
inherit (lib)
mkEnableOption
mkOption
types
;

inherit (pkgs.go)
GOOS
GOARCH
;

cfg = config.virtualisation.containerd;

options = {
k3sIntegration = mkEnableOption "K3s integration";

nixSnapshotterIntegration = mkEnableOption "Nix snapshotter integration";

setAddress = mkOption {
type = types.str;
default = "/run/containerd/containerd.sock";
description = lib.mdDoc ''
Set the default containerd address via environment variable
`CONTAINERD_ADDRESS`.
'';
};

setNamespace = mkOption {
type = types.str;
default = "default";
description = lib.mdDoc ''
Set the default containerd namespace via environment variable
`CONTAINERD_NAMESPACE`.
'';
};

setSnapshotter = mkOption {
type = types.str;
default = "";
description = lib.mdDoc ''
Set the default containerd snapshotter via environment variable
`CONTAINERD_SNAPSHOTTER`.
'';
};
};

mkNixSnapshotterSettings = {
plugins."io.containerd.grpc.v1.cri".containerd = {
snapshotter = "nix";
};

plugins."io.containerd.transfer.v1.local".unpack_config = [{
platform = "${GOOS}/${GOARCH}";
snapshotter = "nix";
}];

proxy_plugins.nix = {
type = "snapshot";
address = "/run/nix-snapshotter/nix-snapshotter.sock";
};
};

in {
options.virtualisation.containerd = {
inherit (options)
k3sIntegration
nixSnapshotterIntegration
setAddress
setNamespace
setSnapshotter
;

lib = mkOption {
type = types.attrs;
description = lib.mdDoc "Common functions for containerd.";
default = {
inherit
options
mkNixSnapshotterSettings
;
};
internal = true;
};
};

config = lib.mkIf cfg.enable {
virtualisation.containerd = lib.mkIf cfg.nixSnapshotterIntegration {
setSnapshotter = lib.mkDefault "nix";
settings = mkNixSnapshotterSettings;
};
};
}
Loading

0 comments on commit d91481f

Please sign in to comment.