Add kubelet to sysadmin boxes

This also creates a generic kubelet profile outside kubernetes, and much
of kubernetes's kubelet resources have been replaced with this profile.
It's not a complete rewrite, but it's a start.

This should definitely be tested in a development environment before
merging.

On the way, this should also fix the naming collision between two
different prometheus node exporter packages by pinning to a specific
version.

manifests/profile/kubelet.pp

@@ -0,0 +1,54 @@
+# Copyright (c) 2023 The Regents of the University of Michigan.
+# All Rights Reserved. Licensed according to the terms of the Revised
+# BSD License. See LICENSE.txt for details.
+class nebula::profile::kubelet (
+  String $kubelet_version,
+  String $pod_manifest_path = "/etc/kubelet_manifests",
+  Boolean $use_pod_manifest_path = true,
+) {
+  include nebula::profile::networking::sysctl
+  include nebula::profile::containerd
+  include nebula::profile::kubernetes::apt
+
+  kmod::load { "overlay": }
+  kmod::load { "br_netfilter": }
+
+  file { "/etc/sysctl.d/kubelet.conf":
+    content => template("nebula/profile/kubernetes/kubelet_sysctl.conf.erb"),
+    notify  => Service["procps"],
+  }
+
+  package { "kubelet":
+    ensure  => $kubelet_version,
+    require => Apt::Source["kubernetes"],
+  }
+
+  apt::pin { "kubelet":
+    packages => ["kubelet"],
+    version  => $kubelet_version,
+  }
+
+  service { "kubelet":
+    ensure  => "running",
+    enable  => true,
+    require => Package["kubelet"],
+  }
+
+  if $use_pod_manifest_path {
+    file { "/etc/systemd/system/kubelet.service.d":
+      ensure => "directory",
+    }
+
+    file { "/etc/systemd/system/kubelet.service.d/20-containerd-and-manifest-dir.conf":
+      content => template("nebula/profile/kubelet/systemd.conf.erb"),
+      require => Package["kubelet"],
+      notify  => Exec["kubelet reload daemon"],
+    }
+
+    exec { 'kubelet reload daemon':
+      command     => "/bin/systemctl daemon-reload",
+      refreshonly => true,
+      notify      => Service["kubelet"],
+    }
+  }
+}

manifests/profile/kubernetes/kubelet.pp

@@ -3,9 +3,6 @@
 # BSD License. See LICENSE.txt for details.
 
 class nebula::profile::kubernetes::kubelet {
-  include nebula::profile::containerd
-  include nebula::profile::kubernetes::apt
-
   $cluster_name = lookup('nebula::profile::kubernetes::cluster')
   $cluster = lookup('nebula::profile::kubernetes::clusters')[$cluster_name]
 
@@ -36,29 +33,10 @@
     fail("You must set a kube api IP address for the cluster's gateway")
   }
 
-  kmod::load { 'br_netfilter': }
-
-  include nebula::profile::networking::sysctl
-  file { '/etc/sysctl.d/kubelet.conf':
-    content => template('nebula/profile/kubernetes/kubelet_sysctl.conf.erb'),
-    notify  => Service['procps'],
-  }
-
-  service { 'kubelet':
-    ensure  => 'running',
-    enable  => true,
-    require => Package['kubelet'],
-  }
-
-  package { 'kubelet':
-    ensure  => "${kubernetes_version}-00",
-    require => [Apt::Source['kubernetes']],
-  }
-
-  apt::pin { 'kubelet':
-    packages => ['kubelet'],
-    version  => "${kubernetes_version}-00",
-    priority => 999,
+  class { "nebula::profile::kubelet":
+    kubelet_version       => "${kubernetes_version}-00",
+    pod_manifest_path     => "/etc/kubernetes/manifests",
+    use_pod_manifest_path => false,
   }
 
   firewall {

manifests/profile/prometheus/exporter/node.pp

@@ -28,6 +28,7 @@
   $log_file = '/var/log/prometheus-node-exporter.log'
 
   include nebula::virtual::users
+  include nebula::profile::apt
   include nebula::profile::groups
   include nebula::subscriber::rsyslog
   include nebula::subscriber::systemctl_daemon_reload
@@ -78,6 +79,13 @@
     require => [User['prometheus'], File['/var/lib/prometheus/node-exporter']],
   }
 
+  if $version != undef {
+    apt::pin { 'prometheus-node-exporter':
+      packages => ['prometheus-node-exporter'],
+      version  => $version,
+    }
+  }
+
   file { '/var/lib/prometheus/node-exporter':
     ensure => 'directory',
     mode   => '2775',

manifests/role/sysadmin_box.pp

@@ -13,6 +13,7 @@
   include nebula::profile::users
   include nebula::profile::ruby
   include nebula::profile::root_ssh_private_keys
+  include nebula::profile::kubelet
 
   class { 'nebula::profile::puppet::query':
     ssl_group => 'sudo',

spec/classes/profile/kubelet_spec.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+# Copyright (c) 2023 The Regents of the University of Michigan.
+# All Rights Reserved. Licensed according to the terms of the Revised
+# BSD License. See LICENSE.txt for details.
+require 'spec_helper'
+
+describe 'nebula::profile::kubelet' do
+  on_supported_os.each do |os, os_facts|
+    context "on #{os}" do
+      let(:facts) { os_facts }
+      let(:params) { { kubelet_version: "invalid-example-version" } }
+
+      it { is_expected.to compile }
+
+      # Prerequisites according to kubernetes documentation:
+      # https://kubernetes.io/docs/setup/production-environment/container-runtimes/
+      it { is_expected.to contain_kmod__load("overlay") }
+      it { is_expected.to contain_kmod__load("br_netfilter") }
+      it { is_expected.to contain_file("/etc/sysctl.d/kubelet.conf").that_notifies("Service[procps]") }
+      ["net.bridge.bridge-nf-call-iptables",
+       "net.bridge.bridge-nf-call-ip6tables",
+       "net.ipv4.ip_forward"].each do |param|
+        it do
+          is_expected.to contain_file("/etc/sysctl.d/kubelet.conf")
+            .with_content(/^#{param} *= *1$/)
+        end
+      end
+
+      it { is_expected.to contain_service("containerd") }
+
+      it do
+        is_expected.to contain_apt__source("kubernetes")
+          .with_location("https://apt.kubernetes.io/")
+          .with_release("kubernetes-xenial")
+      end
+
+      it do
+        is_expected.to contain_package("kubelet")
+          .with_ensure("invalid-example-version")
+          .that_requires("Apt::Source[kubernetes]")
+      end
+
+      it do
+        is_expected.to contain_apt__pin("kubelet")
+          .with_packages(["kubelet"])
+          .with_version("invalid-example-version")
+      end
+
+      it do
+        is_expected.to contain_service("kubelet")
+          .with_ensure("running")
+          .with_enable(true)
+          .that_requires("Package[kubelet]")
+      end
+
+      context "with kubelet_version set to 1.2.3-00" do
+        let(:params) { { kubelet_version: "1.2.3-00" } }
+
+        it { is_expected.to contain_package("kubelet").with_ensure("1.2.3-00") }
+        it { is_expected.to contain_apt__pin("kubelet").with_version("1.2.3-00") }
+      end
+
+      it do
+        is_expected.to contain_exec("kubelet reload daemon")
+          .that_notifies("Service[kubelet]")
+          .with_refreshonly(true)
+          .with_command("/bin/systemctl daemon-reload")
+      end
+
+      it do
+        is_expected.to contain_file("/etc/systemd/system/kubelet.service.d")
+          .with_ensure("directory")
+      end
+
+      it do
+        is_expected.to contain_file("/etc/systemd/system/kubelet.service.d/20-containerd-and-manifest-dir.conf")
+          .that_requires("File[/etc/systemd/system/kubelet.service.d]")
+          .that_requires("Package[kubelet]")
+          .that_notifies("Exec[kubelet reload daemon]")
+      end
+
+      it do
+        is_expected.to contain_file("/etc/systemd/system/kubelet.service.d/20-containerd-and-manifest-dir.conf")
+          .with_content(/^Restart=always$/)
+      end
+
+      it do
+        # This is important because we're using this file to override
+        # the contents of the original systemd file. Without this empty
+        # line, systemd might ignore our preferred ExecStart.
+        is_expected.to contain_file("/etc/systemd/system/kubelet.service.d/20-containerd-and-manifest-dir.conf")
+          .with_content(/^ExecStart=$/)
+      end
+
+      it do
+        is_expected.to contain_file("/etc/systemd/system/kubelet.service.d/20-containerd-and-manifest-dir.conf")
+          .with_content(/^ExecStart=\/usr\/bin\/kubelet/)
+      end
+
+      ["--address=127.0.0.1",
+       "--pod-manifest-path=/etc/kubelet_manifests",
+       "--container-runtime=remote",
+       "--container-runtime-endpoint=unix:///run/containerd/containerd.sock",
+       "--cgroup-driver=systemd"].each do |param|
+        it do
+          is_expected.to contain_file("/etc/systemd/system/kubelet.service.d/20-containerd-and-manifest-dir.conf")
+            .with_content(/^ExecStart=.+ #{param}/)
+        end
+      end
+
+      context "with pod_manifest_path set to /tmp" do
+        let(:params) { { kubelet_version: "123", pod_manifest_path: "/tmp" } }
+
+        it do
+          is_expected.to contain_file("/etc/systemd/system/kubelet.service.d/20-containerd-and-manifest-dir.conf")
+            .with_content(/^ExecStart=.+ --pod-manifest-path=\/tmp/)
+        end
+      end
+
+      context "with use_pod_manifest_path set to false" do
+        let(:params) { { kubelet_version: "123", use_pod_manifest_path: false } }
+
+        it { is_expected.not_to contain_file("/etc/systemd/system/kubelet.service.d") }
+        it { is_expected.not_to contain_file("/etc/systemd/system/kubelet.service.d/20-containerd-and-manifest-dir.conf") }
+        it { is_expected.not_to contain_exec("kubelet reload daemon") }
+      end
+    end
+  end
+end

spec/classes/profile/prometheus/exporter/node_spec.rb

@@ -44,6 +44,30 @@
           .that_requires('File[/var/lib/prometheus/node-exporter]')
       end
 
+      context "with no version set" do
+        it { is_expected.not_to contain_apt__pin('prometheus-node-exporter') }
+
+        it do
+          is_expected.to contain_package('prometheus-node-exporter')
+            .with_ensure("installed")
+        end
+      end
+
+      context "with version set to v1.2.3" do
+        let(:params) { { version: "v1.2.3" } }
+
+        it do
+          is_expected.to contain_package('prometheus-node-exporter')
+            .with_ensure("v1.2.3")
+        end
+
+        it do
+          is_expected.to contain_apt__pin('prometheus-node-exporter')
+            .with_packages(["prometheus-node-exporter"])
+            .with_version("v1.2.3")
+        end
+      end
+
       it do
         is_expected.to contain_file('/var/lib/prometheus/node-exporter')
           .with_ensure('directory')

spec/fixtures/hiera/default.yaml

@@ -170,3 +170,5 @@ nebula::profile::falcon::cid: default-invalid-cid
 nebula::profile::tsm::servername: tsmserver
 nebula::profile::tsm::serveraddress: tsm.default.invalid
 nebula::jdk_version: '8'
+
+nebula::profile::kubelet::kubelet_version: default.invalid

templates/profile/kubelet/systemd.conf.erb

@@ -0,0 +1,5 @@
+# Managed by puppet (nebula/profile/kubelet/systemd.conf.erb)
+[Service]
+ExecStart=
+ExecStart=/usr/bin/kubelet --address=127.0.0.1 --pod-manifest-path=<%= @pod_manifest_path %> --cgroup-driver=systemd --container-runtime=remote --container-runtime-endpoint=unix:///run/containerd/containerd.sock
+Restart=always