AEIM-2592 - Kubernetes cluster with bootstrapping utilities

This adds five new roles:

1.  kubernetes::primary_gateway: load balancer, dns server, and nat
    router for the cluster. All traffic in and out of the cluster goes
    through this gateway.

2.  kubernetes::backup_gateway: almost identical to the primary gateway,
    but only takes its place when the primary gateway stops responding.
    Between the two, we have a highly available load balancer, dns
    server, and nat router.

3.  kubernetes::etcd: the etcd nodes have no direct awareness of
    kubernetes but are ready to be used as a data storage cluster.

4.  kubernetes::controller: the controller nodes make up a highly
    available cluster of kubernetes nodes with the apiserver,
    controller-manager, and scheduler. They use the etcd cluster as
    their data store.

5.  kubernetes::worker: the worker nodes have little responsibility on
    their own. They do whatever the scheduler tells them to do.

.rubocop.yml

@@ -114,3 +114,5 @@ RSpec/VerifiedDoubles:
   Enabled: false # should be able to re-enable once we remove our own function mocking code
 RSpec/NamedSubject:
   Enabled: false # would love to enable, but rspec-puppet plays by its own rules
+Style/NumericLiterals:
+  Enabled: false # all our long numbers are ports, and 8080 should not look like 8_080

lib/puppet/functions/ip_from_cidr.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+# Copyright (c) 2020 The Regents of the University of Michigan.
+# All Rights Reserved. Licensed according to the terms of the Revised
+# BSD License. See LICENSE.txt for details.
+require 'ipaddr'
+
+Puppet::Functions.create_function(:ip_from_cidr) do
+  dispatch :run do
+    required_param 'String',  :cidr
+    required_param 'Integer', :index
+    return_type 'String'
+  end
+
+  def run(cidr, index)
+    base = IPAddr.new(cidr)
+    result = IPAddr.new(base.to_i + index, base.family)
+    raise(ArgumentError, "#{index} too large to fit in #{cidr}") unless base.include? result
+    result.to_s
+  end
+end

manifests/profile/kubernetes/apt.pp

@@ -0,0 +1,15 @@
+# Copyright (c) 2020 The Regents of the University of Michigan.
+# All Rights Reserved. Licensed according to the terms of the Revised
+# BSD License. See LICENSE.txt for details.
+
+class nebula::profile::kubernetes::apt {
+  apt::source { 'kubernetes':
+    location => 'https://apt.kubernetes.io/',
+    release  => 'kubernetes-xenial',
+    repos    => 'main',
+    key      => {
+      'id'     => '54A647F9048D5688D7DA2ABE6A030B21BA07F4FB',
+      'source' => 'https://packages.cloud.google.com/apt/doc/apt-key.gpg',
+    },
+  }
+}

manifests/profile/kubernetes/bootstrap/destination.pp

@@ -0,0 +1,16 @@
+# Copyright (c) 2020 The Regents of the University of Michigan.
+# All Rights Reserved. Licensed according to the terms of the Revised
+# BSD License. See LICENSE.txt for details.
+
+class nebula::profile::kubernetes::bootstrap::destination {
+  include nebula::profile::kubernetes::bootstrap::user
+
+  $cluster_name = lookup('nebula::profile::kubernetes::cluster')
+  $cluster = lookup('nebula::profile::kubernetes::clusters')[$cluster_name]
+  $keys = pick($cluster['bootstrap_keys'], lookup('nebula::profile::kubernetes::bootstrap_keys'))
+
+  file { '/var/lib/kubeadm_bootstrap/.ssh/authorized_keys':
+    owner   => 'kubeadm_bootstrap',
+    content => $keys['public'],
+  }
+}

manifests/profile/kubernetes/bootstrap/etcd_config.pp

@@ -0,0 +1,23 @@
+# Copyright (c) 2020 The Regents of the University of Michigan.
+# All Rights Reserved. Licensed according to the terms of the Revised
+# BSD License. See LICENSE.txt for details.
+
+class nebula::profile::kubernetes::bootstrap::etcd_config {
+  include nebula::profile::kubernetes::kubelet
+
+  file { '/etc/systemd/system/kubelet.service.d/20-etcd-service-manager.conf':
+    content => template('nebula/profile/kubernetes/bootstrap/etcd/systemd.conf.erb'),
+    require => Package['kubelet'],
+    notify  => Exec['kubelet reload daemon'],
+  }
+
+  file { '/etc/systemd/system/kubelet.service.d':
+    ensure => 'directory',
+  }
+
+  exec { 'kubelet reload daemon':
+    command     => '/bin/systemctl daemon-reload',
+    refreshonly => true,
+    notify      => Service['kubelet'],
+  }
+}

manifests/profile/kubernetes/bootstrap/source.pp

@@ -0,0 +1,22 @@
+# Copyright (c) 2020 The Regents of the University of Michigan.
+# All Rights Reserved. Licensed according to the terms of the Revised
+# BSD License. See LICENSE.txt for details.
+
+class nebula::profile::kubernetes::bootstrap::source {
+  include nebula::profile::kubernetes::bootstrap::user
+
+  $cluster_name = lookup('nebula::profile::kubernetes::cluster')
+  $cluster = lookup('nebula::profile::kubernetes::clusters')[$cluster_name]
+  $keys = pick($cluster['bootstrap_keys'], lookup('nebula::profile::kubernetes::bootstrap_keys'))
+
+  file { '/var/lib/kubeadm_bootstrap/.ssh/id_rsa.pub':
+    owner   => 'kubeadm_bootstrap',
+    content => $keys['public'],
+  }
+
+  file { '/var/lib/kubeadm_bootstrap/.ssh/id_rsa':
+    owner   => 'kubeadm_bootstrap',
+    mode    => '0600',
+    content => $keys['private'],
+  }
+}

manifests/profile/kubernetes/bootstrap/user.pp

@@ -0,0 +1,20 @@
+# Copyright (c) 2020 The Regents of the University of Michigan.
+# All Rights Reserved. Licensed according to the terms of the Revised
+# BSD License. See LICENSE.txt for details.
+
+class nebula::profile::kubernetes::bootstrap::user {
+  user { 'kubeadm_bootstrap':
+    home => '/var/lib/kubeadm_bootstrap',
+  }
+
+  file { '/var/lib/kubeadm_bootstrap':
+    ensure => 'directory',
+    owner  => 'kubeadm_bootstrap',
+  }
+
+  file { '/var/lib/kubeadm_bootstrap/.ssh':
+    ensure => 'directory',
+    owner  => 'kubeadm_bootstrap',
+    mode   => '0700',
+  }
+}

manifests/profile/kubernetes/destination_port/api.pp

@@ -0,0 +1,14 @@
+# Copyright (c) 2020 The Regents of the University of Michigan.
+# All Rights Reserved. Licensed according to the terms of the Revised
+# BSD License. See LICENSE.txt for details.
+
+class nebula::profile::kubernetes::destination_port::api {
+  $cluster_name = lookup('nebula::profile::kubernetes::cluster')
+
+  @@concat_fragment { "haproxy kubernetes api ${::hostname}":
+    target  => '/etc/haproxy/services.d/api.cfg',
+    order   => '02',
+    content => "  server ${::hostname} ${::ipaddress}:6443 check\n",
+    tag     => "${cluster_name}_haproxy_kubernetes_api",
+  }
+}

manifests/profile/kubernetes/destination_port/etcd.pp

@@ -0,0 +1,14 @@
+# Copyright (c) 2020 The Regents of the University of Michigan.
+# All Rights Reserved. Licensed according to the terms of the Revised
+# BSD License. See LICENSE.txt for details.
+
+class nebula::profile::kubernetes::destination_port::etcd {
+  $cluster_name = lookup('nebula::profile::kubernetes::cluster')
+
+  @@concat_fragment { "haproxy kubernetes etcd ${::hostname}":
+    target  => '/etc/haproxy/services.d/etcd.cfg',
+    order   => '02',
+    content => "  server ${::hostname} ${::ipaddress}:2379 check\n",
+    tag     => "${cluster_name}_haproxy_kubernetes_etcd",
+  }
+}

manifests/profile/kubernetes/destination_port/http.pp

@@ -0,0 +1,14 @@
+# Copyright (c) 2020 The Regents of the University of Michigan.
+# All Rights Reserved. Licensed according to the terms of the Revised
+# BSD License. See LICENSE.txt for details.
+
+class nebula::profile::kubernetes::destination_port::http {
+  $cluster_name = lookup('nebula::profile::kubernetes::cluster')
+
+  @@concat_fragment { "haproxy kubernetes http ${::hostname}":
+    target  => '/etc/haproxy/services.d/http.cfg',
+    order   => '02',
+    content => "  server ${::hostname} ${::ipaddress}:30080 check\n",
+    tag     => "${cluster_name}_haproxy_kubernetes_http",
+  }
+}

manifests/profile/kubernetes/destination_port/http_alt.pp

@@ -0,0 +1,14 @@
+# Copyright (c) 2020 The Regents of the University of Michigan.
+# All Rights Reserved. Licensed according to the terms of the Revised
+# BSD License. See LICENSE.txt for details.
+
+class nebula::profile::kubernetes::destination_port::http_alt {
+  $cluster_name = lookup('nebula::profile::kubernetes::cluster')
+
+  @@concat_fragment { "haproxy kubernetes http alt ${::hostname}":
+    target  => '/etc/haproxy/services.d/http_alt.cfg',
+    order   => '02',
+    content => "  server ${::hostname} ${::ipaddress}:31080 check\n",
+    tag     => "${cluster_name}_haproxy_kubernetes_http_alt",
+  }
+}

manifests/profile/kubernetes/destination_port/https.pp

@@ -0,0 +1,14 @@
+# Copyright (c) 2020 The Regents of the University of Michigan.
+# All Rights Reserved. Licensed according to the terms of the Revised
+# BSD License. See LICENSE.txt for details.
+
+class nebula::profile::kubernetes::destination_port::https {
+  $cluster_name = lookup('nebula::profile::kubernetes::cluster')
+
+  @@concat_fragment { "haproxy kubernetes https ${::hostname}":
+    target  => '/etc/haproxy/services.d/https.cfg',
+    order   => '02',
+    content => "  server ${::hostname} ${::ipaddress}:30443 check\n",
+    tag     => "${cluster_name}_haproxy_kubernetes_https",
+  }
+}

manifests/profile/kubernetes/destination_port/https_alt.pp

@@ -0,0 +1,14 @@
+# Copyright (c) 2020 The Regents of the University of Michigan.
+# All Rights Reserved. Licensed according to the terms of the Revised
+# BSD License. See LICENSE.txt for details.
+
+class nebula::profile::kubernetes::destination_port::https_alt {
+  $cluster_name = lookup('nebula::profile::kubernetes::cluster')
+
+  @@concat_fragment { "haproxy kubernetes https alt ${::hostname}":
+    target  => '/etc/haproxy/services.d/https_alt.cfg',
+    order   => '02',
+    content => "  server ${::hostname} ${::ipaddress}:31443 check\n",
+    tag     => "${cluster_name}_haproxy_kubernetes_https_alt",
+  }
+}

manifests/profile/kubernetes/dns_client.pp

@@ -0,0 +1,21 @@
+# Copyright (c) 2020 The Regents of the University of Michigan.
+# All Rights Reserved. Licensed according to the terms of the Revised
+# BSD License. See LICENSE.txt for details.
+
+class nebula::profile::kubernetes::dns_client {
+  $cluster_name = lookup('nebula::profile::kubernetes::cluster')
+  $cluster = lookup('nebula::profile::kubernetes::clusters')[$cluster_name]
+  $private_domain = $cluster['private_domain']
+  $router_address = $cluster['router_address']
+
+  @@concat_fragment { "/etc/hosts ipv4 ${::ipaddress}":
+    tag     => "${cluster_name}_etc_hosts_ip4_hostname",
+    target  => '/etc/hosts',
+    order   => '04',
+    content => template('nebula/profile/kubernetes/dns/hosts_04_ipv4_hostname.erb'),
+  }
+
+  file { '/etc/resolv.conf':
+    content => template('nebula/profile/kubernetes/dns/resolv.conf.erb'),
+  }
+}

manifests/profile/kubernetes/dns_server.pp

@@ -0,0 +1,72 @@
+# Copyright (c) 2020 The Regents of the University of Michigan.
+# All Rights Reserved. Licensed according to the terms of the Revised
+# BSD License. See LICENSE.txt for details.
+
+class nebula::profile::kubernetes::dns_server {
+  $cluster_name = lookup('nebula::profile::kubernetes::cluster')
+  $cluster = lookup('nebula::profile::kubernetes::clusters')[$cluster_name]
+  $etcd_address = $cluster['etcd_address']
+  $kube_api_address = $cluster['kube_api_address']
+  $node_cidr = $cluster['node_cidr']
+  $private_domain = $cluster['private_domain']
+
+  package { 'dnsmasq': }
+
+  service { 'dnsmasq':
+    require => Package['dnsmasq'],
+  }
+
+  concat { '/etc/hosts':
+    notify => Service['dnsmasq'],
+  }
+
+  Concat_fragment <<| tag == "${cluster_name}_etc_hosts_ip4_hostname" |>>
+
+  concat_fragment {
+    default:
+      target => '/etc/hosts',
+    ;
+
+    '/etc/hosts ipv4 localhost':
+      content => template('nebula/profile/kubernetes/dns/hosts_01_ipv4_localhost.erb'),
+      order   => '01',
+    ;
+
+    '/etc/hosts ipv4 etcd-all':
+      content => template('nebula/profile/kubernetes/dns/hosts_02_etcd_all.erb'),
+      order   => '02',
+    ;
+
+    '/etc/hosts ipv4 kube-api':
+      content => template('nebula/profile/kubernetes/dns/hosts_03_kube_api.erb'),
+      order   => '03',
+    ;
+
+    '/etc/hosts ipv6 localhost':
+      content => template('nebula/profile/kubernetes/dns/hosts_05_ipv6_localhost.erb'),
+      order   => '05',
+    ;
+
+    '/etc/hosts ipv6 debian':
+      content => template('nebula/profile/kubernetes/dns/hosts_06_ipv6_debian.erb'),
+      order   => '06',
+    ;
+  }
+
+  firewall {
+    default:
+      dport  => 53,
+      source => $node_cidr,
+      state  => 'NEW',
+      action => 'accept',
+    ;
+
+    '200 Nameserver (TCP)':
+      proto => 'tcp',
+    ;
+
+    '200 Nameserver (UDP)':
+      proto => 'udp',
+    ;
+  }
+}

manifests/profile/kubernetes/docker.pp

@@ -0,0 +1,18 @@
+# Copyright (c) 2020 The Regents of the University of Michigan.
+# All Rights Reserved. Licensed according to the terms of the Revised
+# BSD License. See LICENSE.txt for details.
+
+class nebula::profile::kubernetes::docker {
+  $cluster_name = lookup('nebula::profile::kubernetes::cluster')
+  $cluster = lookup('nebula::profile::kubernetes::clusters')[$cluster_name]
+  $docker_version = $cluster['docker_version']
+
+  if $docker_version == undef {
+    fail('You must set a specific docker version')
+  }
+
+  class { 'nebula::profile::docker':
+    version                => $docker_version,
+    docker_compose_version => '',
+  }
+}

manifests/profile/kubernetes/filesystems.pp

@@ -0,0 +1,15 @@
+# Copyright (c) 2019-2020 The Regents of the University of Michigan.
+# All Rights Reserved. Licensed according to the terms of the Revised
+# BSD License. See LICENSE.txt for details.
+
+class nebula::profile::kubernetes::filesystems (
+  Hash[String, Hash] $cifs_mounts = {},
+) {
+  ensure_packages(['nfs-common'], {'ensure' => 'present'})
+
+  $cifs_mounts.each |$mount_title, $mount_parameters| {
+    nebula::cifs_mount { "/mnt/legacy_cifs_${mount_title}":
+      * => $mount_parameters,
+    }
+  }
+}