feat: nix: fully working leviathan-alpha real life setup

k8s/apps/nix-course/ingress.yml

@@ -5,7 +5,7 @@ metadata:
   name: nix-course
 spec:
   rules:
-    - host: nix.litarvan.dev
+    - host: nix.litarvan.dev # TODO
       http:
         paths:
           - path: /

modules/base/core.nix

@@ -18,7 +18,7 @@
   networking = {
     useDHCP = true;
     dhcpcd = {
-      wait = "any"; # Make sure we get an IP before marking the service as up
+      wait = "both"; # Without this, rke2 will not be able to start
       extraConfig = ''
         noipv4ll
       '';

modules/boot/netboot.nix

@@ -7,16 +7,9 @@ with lib;
 
 let
   systemName = config.networking.hostName;
-  downloadRoot = https://pxe.alligator.litarvan.dev;
   storeFile = ".lvth_store.squashfs";
 
-  defaultNameserver = "1.1.1.1";
-
-  labels = {
-    root = "lvth_root";
-    rw_store = "lvth_rw_store";
-    home = "lvth_home";
-  };
+  vars = import ../../vars;
 in
 {
   options.boot.netboot.enable = mkEnableOption "Set defaults for creating a netboot image";
@@ -31,12 +24,12 @@ in
 
     fileSystems = {
       "/" = {
-        label = labels.root;
+        label = vars.diskLabels.root;
         fsType = "ext4";
       };
 
       "/home" = {
-        label = labels.home;
+        label = vars.diskLabels.home;
         fsType = "ext4";
         neededForBoot = true; # For some reason, /home/litarvan is created before the mount if we omit this
       };
@@ -51,7 +44,7 @@ in
       };
 
       "/nix/.rw-store" = {
-        label = labels.rw_store;
+        label = vars.diskLabels.rw_store;
         fsType = "ext4";
         neededForBoot = true;
       };
@@ -73,6 +66,9 @@ in
         "squashfs"
         "overlay"
 
+        # To mount USB keys such as /data/usb1, needed to set litarvan's password
+        "usb_storage"
+
         # SATA support
         "ahci"
         "ata_piix"
@@ -143,23 +139,23 @@ in
     # Network is done in preLVMCommands, which means it is already set up when
     # we get to postDeviceCommands
     boot.initrd.postDeviceCommands = ''
-      echo "nameserver ${defaultNameserver}" > /etc/resolv.conf
+      echo "nameserver ${vars.nameserver}" > /etc/resolv.conf
 
-      if ! mkfs.ext4 -F -L ${labels.rw_store} /dev/disk/by-label/${labels.rw_store}; then
+      if ! mkfs.ext4 -F -L ${vars.diskLabels.rw_store} /dev/disk/by-label/${vars.diskLabels.rw_store}; then
         echo "Failed to cleanup rw store partition"
       fi
 
-      if ! mkfs.ext4 -F -L ${labels.root} /dev/disk/by-label/${labels.root}; then
+      if ! mkfs.ext4 -F -L ${vars.diskLabels.root} /dev/disk/by-label/${vars.diskLabels.root}; then
         echo "Failed to cleanup root partition"
       fi
 
       mkdir -p $targetRoot # TODO: Better way?
-      mount -t ext4 /dev/disk/by-label/${labels.root} $targetRoot
-      if ! curl ${downloadRoot}/${systemName}.squashfs -o $targetRoot/${storeFile}; then
+      mount -t ext4 /dev/disk/by-label/${vars.diskLabels.root} $targetRoot
+      if ! curl ${vars.pxeRemote}/${systemName}.squashfs -o $targetRoot/${storeFile}; then
         echo "Failed to download squashfs"
         fail
       fi
-      umount $targetRoot # X D
+      umount $targetRoot # Without this, the real mount fails
     '';
 
     # Usually, stage2Init is passed using the init kernel command line argument

modules/services/rke2.nix

@@ -161,6 +161,14 @@ in
         Environment = "PATH=/run/current-system/sw/bin";
         ExecStart = lib.getExe (pkgs.writeShellScriptBin "leviathan-bootstrap" (concatStringsSep "\n" (map ({ type ? "resource", path }: ''
           while true; do
+            echo Waiting for cluster to be ready...
+            if ! KUBECONFIG=/etc/rancher/rke2/rke2.yaml /var/lib/rancher/rke2/bin/kubectl wait --for=condition=Ready nodes --all --timeout=-1s; then
+              echo Error waiting for cluster! Retrying in 5 seconds... # We can't use the Restart= option with oneshot services
+              sleep 5
+              echo
+              continue
+            fi
+
             echo Applying "${type}" "${path}"...
             KUBECONFIG=/etc/rancher/rke2/rke2.yaml /var/lib/rancher/rke2/bin/kubectl apply -"${if type == "kustomization" then "k" else "f"}" "${path}" && break
             echo Command failed! Retrying in 5 seconds...

systems/alligator/nginx.nix

@@ -11,16 +11,18 @@ in
 
   security.acme = {
     acceptTerms = true;
-    defaults.email = "adrien1975" + "@" + "live.fr"; # TODO: Change
+    defaults.email = vars.acmeEmail;
 
     certs = {
-      "pxe.alligator.litarvan.dev" = {
+      ${vars.domains.pxe} = {
         keyType = "rsa2048"; # iPXE does not support EC*
         extraLegoRunFlags = [ "--preferred-chain" "ISRG Root X1" ]; # iPXE is missing some root certificates
       };
 
-      "litarvan.dev" = {
-        domain = "*.litarvan.dev";
+      ${vars.domains.root} = {
+        domain = "*.${vars.domains.root}";
+        extraDomainNames = map (x: "*.${x}") vars.domains.subRoots;
+
         dnsProvider = "netlify";
         credentialsFile = "/data/secrets/netlify.env";
         group = "nginx";
@@ -44,34 +46,42 @@ in
       ssl_early_data on;
       ssl_ecdh_curve secp384r1;
 
-      add_header Expect-CT "max-age=0";
-      add_header Strict-Transport-Security "max-age=63072000; includeSubDomains; preload" always;
-      add_header X-Content-Type-Options "nosniff" always;
-      add_header X-Frame-Options "SAMEORIGIN" always;
-      add_header Referrer-Policy "no-referrer-when-downgrade" always;
-
       charset UTF-8;
     '';
 
-    virtualHosts = {
-      "pxe.alligator.litarvan.dev" = {
-        enableACME = true;
-        forceSSL = true;
+    virtualHosts =
+      let
+        extraConfig = ''
+          add_header Expect-CT "max-age=0";
+          add_header Strict-Transport-Security "max-age=63072000; includeSubDomains; preload" always;
+          add_header X-Content-Type-Options "nosniff" always;
+          add_header X-Frame-Options "SAMEORIGIN" always;
+          add_header Referrer-Policy "no-referrer-when-downgrade" always;
+        '';
+      in
+      {
+        ${vars.domains.pxe} = {
+          enableACME = true;
+          forceSSL = true;
 
-        root = "/var/www/pxe";
-      };
+          root = "/var/www/pxe";
 
-      "*.litarvan.dev" = {
-        http2 = true;
-        quic = true;
+          inherit extraConfig;
+        };
 
-        useACMEHost = "litarvan.dev";
-        forceSSL = true;
+        "*.${vars.domains.root}" = {
+          http2 = true;
+          quic = true;
 
-        locations."/" = {
-          proxyPass = "https://${vars.wireguard.peers.leviathan-alpha.ips.v4}";
+          useACMEHost = vars.domains.root;
+          forceSSL = true;
+
+          inherit extraConfig;
+
+          locations."/" = {
+            proxyPass = "https://${builtins.head (builtins.split "/" vars.wireguard.peers.leviathan-alpha.ips.v4)}";
+          };
         };
       };
-    };
   };
 }

systems/alligator/wireguard.nix

@@ -23,15 +23,15 @@ in
       privateKeyFile = "/data/secrets/wireguard-private-key";
 
       postUp = ''
-        ${pkgs.iptables}/bin/iptables -A FORWARD -i wg0 -j ACCEPT
+        ${pkgs.iptables}/bin/iptables -A FORWARD -i ${vars.wireguard.interface} -j ACCEPT
         ${pkgs.iptables}/bin/iptables -t nat -A POSTROUTING -s ${vars.wireguard.peers.alligator.ips.v4} -o eth0 -j MASQUERADE
-        ${pkgs.iptables}/bin/ip6tables -A FORWARD -i wg0 -j ACCEPT
+        ${pkgs.iptables}/bin/ip6tables -A FORWARD -i ${vars.wireguard.interface} -j ACCEPT
         ${pkgs.iptables}/bin/ip6tables -t nat -A POSTROUTING -s ${vars.wireguard.peers.alligator.ips.v6} -o eth0 -j MASQUERADE
       '';
       preDown = ''
-        ${pkgs.iptables}/bin/iptables -D FORWARD -i wg0 -j ACCEPT
+        ${pkgs.iptables}/bin/iptables -D FORWARD -i ${vars.wireguard.interface} -j ACCEPT
         ${pkgs.iptables}/bin/iptables -t nat -D POSTROUTING -s ${vars.wireguard.peers.alligator.ips.v4} -o eth0 -j MASQUERADE
-        ${pkgs.iptables}/bin/ip6tables -D FORWARD -i wg0 -j ACCEPT
+        ${pkgs.iptables}/bin/ip6tables -D FORWARD -i ${vars.wireguard.interface} -j ACCEPT
         ${pkgs.iptables}/bin/ip6tables -t nat -D POSTROUTING -s ${vars.wireguard.peers.alligator.ips.v6} -o eth0 -j MASQUERADE
       '';
 

systems/alpha/default.nix

@@ -1,16 +1,16 @@
 { lib, pkgs, ... }:
 
 let
-  alligatorHost = "alligator.litarvan.dev";
-  vars = import ../vars;
+  vars = import ../../vars;
 in
 {
   boot = {
     initrd.availableKernelModules = [ "xhci_pci" "ehci_pci" "ahci" "usbhid" "sd_mod" ];
     kernelModules = [ "kvm-intel" "nvme" ];
 
-    netboot = true;
+    kernelParams = [ "module_blacklist=i915" ];
 
+    netboot.enable = true;
     postBootCommands = ''
       mkdir -p /data/{nvme1,usb1}
       chmod 700 /data
@@ -19,23 +19,22 @@ in
 
   powerManagement.cpuFreqGovernor = lib.mkDefault "performance";
 
-  hardware = {
-    opengl = {
-      enable = true;
-      driSupport32Bit = true;
-    };
-    nvidia.modesetting.enable = true;
+  hardware.opengl = {
+    enable = true;
+    driSupport32Bit = true;
   };
 
   fileSystems = {
     "/data/nvme1" = {
-      label = "lvth_data_nvme1";
+      label = vars.diskLabels.leviathan-alpha.nvme1;
       fsType = "ext4";
     };
 
     "/data/usb1" = {
-      label = "LVTH_ALPHA";
+      label = vars.diskLabels.leviathan-alpha.usb1;
       fsType = "vfat";
+      neededForBoot = true; # Else, /data/usb1 won't exist when litarvan's password file is read
+      options = [ "ro,umask=077" ];
     };
   };
 
@@ -46,20 +45,31 @@ in
     hostName = "leviathan-alpha";
     interfaces.eth0.useDHCP = true;
 
+    firewall = {
+      allowedTCPPorts = [ 443 ];
+      allowedUDPPorts = [ 443 ];
+    };
+
     wg-quick.interfaces.${vars.wireguard.interface} = {
       address = builtins.attrValues vars.wireguard.peers.leviathan-alpha.ips;
       privateKeyFile = "/data/usb1/secrets/wireguard-private-key";
 
-      # When using privateKeyFile, the private-key is set in the postUp hook.
-      # But the PersistentKeepAlive parameter is then "reset", so we must apply it again.
       postUp = ''
+        # When using privateKeyFile, the private-key is set in the postUp hook.
+        # But the PersistentKeepAlive parameter is then "reset", so we must apply it again.
         wg set ${vars.wireguard.interface} peer ${vars.wireguard.peers.alligator.publicKey} persistent-keepalive 25
+
+        # Kubernetes CIDRs must stay local
+        ip route add 10.43.0.0/16 dev lo # Kubernetes intern CIDR must stay local; TODO: Use variables, edit when multi-node
+      '';
+      postDown = ''
+        ip route delete 10.43.0.0/16 dev lo
       '';
 
       peers = [
         # Alligator
         {
-          endpoint = "${alligatorHost}:${toString vars.wireguard.port}";
+          endpoint = "${vars.wireguard.peers.alligator.host}:${toString vars.wireguard.port}";
           publicKey = vars.wireguard.peers.alligator.publicKey;
           allowedIPs = [ "0.0.0.0/0" "::/0" ];
         }
@@ -68,7 +78,13 @@ in
   };
 
   services = {
-    xserver.videoDrivers = [ "nvidia" ];
+    # xserver.videoDrivers = [ "nvidia" ];
+    openssh.hostKeys = [
+      {
+        path = "/data/usb1/secrets/ssh-host-ed25519-key";
+        type = "ed25519";
+      }
+    ];
     rke2 = {
       enable = true;
       bootstrapManifests = [

systems/default.nix

@@ -1,25 +1,25 @@
 { inputs, systemsPkgs, vars, ... }:
 
 let
-  host = with input.nixpkgs.lib.attrsets; system: path:
+  host = with inputs.nixpkgs.lib.attrsets; system: path:
     let
       registry = (filterAttrs (name: _: name != "self") inputs) // { leviathan = inputs.self; };
       modules = [
         path
-        home-manager.nixosModules.home-manager
+        inputs.home-manager.nixosModules.home-manager
         {
           nix = {
-            nixPath = (mapAttrsToList (name: input: "${name}=${input}") inputs) ++ [ "nixos=${input.nixpkgs}" ];
+            nixPath = (mapAttrsToList (name: input: "${name}=${input}") inputs) ++ [ "nixos=${inputs.nixpkgs}" ];
             registry = mapAttrs (_: input: { flake = input; }) registry;
           };
           nixpkgs = {
             pkgs = systemsPkgs.${system};
-            overlays = attrValues self.overlays;
+            overlays = attrValues inputs.self.overlays;
           };
         }
-      ] ++ (attrValues self.nixosModules);
+      ] ++ (attrValues inputs.self.nixosModules);
     in
-    nixpkgs.lib.nixosSystem {
+    inputs.nixpkgs.lib.nixosSystem {
       inherit system;
 
       modules = modules ++ [
@@ -39,5 +39,5 @@ let
 in
 {
   alligator = host "aarch64-linux" ./alligator;
-  leviathan-alpha = host "x86_64-linux" ./alpha.nix;
+  leviathan-alpha = host "x86_64-linux" ./alpha;
 }