feat: untangle specific cluster build needs from ci here

.github/workflows/build-deploy.yaml

@@ -0,0 +1,60 @@
+name: Docker Build and Deploy
+
+on:
+  push:
+    branches: 
+      - develop
+  pull_request: {}
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: converged-computing/usernetes
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to the Container registry
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels)
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          file: Dockerfile.d/Dockerfile.base
+          tags: |
+            # Set node-base as the primary tag for the main branch
+            type=raw,value=node-base,enable=${{ github.ref == 'refs/heads/main' }}
+            # Add SHA tag for traceability
+            type=sha,format=short
+            # Tag PRs with the PR number
+            type=ref,event=pr
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v5
+        with:
+          file: Dockerfile.d/Dockerfile.base       
+          context: .
+          # Only push if it's NOT a pull request
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          # Use GitHub Actions cache to speed up builds
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.github/workflows/main.yaml

@@ -2,15 +2,16 @@
 name: Main
 on: [push, pull_request]
 jobs:
-  single-node:
-    name: "Single node"
-    strategy:
-      fail-fast: false
-      matrix:
-        container_engine: [docker, nerdctl, podman]
-    uses: ./.github/workflows/reusable-single-node.yaml
-    with:
-      container_engine: ${{ matrix.container_engine }}
+  # We would never use usernetes on a single node
+  #single-node:
+  #  name: "Single node"
+  #  strategy:
+  #    fail-fast: false
+  #    matrix:
+  #      container_engine: [docker, nerdctl, podman]
+  #  uses: ./.github/workflows/reusable-single-node.yaml
+  #  with:
+  #    container_engine: ${{ matrix.container_engine }}
 
   multi-node:
     name: "Multi node"
@@ -30,19 +31,3 @@ jobs:
     with:
       lima_template: ${{ matrix.lima_template }}
       container_engine: ${{ matrix.container_engine }}
-
-  # TODO: this test should create multiple instances of Usernetes on each of the hosts
-  multi-node-custom-ports:
-    name: "Multi node with custom service ports"
-    uses: ./.github/workflows/reusable-multi-node.yaml
-    with:
-      lima_template: "template://ubuntu-24.04"
-      container_engine: "docker"
-      # Defaults to 6443
-      kube_apiserver_port: "8080"
-      # Defaults to 8472
-      flannel_port: "9072"
-      # Defaults to 10250
-      kubelet_port: "20250"
-      # Defaults to 2379
-      etcd_port: "9090"

.github/workflows/reusable-single-node.yaml

@@ -83,9 +83,3 @@ jobs:
       - run: make kubeconfig
       - run: kubectl taint nodes --all node-role.kubernetes.io/control-plane-
       - run: ./hack/test-smoke.sh
-      - name: "Test data persistency after restarting the node"
-        run: |
-          make down
-          make up
-          sleep 30
-          ./hack/test-smoke.sh

Dockerfile

@@ -1,40 +1,10 @@
-ARG BASE_IMAGE=docker.io/kindest/node:v1.33.0@sha256:91e9ed777db80279c22d1d1068c091b899b2078506e4a0f797fbf6e397c0b0b2
-ARG CNI_PLUGINS_VERSION=v1.7.1
-ARG HELM_VERSION=v3.17.3
-ARG FLANNEL_VERSION=v0.26.7
+# ARG BASE_IMAGE=ghcr.io/converged-computing/usernetes:node-base
+ARG BASE_IMAGE=usernetes_base
+# Edit this image to add / adopt for your environment
 FROM ${BASE_IMAGE}
-COPY Dockerfile.d/SHA256SUMS.d/ /tmp/SHA256SUMS.d
-ARG CNI_PLUGINS_VERSION
-ARG HELM_VERSION
-ARG FLANNEL_VERSION
 # This are private on our cluster and need to be copied to here
 COPY cspca.llnl.gov.cer.pem /usr/local/share/ca-certificates/
 COPY cspca.cer.pem /usr/local/share/ca-certificates/
+COPY PAN-cspca.llnl.gov.crt /usr/local/share/ca-certificates/
+COPY hpc-profile.json /var/lib/kubelet/seccomp/hpc-profile.json
 RUN update-ca-certificates
-RUN arch="$(uname -m | sed -e s/x86_64/amd64/ -e s/aarch64/arm64/)" && \
-  fname="cni-plugins-linux-${arch}-${CNI_PLUGINS_VERSION}.tgz" && \
-  curl --insecure -o "${fname}" -fSL "https://github.com/containernetworking/plugins/releases/download/${CNI_PLUGINS_VERSION}/${fname}" && \
-  grep "${fname}" "/tmp/SHA256SUMS.d/cni-plugins-${CNI_PLUGINS_VERSION}" | sha256sum -c && \
-  mkdir -p /opt/cni/bin && \
-  tar xzf "${fname}" -C /opt/cni/bin && \
-  rm -f "${fname}" && \
-  fname="helm-${HELM_VERSION}-linux-${arch}.tar.gz" && \
-  curl --insecure -o "${fname}" -fSL "https://get.helm.sh/${fname}" && \
-  grep "${fname}" "/tmp/SHA256SUMS.d/helm-${HELM_VERSION}" | sha256sum -c && \
-  tar xzf "${fname}" -C /usr/local/bin --strip-components=1 -- "linux-${arch}/helm" && \
-  rm -f "${fname}" && \
-  fname="flannel.tgz" && \
-  curl --insecure -o "${fname}" -fSL "https://github.com/flannel-io/flannel/releases/download/${FLANNEL_VERSION}/${fname}" && \
-  grep "${fname}" "/tmp/SHA256SUMS.d/flannel-${FLANNEL_VERSION}" | sha256sum -c && \
-  tar xzf "${fname}" -C / && \
-  rm -f "${fname}"
-# gettext-base: for `envsubst`
-# moreutils: for `sponge`
-# socat: for `socat` (to silence "[WARNING FileExisting-socat]" from kubeadm)
-RUN apt-get update && apt-get install -y --no-install-recommends \
-  gettext-base \
-  moreutils \
-  socat
-ADD Dockerfile.d/etc_udev_rules.d_90-flannel.rules /etc/udev/rules.d/90-flannel.rules
-ADD Dockerfile.d/u7s-entrypoint.sh /
-ENTRYPOINT ["/u7s-entrypoint.sh", "/usr/local/bin/entrypoint", "/sbin/init"]

Dockerfile.d/Dockerfile.base

@@ -0,0 +1,36 @@
+ARG BASE_IMAGE=docker.io/kindest/node:v1.33.0@sha256:91e9ed777db80279c22d1d1068c091b899b2078506e4a0f797fbf6e397c0b0b2
+ARG CNI_PLUGINS_VERSION=v1.7.1
+ARG HELM_VERSION=v3.17.3
+ARG FLANNEL_VERSION=v0.26.7
+FROM ${BASE_IMAGE}
+COPY Dockerfile.d/SHA256SUMS.d/ /tmp/SHA256SUMS.d
+ARG CNI_PLUGINS_VERSION
+ARG HELM_VERSION
+ARG FLANNEL_VERSION
+RUN arch="$(uname -m | sed -e s/x86_64/amd64/ -e s/aarch64/arm64/)" && \
+  fname="cni-plugins-linux-${arch}-${CNI_PLUGINS_VERSION}.tgz" && \
+  curl --insecure -o "${fname}" -fSL "https://github.com/containernetworking/plugins/releases/download/${CNI_PLUGINS_VERSION}/${fname}" && \
+  grep "${fname}" "/tmp/SHA256SUMS.d/cni-plugins-${CNI_PLUGINS_VERSION}" | sha256sum -c && \
+  mkdir -p /opt/cni/bin && \
+  tar xzf "${fname}" -C /opt/cni/bin && \
+  rm -f "${fname}" && \
+  fname="helm-${HELM_VERSION}-linux-${arch}.tar.gz" && \
+  curl --insecure -o "${fname}" -fSL "https://get.helm.sh/${fname}" && \
+  grep "${fname}" "/tmp/SHA256SUMS.d/helm-${HELM_VERSION}" | sha256sum -c && \
+  tar xzf "${fname}" -C /usr/local/bin --strip-components=1 -- "linux-${arch}/helm" && \
+  rm -f "${fname}" && \
+  fname="flannel.tgz" && \
+  curl --insecure -o "${fname}" -fSL "https://github.com/flannel-io/flannel/releases/download/${FLANNEL_VERSION}/${fname}" && \
+  grep "${fname}" "/tmp/SHA256SUMS.d/flannel-${FLANNEL_VERSION}" | sha256sum -c && \
+  tar xzf "${fname}" -C / && \
+  rm -f "${fname}"
+# gettext-base: for `envsubst`
+# moreutils: for `sponge`
+# socat: for `socat` (to silence "[WARNING FileExisting-socat]" from kubeadm)
+RUN apt-get update && apt-get install -y --no-install-recommends \
+  gettext-base \
+  moreutils \
+  socat ipset wget
+ADD Dockerfile.d/etc_udev_rules.d_90-flannel.rules /etc/udev/rules.d/90-flannel.rules
+ADD Dockerfile.d/u7s-entrypoint.sh /
+ENTRYPOINT ["/u7s-entrypoint.sh", "/usr/local/bin/entrypoint", "/sbin/init"]

Makefile

@@ -7,6 +7,8 @@ export PORT_KUBELET ?= 10250
 export PORT_FLANNEL ?= 8472
 export PORT_KUBE_APISERVER ?= 6443
 
+HERE := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
+
 # HOSTNAME is the name of the physical host
 export HOSTNAME ?= $(shell hostname)
 # HOST_IP is the IP address of the physical host. Accessible from other hosts.
@@ -22,9 +24,7 @@ export NODE_SUBNET ?= $(shell $(CURDIR)/Makefile.d/node-subnet.sh)
 export NODE_IP := $(subst .0/24,.100,$(NODE_SUBNET))
 
 export CONTAINER_ENGINE ?= $(shell $(CURDIR)/Makefile.d/detect-container-engine.sh CONTAINER_ENGINE)
-
 export CONTAINER_ENGINE_TYPE ?= $(shell $(CURDIR)/Makefile.d/detect-container-engine.sh CONTAINER_ENGINE_TYPE)
-
 COMPOSE ?= $(shell $(CURDIR)/Makefile.d/detect-container-engine.sh COMPOSE)
 
 NODE_SERVICE_NAME := node
@@ -82,8 +82,14 @@ render: check-preflight
 .PHONY: up
 up: check-preflight
 	# Podman creates cni files in a shared location, this ensures unique names that do not clobbed one another
-	sed -i "s/default_network/$(HOSTNAME)/g" docker-compose.yaml
-	$(COMPOSE) up --build -d
+	sed -i "s/default_network/$(HOSTNAME)/g" $(HERE)/docker-compose.yaml
+	$(COMPOSE) up -d
+
+.PHONY: up-built
+up-built: check-preflight
+	# Podman creates cni files in a shared location, this ensures unique names that do not clobbed one another
+	sed -i "s/default_network/$(HOSTNAME)/g" $(HERE)/docker-compose.yaml
+	$(COMPOSE) -f $(HERE)/docker-compose.yaml -f $(HERE)/compose/prebuilt-node.yaml up -d
 
 .PHONY: down
 down:
@@ -145,7 +151,7 @@ sync-external-ip:
 .PHONY: kubeadm-join
 kubeadm-join:
 	# Our kernel is too old for usernetes, so we need this
-	sed -i "s/--token/--ignore-preflight-errors=all --token/g" join-command
+	sed -i "s/--token/--ignore-preflight-errors=all --token/g" $(HERE)/join-command
 	$(NODE_SHELL) /bin/bash /usernetes/join-command
 	@echo "# Run 'make sync-external-ip' on the control plane"
 

compose/prebuilt-node.yaml

@@ -0,0 +1,3 @@
+services:
+  usernetes_node:
+    image: usernetes_node

docker-compose.yaml

@@ -3,7 +3,9 @@
 ---
 services:
   node:
-    image: usernetes_node
+    build:
+      context: .
+      dockerfile: Dockerfile.d/Dockerfile.base
     hostname: ${NODE_NAME}
     privileged: true
     restart: always

hack/create-cluster-lima.sh

@@ -11,7 +11,7 @@ set -eux -o pipefail
 : "${PORT_FLANNEL:=8472}"
 : "${PORT_KUBELET:=10250}"
 
-guest_home="/home/${USER}.linux"
+guest_home="/home/runner.guest"
 
 if [ "$(id -u)" -le 1000 ]; then
 	# In --plain mode, UID has to be >= 1000 to populate subuids
@@ -23,6 +23,9 @@ fi
 for host in host0 host1; do
 	# Set --plain to minimize Limaism
 	${LIMACTL} start --plain --network lima:user-v2 --name="${host}" ${LIMACTL_CREATE_ARGS} "${LIMA_TEMPLATE}"
+	echo "LISTING ${host}"
+	${LIMACTL} shell "${host}" ls /
+	${LIMACTL} shell "${host}" ls /home
 	${LIMACTL} copy -r "$(pwd)" "${host}:${guest_home}/usernetes"
 	${LIMACTL} shell "${host}" sudo CONTAINER_ENGINE="${CONTAINER_ENGINE}" "${guest_home}/usernetes/init-host/init-host.root.sh"
 	# Terminate the current session so that the cgroup delegation takes an effect. This command exits with status 255 as SSH terminates.
@@ -46,7 +49,8 @@ done
 ${LIMACTL} shell host0 ${SERVICE_PORTS} CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C "${guest_home}/usernetes" kubeadm-init install-flannel kubeconfig join-command
 
 # Let host1 join the cluster
-${LIMACTL} copy host0:~/usernetes/join-command host1:~/usernetes/join-command
+${LIMACTL} copy host0:${guest_home}/usernetes/join-command ./join-command
+${LIMACTL} copy ./join-command host1:${guest_home}/usernetes/join-command
 ${LIMACTL} shell host1 ${SERVICE_PORTS} CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C "${guest_home}/usernetes" kubeadm-join
 ${LIMACTL} shell host0 ${SERVICE_PORTS} CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C "${guest_home}/usernetes" sync-external-ip
 

hack/test-smoke.sh

@@ -54,9 +54,40 @@ EOF
 	INFO "Waiting for 3 replicas to be ready"
 	kubectl rollout status --timeout=5m statefulset
 
+	INFO "GET PODS"
+	kubectl get pods
+	kubectl get pods -n kube-system
+	INFO "DESCRIBE PODS"
+	kubectl describe pods
+	for name in $(kubectl get pods -o json | jq -r .items[].metadata.name)
+	  do
+	     kubectl logs $name
+	     kubectl exec -it $name -- cat /etc/resolv.conf
+	  done
+
+	INFO "Patching CoreDNS to use 8.8.8.8"
+	kubectl get configmap coredns -n kube-system -o yaml | \
+	  sed 's/forward . \/etc\/resolv.conf/forward . 8.8.8.8/' | \
+	  kubectl apply -f -
+
+	INFO "Restarting CoreDNS"
+	kubectl delete pod -n kube-system -l k8s-app=kube-dns
+	kubectl rollout status deployment coredns -n kube-system
+
 	INFO "Connecting to dnstest-{0,1,2}.dnstest.default.svc.cluster.local"
-	kubectl run -i --rm --image=alpine --restart=Never dnstest-shell -- sh -exc 'for f in $(seq 0 2); do wget -O- http://dnstest-${f}.dnstest.default.svc.cluster.local; done'
+	kubectl run -i --rm --image=busybox:1.28 --restart=Never dnstest-shell -- sh -exc '
+  echo "--- Resolv.conf ---"
+  cat /etc/resolv.conf
+
+  echo "--- Testing External DNS (google.com) ---"
+  nslookup google.com || echo "External DNS Failed"
+
+  echo "--- Testing Internal DNS (dnstest-0) ---"
+  nslookup dnstest-0.dnstest || echo "Internal DNS Failed"
 
+  for f in 0 1 2; do 
+    wget -qO- http://dnstest-${f}.dnstest.default.svc.cluster.local
+  done'
 	INFO "Deleting Service \"dnstest\""
 	kubectl delete service dnstest
 	INFO "Deleting StatefulSet \"dnstest\""