🌱 Provision baremetal via --baremetal-image-url-command

api/v1beta1/hetznerbaremetalmachine_types.go

@@ -187,6 +187,11 @@ type Image struct {
 	// URL defines the remote URL for downloading a tar, tar.gz, tar.bz, tar.bz2, tar.xz, tgz, tbz, txz image.
 	URL string `json:"url,omitempty"`
 
+	// UseCustomImageURLCommand makes the controller use the command provided by `--baremetal-image-url-command` instead of installimage.
+	// Docs: https://syself.com/docs/caph/developers/image-url-command
+	// +optional
+	UseCustomImageURLCommand bool `json:"useCustomImageURLCommand"`
+
 	// Name defines the archive name after download. This has to be a valid name for Installimage.
 	Name string `json:"name,omitempty"`
 
@@ -197,6 +202,9 @@ type Image struct {
 // GetDetails returns the path of the image and whether the image has to be downloaded.
 func (image Image) GetDetails() (imagePath string, needsDownload bool, errorMessage string) {
 	// If image is set, then the URL is also set and we have to download a remote file
+	if image.UseCustomImageURLCommand {
+		return "", false, "internal error: image.UseCustomImageURLCommand is active. Method GetDetails() should be used for the traditional way (without image-url-command)."
+	}
 	switch {
 	case image.Name != "" && image.URL != "":
 		suffix, err := GetImageSuffix(image.URL)

config/crd/bases/infrastructure.cluster.x-k8s.io_hetznerbaremetalhosts.yaml

@@ -396,6 +396,11 @@ spec:
                               a tar, tar.gz, tar.bz, tar.bz2, tar.xz, tgz, tbz, txz
                               image.
                             type: string
+                          useCustomImageURLCommand:
+                            description: |-
+                              UseCustomImageURLCommand makes the controller use the command provided by `--baremetal-image-url-command` instead of installimage.
+                              Docs: https://syself.com/docs/caph/developers/image-url-command
+                            type: boolean
                         type: object
                       logicalVolumeDefinitions:
                         description: LVMDefinitions defines the logical volume definitions

config/crd/bases/infrastructure.cluster.x-k8s.io_hetznerbaremetalmachines.yaml

@@ -155,6 +155,11 @@ spec:
                         description: URL defines the remote URL for downloading a
                           tar, tar.gz, tar.bz, tar.bz2, tar.xz, tgz, tbz, txz image.
                         type: string
+                      useCustomImageURLCommand:
+                        description: |-
+                          UseCustomImageURLCommand makes the controller use the command provided by `--baremetal-image-url-command` instead of installimage.
+                          Docs: https://syself.com/docs/caph/developers/image-url-command
+                        type: boolean
                     type: object
                   logicalVolumeDefinitions:
                     description: LVMDefinitions defines the logical volume definitions

config/crd/bases/infrastructure.cluster.x-k8s.io_hetznerbaremetalmachinetemplates.yaml

@@ -142,6 +142,11 @@ spec:
                                   a tar, tar.gz, tar.bz, tar.bz2, tar.xz, tgz, tbz,
                                   txz image.
                                 type: string
+                              useCustomImageURLCommand:
+                                description: |-
+                                  UseCustomImageURLCommand makes the controller use the command provided by `--baremetal-image-url-command` instead of installimage.
+                                  Docs: https://syself.com/docs/caph/developers/image-url-command
+                                type: boolean
                             type: object
                           logicalVolumeDefinitions:
                             description: LVMDefinitions defines the logical volume

controllers/hetznerbaremetalhost_controller.go

@@ -23,9 +23,11 @@ import (
 	"reflect"
 	"time"
 
+	"github.com/google/go-cmp/cmp"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/klog/v2"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	"sigs.k8s.io/cluster-api/util"
@@ -47,6 +49,7 @@ import (
 	robotclient "github.com/syself/cluster-api-provider-hetzner/pkg/services/baremetal/client/robot"
 	sshclient "github.com/syself/cluster-api-provider-hetzner/pkg/services/baremetal/client/ssh"
 	"github.com/syself/cluster-api-provider-hetzner/pkg/services/baremetal/host"
+	"github.com/syself/cluster-api-provider-hetzner/pkg/utils"
 )
 
 // HetznerBareMetalHostReconciler reconciles a HetznerBareMetalHost object.
@@ -59,6 +62,7 @@ type HetznerBareMetalHostReconciler struct {
 	WatchFilterValue     string
 	PreProvisionCommand  string
 	SSHAfterInstallImage bool
+	ImageURLCommand      string
 }
 
 //+kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=hetznerbaremetalhosts,verbs=get;list;watch;create;update;patch;delete
@@ -88,6 +92,58 @@ func (r *HetznerBareMetalHostReconciler) Reconcile(ctx context.Context, req ctrl
 		return reconcile.Result{}, err
 	}
 
+	// ----------------------------------------------------------------
+	// Start: avoid conflict errors. Wait until local cache is up-to-date
+	// Won't be needed once this was implemented:
+	// https://github.com/kubernetes-sigs/controller-runtime/issues/3320
+	initialHost := bmHost.DeepCopy()
+	defer func() {
+		// We can potentially optimize this further by ensuring that the cache is up to date only in
+		// the cases where an outdated cache would lead to problems. Currently, we ensure that the
+		// cache is up to date in all cases, i.e. for all possible changes to the
+		// HetznerBareMetalHost object.
+		if cmp.Equal(initialHost, bmHost) {
+			// Nothing has changed. No need to wait.
+			return
+		}
+		startReadOwnWrite := time.Now()
+
+		// The object changed. Wait until the new version is in the local cache
+
+		// Get the latest version from the apiserver.
+		apiserverHost := &infrav1.HetznerBareMetalHost{}
+
+		// Use uncached APIReader
+		err := r.APIReader.Get(ctx, client.ObjectKeyFromObject(bmHost), apiserverHost)
+		if err != nil {
+			reterr = errors.Join(reterr,
+				fmt.Errorf("failed get HetznerBareMetalHost via uncached APIReader: %w", err))
+			return
+		}
+
+		apiserverRV := apiserverHost.ResourceVersion
+
+		err = wait.PollUntilContextTimeout(ctx, 100*time.Millisecond, 3*time.Second, true, func(ctx context.Context) (done bool, err error) {
+			// new resource, read from local cache
+			latestFromLocalCache := &infrav1.HetznerBareMetalHost{}
+			getErr := r.Get(ctx, client.ObjectKeyFromObject(apiserverHost), latestFromLocalCache)
+			if apierrors.IsNotFound(getErr) {
+				// the object was deleted. All is fine.
+				return true, nil
+			}
+			if getErr != nil {
+				return false, getErr
+			}
+			return utils.IsLocalCacheUpToDate(latestFromLocalCache.ResourceVersion, apiserverRV), nil
+		})
+		if err != nil {
+			log.Error(err, "cache sync failed after BootState change")
+		}
+		log.Info("Wait for update being in local cache", "durationWaitForLocalCacheSync", time.Since(startReadOwnWrite).Round(time.Millisecond))
+	}()
+	// End: avoid conflict errors. Wait until local cache is up-to-date
+	// ----------------------------------------------------------------
+
 	initialProvisioningState := bmHost.Spec.Status.ProvisioningState
 	defer func() {
 		if initialProvisioningState != bmHost.Spec.Status.ProvisioningState {
@@ -203,6 +259,7 @@ func (r *HetznerBareMetalHostReconciler) Reconcile(ctx context.Context, req ctrl
 		RescueSSHSecret:         rescueSSHSecret,
 		SecretManager:           secretManager,
 		PreProvisionCommand:     r.PreProvisionCommand,
+		ImageURLCommand:         r.ImageURLCommand,
 		SSHAfterInstallImage:    r.SSHAfterInstallImage,
 	})
 	if err != nil {

docs/caph/04-developers/06-image-url-command.md

@@ -5,21 +5,29 @@ sidebar: image-url-command
 description: Documentation on the CAPH image-url-command
 ---
 
-The `--hcloud-image-url-command` for the caph controller can be used to execute a custom command to
-install the node image.
+The `--hcloud-image-url-command` and `--baremtal-image-url-command` for the caph controller can be
+used to execute a custom command to install the node image.
 
 This provides you a flexible way to create nodes.
 
-The script/binary will be copied into the Hetzner Rescue System and executed.
+The script/binary will be copied into the rescue system and executed.
 
 You need to enable two things:
 
 * The caph binary must get argument. Example:
-  `--hcloud-image-url-command=/shared/image-url-command.sh`
-* The hcloudmachine resource must have spec.imageURL set (usually via a hcloudmachinetemplate)
+  `--[hcloud|baremetal]-image-url-command=/shared/image-url-command.sh`
+* for hcloud: The hcloudmachine resource must have spec.imageURL set (usually via a
+  hcloudmachinetemplate)
+* for baremetal: The hetznerbaremetal resource must use `useCustomImageURLCommand: true`.
 
-The command will get the imageURL, bootstrap-data and machine-name of the corresponding
-hcloudmachine as argument.
+The command will get the imageURL, bootstrap-data, machine-name of the corresponding
+machine and the root devices (seperated by spaces) as argument.
+
+Example:
+
+```bash
+/root/image-url-command oci://example.com/yourimage:v1 /root/bootstrap.data my-md-bm-kh57r-5z2v8-zdfc9 'sda sdb'
+```
 
 It is up to the command to download from that URL and provision the disk accordingly. This command
 must be accessible by the controller pod. You can use an initContainer to copy the command to a
@@ -36,7 +44,7 @@ A Kubernetes event will be created in both (success, failure) cases containing t
 and stderr) of the script. If the script takes longer than 7 minutes, the controller cancels the
 provisioning.
 
-We measured these durations:
+We measured these durations for hcloud:
 
 | oldState | newState | avg(s) | min(s) | max(s) |
 |----------|----------|-------:|-------:|-------:|

main.go

@@ -85,7 +85,8 @@ var (
 	syncPeriod                         time.Duration
 	rateLimitWaitTime                  time.Duration
 	preProvisionCommand                string
-	imageURLCommand                    string
+	hcloudImageURLCommand              string
+	baremetalImageURLCommand           string
 	skipWebhooks                       bool
 	sshAfterInstallImage               bool
 )
@@ -108,7 +109,8 @@ func main() {
 	fs.DurationVar(&rateLimitWaitTime, "rate-limit", 5*time.Minute, "The rate limiting for HCloud controller (e.g. 5m)")
 	fs.BoolVar(&hcloudclient.DebugAPICalls, "debug-hcloud-api-calls", false, "Debug all calls to the hcloud API.")
 	fs.StringVar(&preProvisionCommand, "pre-provision-command", "", "Command to run (in rescue-system) before installing the image on bare metal servers. You can use that to check if the machine is healthy before installing the image. If the exit value is non-zero, the machine is considered unhealthy. This command must be accessible by the controller pod. You can use an initContainer to copy the command to a shared emptyDir.")
-	fs.StringVar(&imageURLCommand, "hcloud-image-url-command", "", "Command to run (in rescue-system) to provision an hcloud machine. The command will get the imageURL, bootstrap-data and machine-name of the corresponding hcloudmachine as argument. It is up to the command to download from that URL and provision the disk accordingly. This command must be accessible by the controller pod. You can use an initContainer to copy the command to a shared emptyDir. The env var OCI_REGISTRY_AUTH_TOKEN from the caph process will be set for the command, too. The command must end with the last line containing IMAGE_URL_DONE. Otherwise the execution is considered to have failed. Docs: https://syself.com/docs/caph/developers/image-url-command")
+	fs.StringVar(&hcloudImageURLCommand, "hcloud-image-url-command", "", "Command to run (in rescue-system) to provision an hcloud machine. Docs: https://syself.com/docs/caph/developers/image-url-command")
+	fs.StringVar(&baremetalImageURLCommand, "baremetal-image-url-command", "", "Command to run (in rescue-system) to provision an baremetal machine. Docs: https://syself.com/docs/caph/developers/image-url-command")
 	fs.BoolVar(&skipWebhooks, "skip-webhooks", false, "Skip setting up of webhooks. Together with --leader-elect=false, you can use `go run main.go` to run CAPH in a cluster connected via KUBECONFIG. You should scale down the caph deployment to 0 before doing that. This is only for testing!")
 	fs.BoolVar(&sshAfterInstallImage, "baremetal-ssh-after-install-image", true, "Connect to the baremetal machine after install-image and ensure it is provisioned. Current default is true, but we might change that to false. Background: Users might not want the controller to be able to ssh onto the servers")
 
@@ -133,22 +135,38 @@ func main() {
 		}
 	}
 
-	// If ImageURLCommand is set, check if the file exists and validate the basename.
-	if imageURLCommand != "" {
-		baseName := filepath.Base(imageURLCommand)
+	// If hcloudImageURLCommand is set, check if the file exists and validate the basename.
+	if hcloudImageURLCommand != "" {
+		baseName := filepath.Base(hcloudImageURLCommand)
 		if !commandRegex.MatchString(baseName) {
 			msg := fmt.Sprintf("basename (%s) must match the regex %s", baseName, commandRegex.String())
 			setupLog.Error(errors.New(msg), "")
 			os.Exit(1)
 		}
 
-		_, err := os.Stat(imageURLCommand)
+		_, err := os.Stat(hcloudImageURLCommand)
 		if err != nil {
 			setupLog.Error(err, "hcloud-image-url-command not found")
 			os.Exit(1)
 		}
 	}
 
+	// If baremetalImageURLCommand is set, check if the file exists and validate the basename.
+	if baremetalImageURLCommand != "" {
+		baseName := filepath.Base(baremetalImageURLCommand)
+		if !commandRegex.MatchString(baseName) {
+			msg := fmt.Sprintf("basename (%s) must match the regex %s", baseName, commandRegex.String())
+			setupLog.Error(errors.New(msg), "")
+			os.Exit(1)
+		}
+
+		_, err := os.Stat(baremetalImageURLCommand)
+		if err != nil {
+			setupLog.Error(err, "baremetal-image-url-command not found")
+			os.Exit(1)
+		}
+	}
+
 	var watchNamespaces map[string]cache.Config
 	if watchNamespace != "" {
 		watchNamespaces = map[string]cache.Config{
@@ -215,7 +233,7 @@ func main() {
 		HCloudClientFactory: hcloudClientFactory,
 		SSHClientFactory:    sshclient.NewFactory(),
 		WatchFilterValue:    watchFilterValue,
-		ImageURLCommand:     imageURLCommand,
+		ImageURLCommand:     hcloudImageURLCommand,
 	}).SetupWithManager(ctx, mgr, controller.Options{MaxConcurrentReconciles: hcloudMachineConcurrency}); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "HCloudMachine")
 		os.Exit(1)
@@ -240,6 +258,7 @@ func main() {
 		RateLimitWaitTime:    rateLimitWaitTime,
 		WatchFilterValue:     watchFilterValue,
 		PreProvisionCommand:  preProvisionCommand,
+		ImageURLCommand:      baremetalImageURLCommand,
 		SSHAfterInstallImage: sshAfterInstallImage,
 	}).SetupWithManager(ctx, mgr, controller.Options{MaxConcurrentReconciles: hetznerBareMetalHostConcurrency}); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "HetznerBareMetalHost")

pkg/scope/baremetalhost.go

@@ -48,6 +48,7 @@ type BareMetalHostScopeParams struct {
 	RescueSSHSecret         *corev1.Secret
 	SecretManager           *secretutil.SecretManager
 	PreProvisionCommand     string
+	ImageURLCommand         string
 	SSHAfterInstallImage    bool
 }
 
@@ -101,6 +102,7 @@ func NewBareMetalHostScope(params BareMetalHostScopeParams) (*BareMetalHostScope
 			cluster:        params.Cluster,
 			hetznerCluster: params.HetznerCluster,
 		},
+		ImageURLCommand: params.ImageURLCommand,
 	}, nil
 }
 
@@ -120,6 +122,7 @@ type BareMetalHostScope struct {
 	PreProvisionCommand          string
 	SSHAfterInstallImage         bool
 	WorkloadClusterClientFactory WorkloadClusterClientFactory
+	ImageURLCommand              string
 }
 
 // Name returns the HetznerCluster name.

pkg/scope/cluster.go

@@ -37,13 +37,14 @@ import (
 
 // ClusterScopeParams defines the input parameters used to create a new scope.
 type ClusterScopeParams struct {
-	Client         client.Client
-	APIReader      client.Reader
-	Logger         logr.Logger
-	HetznerSecret  *corev1.Secret
-	HCloudClient   hcloudclient.Client
-	Cluster        *clusterv1.Cluster
-	HetznerCluster *infrav1.HetznerCluster
+	Client          client.Client
+	APIReader       client.Reader
+	Logger          logr.Logger
+	HetznerSecret   *corev1.Secret
+	HCloudClient    hcloudclient.Client
+	Cluster         *clusterv1.Cluster
+	HetznerCluster  *infrav1.HetznerCluster
+	ImageURLCommand string
 }
 
 // NewClusterScope creates a new Scope from the supplied parameters.