diff --git a/cmd/machine-config-controller/start.go b/cmd/machine-config-controller/start.go index 95c24badf1..5085bd64d2 100644 --- a/cmd/machine-config-controller/start.go +++ b/cmd/machine-config-controller/start.go @@ -17,11 +17,13 @@ import ( kubeletconfig "github.com/openshift/machine-config-operator/pkg/controller/kubelet-config" machinesetbootimage "github.com/openshift/machine-config-operator/pkg/controller/machine-set-boot-image" "github.com/openshift/machine-config-operator/pkg/controller/node" + "github.com/openshift/machine-config-operator/pkg/controller/osimagestream" "github.com/openshift/machine-config-operator/pkg/controller/pinnedimageset" "github.com/openshift/machine-config-operator/pkg/controller/render" "github.com/openshift/machine-config-operator/pkg/controller/template" "github.com/openshift/machine-config-operator/pkg/version" "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/leaderelection" "k8s.io/klog/v2" @@ -73,6 +75,12 @@ func runStartCmd(_ *cobra.Command, _ []string) { ctrlctx := ctrlcommon.CreateControllerContext(ctx, cb) + // Early start the config informer because feature gate depends on it + ctrlctx.ConfigInformerFactory.Start(ctrlctx.Stop) + if fgErr := ctrlctx.FeatureGatesHandler.Connect(ctx); fgErr != nil { + klog.Fatal(fmt.Errorf("failed to connect to feature gates %w", fgErr)) + } + go ctrlcommon.StartMetricsListener(startOpts.promMetricsListenAddress, ctrlctx.Stop, ctrlcommon.RegisterMCCMetrics) controllers := createControllers(ctrlctx) @@ -111,10 +119,6 @@ func runStartCmd(_ *cobra.Command, _ []string) { close(ctrlctx.InformersStarted) - if fgErr := ctrlctx.FeatureGatesHandler.Connect(ctx); fgErr != nil { - klog.Fatal(fmt.Errorf("failed to connect to feature gates %w", fgErr)) - } - if ctrlctx.FeatureGatesHandler.Enabled(features.FeatureGatePinnedImages) && ctrlctx.FeatureGatesHandler.Enabled(features.FeatureGateMachineConfigNodes) { pinnedImageSet := pinnedimageset.New( ctrlctx.InformerFactory.Machineconfiguration().V1().PinnedImageSets(), @@ -164,6 +168,38 @@ func runStartCmd(_ *cobra.Command, _ []string) { ctrlctx.OperatorInformerFactory.Start(ctrlctx.Stop) } + // Enable OSImageStreams if the FeatureGate is active and the deployment is not OKD + if osimagestream.IsFeatureEnabled(ctrlctx.FeatureGatesHandler) { + osImageStreamController := osimagestream.NewController( + ctrlctx.ClientBuilder.KubeClientOrDie("osimagestream-controller"), + ctrlctx.ClientBuilder.MachineConfigClientOrDie("osimagestream-controller"), + ctrlctx.InformerFactory.Machineconfiguration().V1().ControllerConfigs(), + ctrlctx.KubeNamespacedInformerFactory.Core().V1().ConfigMaps(), + ctrlctx.InformerFactory.Machineconfiguration().V1alpha1().OSImageStreams(), + ctrlctx.ConfigInformerFactory.Config().V1().ClusterVersions(), + nil, + ) + + go osImageStreamController.Run(ctrlctx.Stop) + // start the informers again to enable feature gated types. + // see comments in SharedInformerFactory interface. + ctrlctx.KubeNamespacedInformerFactory.Start(ctrlctx.Stop) + ctrlctx.InformerFactory.Start(ctrlctx.Stop) + ctrlctx.ConfigInformerFactory.Start(ctrlctx.Stop) + + if err = osImageStreamController.WaitBoot(); err != nil { + if !errors.IsNotFound(err) { + klog.Fatal(fmt.Errorf("failed to initialize the OSImageStream controller %w", err)) + } + + // Note: Till the feature is stable and the CoreOS images with the streams embedded have been + // running a for a few versions/cycles it's not a bad idea to ignore the situation where + // no streams are available and let the cluster start with zero available streams. + // TODO @pablintino: Track with a Jira card + klog.Warningf("Boot of the OSImageStreams Controller failed due to no available streams.") + } + } + for _, c := range controllers { go c.Run(2, ctrlctx.Stop) } @@ -202,10 +238,13 @@ func createControllers(ctx *ctrlcommon.ControllerContext) []ctrlcommon.Controlle rootOpts.templates, ctx.InformerFactory.Machineconfiguration().V1().ControllerConfigs(), ctx.InformerFactory.Machineconfiguration().V1().MachineConfigs(), + ctx.InformerFactory.Machineconfiguration().V1().MachineConfigPools(), + ctx.InformerFactory.Machineconfiguration().V1alpha1().OSImageStreams(), ctx.OpenShiftConfigKubeNamespacedInformerFactory.Core().V1().Secrets(), ctx.ConfigInformerFactory.Config().V1().APIServers(), ctx.ClientBuilder.KubeClientOrDie("template-controller"), ctx.ClientBuilder.MachineConfigClientOrDie("template-controller"), + ctx.FeatureGatesHandler, ), // Add all "sub-renderers here" kubeletconfig.New( @@ -213,6 +252,7 @@ func createControllers(ctx *ctrlcommon.ControllerContext) []ctrlcommon.Controlle ctx.InformerFactory.Machineconfiguration().V1().MachineConfigPools(), ctx.InformerFactory.Machineconfiguration().V1().ControllerConfigs(), ctx.InformerFactory.Machineconfiguration().V1().KubeletConfigs(), + ctx.InformerFactory.Machineconfiguration().V1alpha1().OSImageStreams(), ctx.ConfigInformerFactory.Config().V1().FeatureGates(), ctx.ConfigInformerFactory.Config().V1().Nodes(), ctx.ConfigInformerFactory.Config().V1().APIServers(), @@ -226,6 +266,7 @@ func createControllers(ctx *ctrlcommon.ControllerContext) []ctrlcommon.Controlle ctx.InformerFactory.Machineconfiguration().V1().MachineConfigPools(), ctx.InformerFactory.Machineconfiguration().V1().ControllerConfigs(), ctx.InformerFactory.Machineconfiguration().V1().ContainerRuntimeConfigs(), + ctx.InformerFactory.Machineconfiguration().V1alpha1().OSImageStreams(), ctx.ConfigInformerFactory.Config().V1().Images(), ctx.ConfigInformerFactory.Config().V1().ImageDigestMirrorSets(), ctx.ConfigInformerFactory.Config().V1().ImageTagMirrorSets(), @@ -246,6 +287,7 @@ func createControllers(ctx *ctrlcommon.ControllerContext) []ctrlcommon.Controlle ctx.InformerFactory.Machineconfiguration().V1().ContainerRuntimeConfigs(), ctx.InformerFactory.Machineconfiguration().V1().KubeletConfigs(), ctx.OperatorInformerFactory.Operator().V1().MachineConfigurations(), + ctx.InformerFactory.Machineconfiguration().V1alpha1().OSImageStreams(), ctx.ClientBuilder.KubeClientOrDie("render-controller"), ctx.ClientBuilder.MachineConfigClientOrDie("render-controller"), ctx.FeatureGatesHandler, diff --git a/pkg/controller/bootstrap/bootstrap.go b/pkg/controller/bootstrap/bootstrap.go index b99ad49859..c1be14ec37 100644 --- a/pkg/controller/bootstrap/bootstrap.go +++ b/pkg/controller/bootstrap/bootstrap.go @@ -2,11 +2,13 @@ package bootstrap import ( "bytes" + "context" "errors" "fmt" "io" "os" "path/filepath" + "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -23,6 +25,7 @@ import ( apicfgv1 "github.com/openshift/api/config/v1" apicfgv1alpha1 "github.com/openshift/api/config/v1alpha1" "github.com/openshift/api/features" + imagev1 "github.com/openshift/api/image/v1" mcfgv1 "github.com/openshift/api/machineconfiguration/v1" mcfgv1alpha1 "github.com/openshift/api/machineconfiguration/v1alpha1" apioperatorsv1alpha1 "github.com/openshift/api/operator/v1alpha1" @@ -32,6 +35,7 @@ import ( containerruntimeconfig "github.com/openshift/machine-config-operator/pkg/controller/container-runtime-config" "github.com/openshift/machine-config-operator/pkg/controller/internalreleaseimage" kubeletconfig "github.com/openshift/machine-config-operator/pkg/controller/kubelet-config" + "github.com/openshift/machine-config-operator/pkg/controller/osimagestream" "github.com/openshift/machine-config-operator/pkg/controller/render" "github.com/openshift/machine-config-operator/pkg/controller/template" "github.com/openshift/machine-config-operator/pkg/version" @@ -70,7 +74,7 @@ func (b *Bootstrap) Run(destDir string) error { return err } - psraw, err := getPullSecretFromSecret(psfraw) + pullSecret, err := getValidatePullSecretFromBytes(psfraw) if err != nil { return err } @@ -82,8 +86,13 @@ func (b *Bootstrap) Run(destDir string) error { apicfgv1.Install(scheme) apicfgv1alpha1.Install(scheme) corev1.AddToScheme(scheme) + imagev1.AddToScheme(scheme) codecFactory := serializer.NewCodecFactory(scheme) - decoder := codecFactory.UniversalDecoder(mcfgv1.GroupVersion, apioperatorsv1alpha1.GroupVersion, apicfgv1.GroupVersion, apicfgv1alpha1.GroupVersion, corev1.SchemeGroupVersion, mcfgv1alpha1.GroupVersion) + decoder := codecFactory.UniversalDecoder( + mcfgv1.GroupVersion, apioperatorsv1alpha1.GroupVersion, + apicfgv1.GroupVersion, apicfgv1alpha1.GroupVersion, + corev1.SchemeGroupVersion, mcfgv1alpha1.GroupVersion, + imagev1.SchemeGroupVersion) var ( cconfig *mcfgv1.ControllerConfig @@ -101,6 +110,7 @@ func (b *Bootstrap) Run(destDir string) error { imagePolicies []*apicfgv1.ImagePolicy imgCfg *apicfgv1.Image apiServer *apicfgv1.APIServer + imageStream *imagev1.ImageStream iri *mcfgv1alpha1.InternalReleaseImage ) for _, info := range infos { @@ -171,6 +181,17 @@ func (b *Bootstrap) Run(destDir string) error { if obj.GetName() == ctrlcommon.InternalReleaseImageInstanceName { iri = obj } + case *imagev1.ImageStream: + for _, tag := range obj.Spec.Tags { + if tag.Name == "machine-config-operator" { + if imageStream != nil { + klog.Infof("multiple ImageStream found. Previous ImageStream %s replaced by %s", imageStream.Name, obj.Name) + } + imageStream = obj + + } + } + // It's an ImageStream that doesn't look like the Release one (doesn't have our tag) default: klog.Infof("skipping %q [%d] manifest because of unhandled %T", file.Name(), idx+1, obji) } @@ -191,7 +212,41 @@ func (b *Bootstrap) Run(destDir string) error { return fmt.Errorf("error creating feature gates handler: %w", err) } - iconfigs, err := template.RunBootstrap(b.templatesDir, cconfig, psraw, apiServer) + var osImageStream *mcfgv1alpha1.OSImageStream + // Enable OSImageStreams if the FeatureGate is active and the deployment is not OKD + if osimagestream.IsFeatureEnabled(fgHandler) { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + // TODO @pablintino we need to change the factory API to avoid passing that cmLister at bootstrap + osImageStream, err = osimagestream.BuildOsImageStreamBootstrap(ctx, + pullSecret, + cconfig, + imageStream, + &osimagestream.OSImageTuple{ + OSImage: cconfig.Spec.BaseOSContainerImage, + OSExtensionsImage: cconfig.Spec.BaseOSExtensionsContainerImage, + }, + osimagestream.NewDefaultStreamSourceFactory(nil, &osimagestream.DefaultImagesInspectorFactory{}), + ) + if err != nil { + return fmt.Errorf("error inspecting available OSImageStreams: %w", err) + } + + // If no error happened override the ControllerConfig URLs with the default stream ones + if err == nil { + defaultStreamSet, err := osimagestream.GetOSImageStreamSetByName(osImageStream, "") + if err != nil { + // Should never happen + return fmt.Errorf("error getting default OSImageStreamSet: %w", err) + } + cconfig.Spec.BaseOSContainerImage = string(defaultStreamSet.OSImage) + cconfig.Spec.BaseOSExtensionsContainerImage = string(defaultStreamSet.OSExtensionsImage) + } + } + + pullSecretBytes := pullSecret.Data[corev1.DockerConfigJsonKey] + iconfigs, err := template.RunBootstrap(b.templatesDir, cconfig, pools, pullSecretBytes, apiServer, osImageStream) if err != nil { return err } @@ -199,7 +254,7 @@ func (b *Bootstrap) Run(destDir string) error { configs = append(configs, iconfigs...) - rconfigs, err := containerruntimeconfig.RunImageBootstrap(b.templatesDir, cconfig, pools, icspRules, idmsRules, itmsRules, imgCfg, clusterImagePolicies, imagePolicies, fgHandler) + rconfigs, err := containerruntimeconfig.RunImageBootstrap(b.templatesDir, cconfig, pools, icspRules, idmsRules, itmsRules, imgCfg, clusterImagePolicies, imagePolicies, fgHandler, osImageStream) if err != nil { return err } @@ -208,7 +263,7 @@ func (b *Bootstrap) Run(destDir string) error { configs = append(configs, rconfigs...) if len(crconfigs) > 0 { - containerRuntimeConfigs, err := containerruntimeconfig.RunContainerRuntimeBootstrap(b.templatesDir, crconfigs, cconfig, pools) + containerRuntimeConfigs, err := containerruntimeconfig.RunContainerRuntimeBootstrap(b.templatesDir, crconfigs, cconfig, pools, osImageStream) if err != nil { return err } @@ -217,7 +272,7 @@ func (b *Bootstrap) Run(destDir string) error { klog.Infof("Successfully generated MachineConfigs from containerruntime.") if featureGate != nil { - featureConfigs, err := kubeletconfig.RunFeatureGateBootstrap(b.templatesDir, fgHandler, nodeConfig, cconfig, pools, apiServer) + featureConfigs, err := kubeletconfig.RunFeatureGateBootstrap(b.templatesDir, fgHandler, nodeConfig, cconfig, pools, apiServer, osImageStream) if err != nil { return err } @@ -234,7 +289,7 @@ func (b *Bootstrap) Run(destDir string) error { } } if nodeConfig != nil { - nodeConfigs, err := kubeletconfig.RunNodeConfigBootstrap(b.templatesDir, fgHandler, cconfig, nodeConfig, pools, apiServer) + nodeConfigs, err := kubeletconfig.RunNodeConfigBootstrap(b.templatesDir, fgHandler, cconfig, nodeConfig, pools, apiServer, osImageStream) if err != nil { return err } @@ -243,7 +298,7 @@ func (b *Bootstrap) Run(destDir string) error { klog.Infof("Successfully generated MachineConfigs from node.Configs.") if len(kconfigs) > 0 { - kconfigs, err := kubeletconfig.RunKubeletBootstrap(b.templatesDir, kconfigs, cconfig, fgHandler, nodeConfig, pools, apiServer) + kconfigs, err := kubeletconfig.RunKubeletBootstrap(b.templatesDir, kconfigs, cconfig, fgHandler, nodeConfig, pools, apiServer, osImageStream) if err != nil { return err } @@ -274,7 +329,7 @@ func (b *Bootstrap) Run(destDir string) error { klog.Infof("Successfully created %d pre-built image component MachineConfigs for hybrid OCL.", len(preBuiltImageMCs)) } - fpools, gconfigs, err := render.RunBootstrap(pools, configs, cconfig) + fpools, gconfigs, err := render.RunBootstrap(pools, configs, cconfig, osImageStream) if err != nil { return err } @@ -357,7 +412,7 @@ func (b *Bootstrap) Run(destDir string) error { } -func getPullSecretFromSecret(sData []byte) ([]byte, error) { +func getValidatePullSecretFromBytes(sData []byte) (*corev1.Secret, error) { obji, err := runtime.Decode(kscheme.Codecs.UniversalDecoder(corev1.SchemeGroupVersion), sData) if err != nil { return nil, err @@ -369,7 +424,7 @@ func getPullSecretFromSecret(sData []byte) ([]byte, error) { if s.Type != corev1.SecretTypeDockerConfigJson { return nil, fmt.Errorf("expected secret type %s found %s", corev1.SecretTypeDockerConfigJson, s.Type) } - return s.Data[corev1.DockerConfigJsonKey], nil + return s, nil } type manifest struct { diff --git a/pkg/controller/common/helpers.go b/pkg/controller/common/helpers.go index c8ed0ac473..a66e85c4a7 100644 --- a/pkg/controller/common/helpers.go +++ b/pkg/controller/common/helpers.go @@ -28,6 +28,7 @@ import ( ign2types "github.com/coreos/ignition/config/v2_2/types" validate2 "github.com/coreos/ignition/config/validate" ign3error "github.com/coreos/ignition/v2/config/shared/errors" + "github.com/openshift/api/machineconfiguration/v1alpha1" ign3 "github.com/coreos/ignition/v2/config/v3_5" ign3types "github.com/coreos/ignition/v2/config/v3_5/types" @@ -68,7 +69,7 @@ func boolToPtr(b bool) *bool { // It uses the Ignition config from first object as base and appends all the rest. // Kernel arguments are concatenated. // It defaults to the OSImageURL provided by the CVO but allows a MC provided OSImageURL to take precedence. -func MergeMachineConfigs(configs []*mcfgv1.MachineConfig, cconfig *mcfgv1.ControllerConfig) (*mcfgv1.MachineConfig, error) { +func MergeMachineConfigs(configs []*mcfgv1.MachineConfig, cconfig *mcfgv1.ControllerConfig, imageStream *v1alpha1.OSImageStreamSet) (*mcfgv1.MachineConfig, error) { if len(configs) == 0 { return nil, nil } @@ -187,7 +188,7 @@ func MergeMachineConfigs(configs []*mcfgv1.MachineConfig, cconfig *mcfgv1.Contro // For layering, we want to let the user override OSImageURL again // The template configs always match what's in controllerconfig because they get rendered from there, // so the only way we get an override here is if the user adds something different - osImageURL := GetDefaultBaseImageContainer(&cconfig.Spec) + osImageURL := GetBaseImageContainer(&cconfig.Spec, imageStream) for _, cfg := range configs { if cfg.Spec.OSImageURL != "" { osImageURL = cfg.Spec.OSImageURL @@ -195,7 +196,7 @@ func MergeMachineConfigs(configs []*mcfgv1.MachineConfig, cconfig *mcfgv1.Contro } // Allow overriding the extensions container - baseOSExtensionsContainerImage := cconfig.Spec.BaseOSExtensionsContainerImage + baseOSExtensionsContainerImage := GetBaseExtensionsImageContainer(&cconfig.Spec, imageStream) for _, cfg := range configs { if cfg.Spec.BaseOSExtensionsContainerImage != "" { baseOSExtensionsContainerImage = cfg.Spec.BaseOSExtensionsContainerImage @@ -1035,9 +1036,20 @@ func GetIgnitionFileDataByPath(config *ign3types.Config, path string) ([]byte, e return nil, nil } -// GetDefaultBaseImageContainer returns the default bootable host base image. -func GetDefaultBaseImageContainer(cconfigspec *mcfgv1.ControllerConfigSpec) string { - return cconfigspec.BaseOSContainerImage +// GetBaseImageContainer returns the default bootable host base image. +func GetBaseImageContainer(cconfigspec *mcfgv1.ControllerConfigSpec, imageStream *v1alpha1.OSImageStreamSet) string { + if imageStream == nil { + return cconfigspec.BaseOSContainerImage + } + return string(imageStream.OSImage) +} + +// GetBaseExtensionsImageContainer returns the default bootable host base image. +func GetBaseExtensionsImageContainer(cconfigspec *mcfgv1.ControllerConfigSpec, imageStream *v1alpha1.OSImageStreamSet) string { + if imageStream == nil { + return cconfigspec.BaseOSExtensionsContainerImage + } + return string(imageStream.OSExtensionsImage) } // Configures common template FuncMaps used across all renderers. diff --git a/pkg/controller/common/helpers_test.go b/pkg/controller/common/helpers_test.go index c1f2f9e103..a1ff947dff 100644 --- a/pkg/controller/common/helpers_test.go +++ b/pkg/controller/common/helpers_test.go @@ -383,7 +383,7 @@ func TestMergeMachineConfigs(t *testing.T) { }, } inMachineConfigs := []*mcfgv1.MachineConfig{machineConfigFIPS} - mergedMachineConfig, err := MergeMachineConfigs(inMachineConfigs, cconfig) + mergedMachineConfig, err := MergeMachineConfigs(inMachineConfigs, cconfig, nil) require.Nil(t, err) // check that the outgoing config does have the version string set, @@ -397,7 +397,7 @@ func TestMergeMachineConfigs(t *testing.T) { require.Nil(t, err) expectedMachineConfig := &mcfgv1.MachineConfig{ Spec: mcfgv1.MachineConfigSpec{ - OSImageURL: GetDefaultBaseImageContainer(&cconfig.Spec), + OSImageURL: GetBaseImageContainer(&cconfig.Spec, nil), KernelArguments: []string{}, Config: runtime.RawExtension{ Raw: rawOutIgn, @@ -504,7 +504,7 @@ func TestMergeMachineConfigs(t *testing.T) { machineConfigIgnV2Merge, } - mergedMachineConfig, err = MergeMachineConfigs(inMachineConfigs, cconfig) + mergedMachineConfig, err = MergeMachineConfigs(inMachineConfigs, cconfig, nil) require.Nil(t, err) expectedMachineConfig = &mcfgv1.MachineConfig{ @@ -588,7 +588,7 @@ func TestMergeMachineConfigs(t *testing.T) { } cconfig = &mcfgv1.ControllerConfig{} - mergedMachineConfig, err = MergeMachineConfigs(inMachineConfigs, cconfig) + mergedMachineConfig, err = MergeMachineConfigs(inMachineConfigs, cconfig, nil) require.Nil(t, err) // The expectation here is that the merged config contains the MCs with name bbb (overrides aaa due to name) and ccc (overrides ddd due to pool) @@ -795,7 +795,7 @@ func TestSetDefaultFileOverwrite(t *testing.T) { require.Nil(t, err) cconfig := &mcfgv1.ControllerConfig{} - mergedMachineConfig, err := MergeMachineConfigs([]*mcfgv1.MachineConfig{machineConfigPreMerge}, cconfig) + mergedMachineConfig, err := MergeMachineConfigs([]*mcfgv1.MachineConfig{machineConfigPreMerge}, cconfig, nil) require.Nil(t, err) // Convert and create the expected post-merge config diff --git a/pkg/controller/container-runtime-config/container_runtime_config_bootstrap.go b/pkg/controller/container-runtime-config/container_runtime_config_bootstrap.go index a00f47f4c8..883902f63b 100644 --- a/pkg/controller/container-runtime-config/container_runtime_config_bootstrap.go +++ b/pkg/controller/container-runtime-config/container_runtime_config_bootstrap.go @@ -4,7 +4,9 @@ import ( "fmt" mcfgv1 "github.com/openshift/api/machineconfiguration/v1" + "github.com/openshift/api/machineconfiguration/v1alpha1" ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common" + "github.com/openshift/machine-config-operator/pkg/controller/osimagestream" "github.com/openshift/machine-config-operator/pkg/version" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -12,7 +14,7 @@ import ( ) // RunContainerRuntimeBootstrap generates ignition configs at bootstrap -func RunContainerRuntimeBootstrap(templateDir string, crconfigs []*mcfgv1.ContainerRuntimeConfig, controllerConfig *mcfgv1.ControllerConfig, mcpPools []*mcfgv1.MachineConfigPool) ([]*mcfgv1.MachineConfig, error) { +func RunContainerRuntimeBootstrap(templateDir string, crconfigs []*mcfgv1.ContainerRuntimeConfig, controllerConfig *mcfgv1.ControllerConfig, mcpPools []*mcfgv1.MachineConfigPool, osImageStream *v1alpha1.OSImageStream) ([]*mcfgv1.MachineConfig, error) { var res []*mcfgv1.MachineConfig managedKeyExist := make(map[string]bool) for _, cfg := range crconfigs { @@ -32,7 +34,12 @@ func RunContainerRuntimeBootstrap(templateDir string, crconfigs []*mcfgv1.Contai } role := pool.Name // Generate the original ContainerRuntimeConfig - originalStorageIgn, _, _, err := generateOriginalContainerRuntimeConfigs(templateDir, controllerConfig, role) + originalStorageIgn, _, _, err := generateOriginalContainerRuntimeConfigs( + templateDir, + controllerConfig, + role, + osimagestream.TryGetOSImageStreamFromPoolListByPoolName(osImageStream, mcpPools, pool.Name), + ) if err != nil { return nil, fmt.Errorf("could not generate origin ContainerRuntime Configs: %w", err) } diff --git a/pkg/controller/container-runtime-config/container_runtime_config_bootstrap_test.go b/pkg/controller/container-runtime-config/container_runtime_config_bootstrap_test.go index 5efeebc07d..df5e036cd9 100644 --- a/pkg/controller/container-runtime-config/container_runtime_config_bootstrap_test.go +++ b/pkg/controller/container-runtime-config/container_runtime_config_bootstrap_test.go @@ -29,7 +29,7 @@ func TestAddKubeletCfgAfterBootstrapKubeletCfg(t *testing.T) { f.mccrLister = append(f.mccrLister, ctrcfg) f.objects = append(f.objects, ctrcfg) - mcs, err := RunContainerRuntimeBootstrap("../../../templates", []*mcfgv1.ContainerRuntimeConfig{ctrcfg}, cc, pools) + mcs, err := RunContainerRuntimeBootstrap("../../../templates", []*mcfgv1.ContainerRuntimeConfig{ctrcfg}, cc, pools, nil) require.NoError(t, err) require.Len(t, mcs, 1) diff --git a/pkg/controller/container-runtime-config/container_runtime_config_controller.go b/pkg/controller/container-runtime-config/container_runtime_config_controller.go index 390663c721..787078ec43 100644 --- a/pkg/controller/container-runtime-config/container_runtime_config_controller.go +++ b/pkg/controller/container-runtime-config/container_runtime_config_controller.go @@ -13,11 +13,15 @@ import ( ign3types "github.com/coreos/ignition/v2/config/v3_5/types" apicfgv1 "github.com/openshift/api/config/v1" features "github.com/openshift/api/features" + "github.com/openshift/api/machineconfiguration/v1alpha1" apioperatorsv1alpha1 "github.com/openshift/api/operator/v1alpha1" configclientset "github.com/openshift/client-go/config/clientset/versioned" configinformers "github.com/openshift/client-go/config/informers/externalversions" cligoinformersv1 "github.com/openshift/client-go/config/informers/externalversions/config/v1" cligolistersv1 "github.com/openshift/client-go/config/listers/config/v1" + mcfginformersv1alpha1 "github.com/openshift/client-go/machineconfiguration/informers/externalversions/machineconfiguration/v1alpha1" + mcfglistersv1alpha1 "github.com/openshift/client-go/machineconfiguration/listers/machineconfiguration/v1alpha1" + "github.com/openshift/machine-config-operator/pkg/controller/osimagestream" runtimeutils "github.com/openshift/runtime-utils/pkg/registries" operatorinformersv1alpha1 "github.com/openshift/client-go/operator/informers/externalversions/operator/v1alpha1" @@ -95,6 +99,9 @@ type Controller struct { mccrLister mcfglistersv1.ContainerRuntimeConfigLister mccrListerSynced cache.InformerSynced + osImageStreamLister mcfglistersv1alpha1.OSImageStreamLister + osImageStreamListerSynced cache.InformerSynced + imgLister cligolistersv1.ImageLister imgListerSynced cache.InformerSynced @@ -133,6 +140,7 @@ func New( mcpInformer mcfginformersv1.MachineConfigPoolInformer, ccInformer mcfginformersv1.ControllerConfigInformer, mcrInformer mcfginformersv1.ContainerRuntimeConfigInformer, + osImageStreamInformer mcfginformersv1alpha1.OSImageStreamInformer, imgInformer cligoinformersv1.ImageInformer, idmsInformer cligoinformersv1.ImageDigestMirrorSetInformer, itmsInformer cligoinformersv1.ImageTagMirrorSetInformer, @@ -223,6 +231,11 @@ func New( ctrl.configInformerFactory = configInformerFactory + if osImageStreamInformer != nil && osimagestream.IsFeatureEnabled(ctrl.fgHandler) { + ctrl.osImageStreamLister = osImageStreamInformer.Lister() + ctrl.osImageStreamListerSynced = osImageStreamInformer.Informer().HasSynced + } + return ctrl } @@ -231,8 +244,11 @@ func (ctrl *Controller) Run(workers int, stopCh <-chan struct{}) { defer utilruntime.HandleCrash() defer ctrl.queue.ShutDown() defer ctrl.imgQueue.ShutDown() - listerCaches := []cache.InformerSynced{ctrl.mcpListerSynced, ctrl.mccrListerSynced, ctrl.ccListerSynced, - ctrl.imgListerSynced, ctrl.icspListerSynced, ctrl.idmsListerSynced, ctrl.itmsListerSynced, ctrl.clusterVersionListerSynced} + listerCaches := []cache.InformerSynced{ + ctrl.mcpListerSynced, ctrl.mccrListerSynced, ctrl.ccListerSynced, + ctrl.imgListerSynced, ctrl.icspListerSynced, ctrl.idmsListerSynced, + ctrl.itmsListerSynced, ctrl.clusterVersionListerSynced, + } if ctrl.sigstoreAPIEnabled() { ctrl.addImagePolicyObservers() @@ -241,6 +257,9 @@ func (ctrl *Controller) Run(workers int, stopCh <-chan struct{}) { listerCaches = append(listerCaches, ctrl.clusterImagePolicyListerSynced, ctrl.imagePolicyListerSynced) ctrl.addedPolicyObservers = true } + if ctrl.osImageStreamListerSynced != nil { + listerCaches = append(listerCaches, ctrl.osImageStreamListerSynced) + } if !cache.WaitForCacheSync(stopCh, listerCaches...) { return @@ -505,12 +524,12 @@ func (ctrl *Controller) handleImgErr(err error, key string) { } // generateOriginalContainerRuntimeConfigs returns rendered default storage, registries and policy config files -func generateOriginalContainerRuntimeConfigs(templateDir string, cc *mcfgv1.ControllerConfig, role string) (*ign3types.File, *ign3types.File, *ign3types.File, error) { +func generateOriginalContainerRuntimeConfigs(templateDir string, cc *mcfgv1.ControllerConfig, role string, imageStream *v1alpha1.OSImageStreamSet) (*ign3types.File, *ign3types.File, *ign3types.File, error) { // Render the default templates rc := &mtmpl.RenderConfig{ ControllerConfigSpec: &cc.Spec, } - generatedConfigs, err := mtmpl.GenerateMachineConfigsForRole(rc, role, templateDir) + generatedConfigs, err := mtmpl.GenerateMachineConfigsForRole(rc, role, templateDir, imageStream) if err != nil { return nil, nil, nil, fmt.Errorf("generateMachineConfigsforRole failed with error %w", err) } @@ -713,6 +732,15 @@ func (ctrl *Controller) syncContainerRuntimeConfig(key string) error { return fmt.Errorf("could not get ControllerConfig %w", err) } + var osImageStream *v1alpha1.OSImageStream + if ctrl.osImageStreamLister != nil { + osImageStream, err = ctrl.osImageStreamLister.Get(ctrlcommon.ClusterInstanceNameOSImageStream) + // TODO @pablintino For now consider the situation where no OSImageStreams are available + if err != nil && !errors.IsNotFound(err) { + return fmt.Errorf("could not get OSImageStream, err: %w", err) + } + } + // Find all MachineConfigPools mcpPools, err := ctrl.getPoolsForContainerRuntimeConfig(cfg) if err != nil { @@ -746,7 +774,12 @@ func (ctrl *Controller) syncContainerRuntimeConfig(key string) error { } } // Generate the original ContainerRuntimeConfig - originalStorageIgn, _, _, err := generateOriginalContainerRuntimeConfigs(ctrl.templatesDir, controllerConfig, role) + originalStorageIgn, _, _, err := generateOriginalContainerRuntimeConfigs( + ctrl.templatesDir, + controllerConfig, + role, + osimagestream.TryGetOSImageStreamFromPoolListByPoolName(osImageStream, mcpPools, pool.Name), + ) if err != nil { return ctrl.syncStatusOnly(cfg, err, "could not generate origin ContainerRuntime Configs: %v", err) } @@ -986,6 +1019,15 @@ func (ctrl *Controller) syncImageConfig(key string) error { return fmt.Errorf("could not get ControllerConfig %w", err) } + var osImageStream *v1alpha1.OSImageStream + if ctrl.osImageStreamLister != nil { + osImageStream, err = ctrl.osImageStreamLister.Get(ctrlcommon.ClusterInstanceNameOSImageStream) + // TODO @pablintino For now consider the situation where no OSImageStreams are available + if err != nil && !errors.IsNotFound(err) { + return fmt.Errorf("could not get OSImageStream, err: %w", err) + } + } + sel, err := metav1.LabelSelectorAsSelector(metav1.AddLabelToSelector(&metav1.LabelSelector{}, builtInLabelKey, "")) if err != nil { return err @@ -998,16 +1040,15 @@ func (ctrl *Controller) syncImageConfig(key string) error { for _, pool := range mcpPools { // To keep track of whether we "actually" got an updated image config applied := true - role := pool.Name // Get MachineConfig managedKey, err := getManagedKeyReg(pool, ctrl.client) if err != nil { return err } if err := retry.RetryOnConflict(updateBackoff, func() error { - registriesIgn, err := registriesConfigIgnition(ctrl.templatesDir, controllerConfig, role, releaseImage, + registriesIgn, err := registriesConfigIgnition(ctrl.templatesDir, controllerConfig, mcpPools, pool, releaseImage, imgcfg.Spec.RegistrySources.InsecureRegistries, registriesBlocked, policyBlocked, allowedRegs, - imgcfg.Spec.RegistrySources.ContainerRuntimeSearchRegistries, icspRules, idmsRules, itmsRules, clusterScopePolicies, scopeNamespacePolicies) + imgcfg.Spec.RegistrySources.ContainerRuntimeSearchRegistries, icspRules, idmsRules, itmsRules, clusterScopePolicies, scopeNamespacePolicies, osImageStream) if err != nil { return err } @@ -1068,10 +1109,10 @@ func (ctrl *Controller) syncIgnitionConfig(managedKey string, ignFile *ign3types return true, err } -func registriesConfigIgnition(templateDir string, controllerConfig *mcfgv1.ControllerConfig, role, releaseImage string, +func registriesConfigIgnition(templateDir string, controllerConfig *mcfgv1.ControllerConfig, pools []*mcfgv1.MachineConfigPool, pool *mcfgv1.MachineConfigPool, releaseImage string, insecureRegs, registriesBlocked, policyBlocked, allowedRegs, searchRegs []string, icspRules []*apioperatorsv1alpha1.ImageContentSourcePolicy, idmsRules []*apicfgv1.ImageDigestMirrorSet, itmsRules []*apicfgv1.ImageTagMirrorSet, - clusterScopePolicies map[string]signature.PolicyRequirements, scopeNamespacePolicies map[string]map[string]signature.PolicyRequirements) (*ign3types.Config, error) { + clusterScopePolicies map[string]signature.PolicyRequirements, scopeNamespacePolicies map[string]map[string]signature.PolicyRequirements, osImageStream *v1alpha1.OSImageStream) (*ign3types.Config, error) { var ( registriesTOML []byte @@ -1082,7 +1123,12 @@ func registriesConfigIgnition(templateDir string, controllerConfig *mcfgv1.Contr ) // Generate the original registries config - _, originalRegistriesIgn, originalPolicyIgn, err := generateOriginalContainerRuntimeConfigs(templateDir, controllerConfig, role) + _, originalRegistriesIgn, originalPolicyIgn, err := generateOriginalContainerRuntimeConfigs( + templateDir, + controllerConfig, + pool.Name, + osimagestream.TryGetOSImageStreamFromPoolListByPoolName(osImageStream, pools, pool.Name), + ) if err != nil { return nil, fmt.Errorf("could not generate original ContainerRuntime Configs: %w", err) } @@ -1215,7 +1261,7 @@ func (ctrl *Controller) syncImagePolicyStatusOnly(namespace, imagepolicy, condit // except that mcfgv1.Image is not available. func RunImageBootstrap(templateDir string, controllerConfig *mcfgv1.ControllerConfig, mcpPools []*mcfgv1.MachineConfigPool, icspRules []*apioperatorsv1alpha1.ImageContentSourcePolicy, idmsRules []*apicfgv1.ImageDigestMirrorSet, itmsRules []*apicfgv1.ImageTagMirrorSet, imgCfg *apicfgv1.Image, clusterImagePolicies []*apicfgv1.ClusterImagePolicy, imagePolicies []*apicfgv1.ImagePolicy, - fgHandler ctrlcommon.FeatureGatesHandler) ([]*mcfgv1.MachineConfig, error) { + fgHandler ctrlcommon.FeatureGatesHandler, osImageStream *v1alpha1.OSImageStream) ([]*mcfgv1.MachineConfig, error) { var ( insecureRegs, registriesBlocked, policyBlocked, allowedRegs, searchRegs []string @@ -1245,17 +1291,16 @@ func RunImageBootstrap(templateDir string, controllerConfig *mcfgv1.ControllerCo var res []*mcfgv1.MachineConfig for _, pool := range mcpPools { - role := pool.Name managedKey, err := getManagedKeyReg(pool, nil) if err != nil { return nil, err } - registriesIgn, err := registriesConfigIgnition(templateDir, controllerConfig, role, controllerConfig.Spec.ReleaseImage, - insecureRegs, registriesBlocked, policyBlocked, allowedRegs, searchRegs, icspRules, idmsRules, itmsRules, clusterScopePolicies, scopeNamespacePolicies) + registriesIgn, err := registriesConfigIgnition(templateDir, controllerConfig, mcpPools, pool, controllerConfig.Spec.ReleaseImage, + insecureRegs, registriesBlocked, policyBlocked, allowedRegs, searchRegs, icspRules, idmsRules, itmsRules, clusterScopePolicies, scopeNamespacePolicies, osImageStream) if err != nil { return nil, err } - mc, err := ctrlcommon.MachineConfigFromIgnConfig(role, managedKey, registriesIgn) + mc, err := ctrlcommon.MachineConfigFromIgnConfig(pool.Name, managedKey, registriesIgn) if err != nil { return nil, err } diff --git a/pkg/controller/container-runtime-config/container_runtime_config_controller_test.go b/pkg/controller/container-runtime-config/container_runtime_config_controller_test.go index eb8cf332c8..daf54b28fd 100644 --- a/pkg/controller/container-runtime-config/container_runtime_config_controller_test.go +++ b/pkg/controller/container-runtime-config/container_runtime_config_controller_test.go @@ -266,6 +266,7 @@ func (f *fixture) newController() *Controller { i.Machineconfiguration().V1().MachineConfigPools(), i.Machineconfiguration().V1().ControllerConfigs(), i.Machineconfiguration().V1().ContainerRuntimeConfigs(), + i.Machineconfiguration().V1alpha1().OSImageStreams(), ci.Config().V1().Images(), ci.Config().V1().ImageDigestMirrorSets(), ci.Config().V1().ImageTagMirrorSets(), @@ -1291,7 +1292,7 @@ func TestRunImageBootstrap(t *testing.T) { // set FeatureGateSigstoreImageVerification enabled for testing fgHandler := ctrlcommon.NewFeatureGatesHardcodedHandler([]apicfgv1.FeatureGateName{features.FeatureGateSigstoreImageVerification}, []apicfgv1.FeatureGateName{}) - mcs, err := RunImageBootstrap("../../../templates", cc, pools, tc.icspRules, tc.idmsRules, tc.itmsRules, imgCfg, tc.clusterImagePolicies, tc.imagePolicies, fgHandler) + mcs, err := RunImageBootstrap("../../../templates", cc, pools, tc.icspRules, tc.idmsRules, tc.itmsRules, imgCfg, tc.clusterImagePolicies, tc.imagePolicies, fgHandler, nil) require.NoError(t, err) require.Len(t, mcs, len(pools)) diff --git a/pkg/controller/kubelet-config/kubelet_config_bootstrap.go b/pkg/controller/kubelet-config/kubelet_config_bootstrap.go index 2dced82bab..e34bc9c568 100644 --- a/pkg/controller/kubelet-config/kubelet_config_bootstrap.go +++ b/pkg/controller/kubelet-config/kubelet_config_bootstrap.go @@ -6,14 +6,16 @@ import ( configv1 "github.com/openshift/api/config/v1" mcfgv1 "github.com/openshift/api/machineconfiguration/v1" + "github.com/openshift/api/machineconfiguration/v1alpha1" ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common" + "github.com/openshift/machine-config-operator/pkg/controller/osimagestream" "github.com/openshift/machine-config-operator/pkg/version" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" ) // RunKubeletBootstrap generates MachineConfig objects for mcpPools that would have been generated by syncKubeletConfig -func RunKubeletBootstrap(templateDir string, kubeletConfigs []*mcfgv1.KubeletConfig, controllerConfig *mcfgv1.ControllerConfig, fgHandler ctrlcommon.FeatureGatesHandler, nodeConfig *configv1.Node, mcpPools []*mcfgv1.MachineConfigPool, apiServer *configv1.APIServer) ([]*mcfgv1.MachineConfig, error) { +func RunKubeletBootstrap(templateDir string, kubeletConfigs []*mcfgv1.KubeletConfig, controllerConfig *mcfgv1.ControllerConfig, fgHandler ctrlcommon.FeatureGatesHandler, nodeConfig *configv1.Node, mcpPools []*mcfgv1.MachineConfigPool, apiServer *configv1.APIServer, osImageStream *v1alpha1.OSImageStream) ([]*mcfgv1.MachineConfig, error) { var res []*mcfgv1.MachineConfig managedKeyExist := make(map[string]bool) // Validate the KubeletConfig CR if exists @@ -40,7 +42,14 @@ func RunKubeletBootstrap(templateDir string, kubeletConfigs []*mcfgv1.KubeletCon } role := pool.Name - originalKubeConfig, err := generateOriginalKubeletConfigWithFeatureGates(controllerConfig, templateDir, role, fgHandler, apiServer) + originalKubeConfig, err := generateOriginalKubeletConfigWithFeatureGates( + controllerConfig, + templateDir, + role, + fgHandler, + apiServer, + osimagestream.TryGetOSImageStreamFromPoolListByPoolName(osImageStream, mcpPools, pool.Name), + ) if err != nil { return nil, err } diff --git a/pkg/controller/kubelet-config/kubelet_config_bootstrap_test.go b/pkg/controller/kubelet-config/kubelet_config_bootstrap_test.go index 1d83e9a5ab..a84a3c152d 100644 --- a/pkg/controller/kubelet-config/kubelet_config_bootstrap_test.go +++ b/pkg/controller/kubelet-config/kubelet_config_bootstrap_test.go @@ -77,7 +77,7 @@ func TestRunKubeletBootstrap(t *testing.T) { } fgHandler := ctrlcommon.NewFeatureGatesHardcodedHandler([]osev1.FeatureGateName{"Example"}, nil) - mcs, err := RunKubeletBootstrap("../../../templates", cfgs, cc, fgHandler, nil, pools, nil) + mcs, err := RunKubeletBootstrap("../../../templates", cfgs, cc, fgHandler, nil, pools, nil, nil) require.NoError(t, err) require.Len(t, mcs, len(cfgs)) @@ -216,7 +216,7 @@ func TestAddKubeletCfgAfterBootstrapKubeletCfg(t *testing.T) { f.mckLister = append(f.mckLister, kc) f.objects = append(f.objects, kc) - mcs, err := RunKubeletBootstrap("../../../templates", []*mcfgv1.KubeletConfig{kc}, cc, fgHandler, nil, pools, nil) + mcs, err := RunKubeletBootstrap("../../../templates", []*mcfgv1.KubeletConfig{kc}, cc, fgHandler, nil, pools, nil, nil) require.NoError(t, err) require.Len(t, mcs, 1) diff --git a/pkg/controller/kubelet-config/kubelet_config_controller.go b/pkg/controller/kubelet-config/kubelet_config_controller.go index 2316c0827d..337424967d 100644 --- a/pkg/controller/kubelet-config/kubelet_config_controller.go +++ b/pkg/controller/kubelet-config/kubelet_config_controller.go @@ -12,6 +12,10 @@ import ( "github.com/clarketm/json" ign3types "github.com/coreos/ignition/v2/config/v3_5/types" "github.com/imdario/mergo" + "github.com/openshift/api/machineconfiguration/v1alpha1" + mcfginformersv1alpha1 "github.com/openshift/client-go/machineconfiguration/informers/externalversions/machineconfiguration/v1alpha1" + mcfglistersv1alpha1 "github.com/openshift/client-go/machineconfiguration/listers/machineconfiguration/v1alpha1" + "github.com/openshift/machine-config-operator/pkg/controller/osimagestream" corev1 "k8s.io/api/core/v1" macherrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -93,6 +97,9 @@ type Controller struct { nodeConfigLister oselistersv1.NodeLister nodeConfigListerSynced cache.InformerSynced + osImageStreamLister mcfglistersv1alpha1.OSImageStreamLister + osImageStreamListerSynced cache.InformerSynced + apiserverLister oselistersv1.APIServerLister apiserverListerSynced cache.InformerSynced @@ -109,6 +116,7 @@ func New( mcpInformer mcfginformersv1.MachineConfigPoolInformer, ccInformer mcfginformersv1.ControllerConfigInformer, mkuInformer mcfginformersv1.KubeletConfigInformer, + osImageStreamInformer mcfginformersv1alpha1.OSImageStreamInformer, featInformer oseinformersv1.FeatureGateInformer, nodeConfigInformer oseinformersv1.NodeInformer, apiserverInformer oseinformersv1.APIServerInformer, @@ -183,6 +191,10 @@ func New( ctrl.apiserverLister = apiserverInformer.Lister() ctrl.apiserverListerSynced = apiserverInformer.Informer().HasSynced + if osImageStreamInformer != nil && osimagestream.IsFeatureEnabled(ctrl.fgHandler) { + ctrl.osImageStreamLister = osImageStreamInformer.Lister() + ctrl.osImageStreamListerSynced = osImageStreamInformer.Informer().HasSynced + } return ctrl } @@ -193,7 +205,18 @@ func (ctrl *Controller) Run(workers int, stopCh <-chan struct{}) { defer ctrl.featureQueue.ShutDown() defer ctrl.nodeConfigQueue.ShutDown() - if !cache.WaitForCacheSync(stopCh, ctrl.mcpListerSynced, ctrl.mckListerSynced, ctrl.ccListerSynced, ctrl.featListerSynced, ctrl.apiserverListerSynced) { + listerCaches := []cache.InformerSynced{ + ctrl.mcpListerSynced, + ctrl.mckListerSynced, + ctrl.ccListerSynced, + ctrl.featListerSynced, + ctrl.apiserverListerSynced, + } + if ctrl.osImageStreamListerSynced != nil { + listerCaches = append(listerCaches, ctrl.osImageStreamListerSynced) + } + + if !cache.WaitForCacheSync(stopCh, listerCaches...) { return } @@ -419,8 +442,8 @@ func (ctrl *Controller) handleFeatureErr(err error, key string) { // generateOriginalKubeletConfigWithFeatureGates generates a KubeletConfig and ensure the correct feature gates are set // based on the given FeatureGate. -func generateOriginalKubeletConfigWithFeatureGates(cc *mcfgv1.ControllerConfig, templatesDir, role string, fgHandler ctrlcommon.FeatureGatesHandler, apiServer *configv1.APIServer) (*kubeletconfigv1beta1.KubeletConfiguration, error) { - originalKubeletIgn, err := generateOriginalKubeletConfigIgn(cc, templatesDir, role, apiServer) +func generateOriginalKubeletConfigWithFeatureGates(cc *mcfgv1.ControllerConfig, templatesDir, role string, fgHandler ctrlcommon.FeatureGatesHandler, apiServer *configv1.APIServer, imageStream *v1alpha1.OSImageStreamSet) (*kubeletconfigv1beta1.KubeletConfiguration, error) { + originalKubeletIgn, err := generateOriginalKubeletConfigIgn(cc, templatesDir, role, apiServer, imageStream) if err != nil { return nil, fmt.Errorf("could not generate the original Kubelet config ignition: %w", err) } @@ -448,11 +471,11 @@ func generateOriginalKubeletConfigWithFeatureGates(cc *mcfgv1.ControllerConfig, return originalKubeConfig, nil } -func generateOriginalKubeletConfigIgn(cc *mcfgv1.ControllerConfig, templatesDir, role string, apiServer *osev1.APIServer) (*ign3types.File, error) { +func generateOriginalKubeletConfigIgn(cc *mcfgv1.ControllerConfig, templatesDir, role string, apiServer *osev1.APIServer, imageStream *v1alpha1.OSImageStreamSet) (*ign3types.File, error) { // Render the default templates tlsMinVersion, tlsCipherSuites := ctrlcommon.GetSecurityProfileCiphersFromAPIServer(apiServer) rc := &mtmpl.RenderConfig{ControllerConfigSpec: &cc.Spec, TLSMinVersion: tlsMinVersion, TLSCipherSuites: tlsCipherSuites} - generatedConfigs, err := mtmpl.GenerateMachineConfigsForRole(rc, role, templatesDir) + generatedConfigs, err := mtmpl.GenerateMachineConfigsForRole(rc, role, templatesDir, imageStream) if err != nil { return nil, fmt.Errorf("GenerateMachineConfigsforRole failed with error: %w", err) } @@ -597,6 +620,15 @@ func (ctrl *Controller) syncKubeletConfig(key string) error { return ctrl.syncStatusOnly(cfg, err, "could not get the TLSSecurityProfile from %v: %v", ctrlcommon.APIServerInstanceName, err) } + var osImageStream *v1alpha1.OSImageStream + if ctrl.osImageStreamLister != nil { + osImageStream, err = ctrl.osImageStreamLister.Get(ctrlcommon.ClusterInstanceNameOSImageStream) + // TODO @pablintino For now consider the situation where no OSImageStreams are available + if err != nil && !macherrors.IsNotFound(err) { + return fmt.Errorf("could not get OSImageStream, err: %w", err) + } + } + for _, pool := range mcpPools { role := pool.Name // Get MachineConfig @@ -616,7 +648,14 @@ func (ctrl *Controller) syncKubeletConfig(key string) error { return fmt.Errorf("could not get ControllerConfig %w", err) } - originalKubeConfig, err := generateOriginalKubeletConfigWithFeatureGates(cc, ctrl.templatesDir, role, ctrl.fgHandler, apiServer) + originalKubeConfig, err := generateOriginalKubeletConfigWithFeatureGates( + cc, + ctrl.templatesDir, + role, + ctrl.fgHandler, + apiServer, + osimagestream.TryGetOSImageStreamFromPoolListByPoolName(osImageStream, mcpPools, pool.Name), + ) if err != nil { return ctrl.syncStatusOnly(cfg, err, "could not get original kubelet config: %v", err) } diff --git a/pkg/controller/kubelet-config/kubelet_config_controller_test.go b/pkg/controller/kubelet-config/kubelet_config_controller_test.go index e1b6c84454..81ea5a6f71 100644 --- a/pkg/controller/kubelet-config/kubelet_config_controller_test.go +++ b/pkg/controller/kubelet-config/kubelet_config_controller_test.go @@ -220,6 +220,7 @@ func (f *fixture) newController(fgHandler ctrlcommon.FeatureGatesHandler) *Contr i.Machineconfiguration().V1().MachineConfigPools(), i.Machineconfiguration().V1().ControllerConfigs(), i.Machineconfiguration().V1().KubeletConfigs(), + i.Machineconfiguration().V1alpha1().OSImageStreams(), featinformer.Config().V1().FeatureGates(), featinformer.Config().V1().Nodes(), featinformer.Config().V1().APIServers(), @@ -1394,7 +1395,7 @@ func TestKubeletConfigTLSRender(t *testing.T) { fgHandler := ctrlcommon.NewFeatureGatesHardcodedHandler([]osev1.FeatureGateName{"Example"}, nil) ctrl := f.newController(fgHandler) - kubeletConfig, err := generateOriginalKubeletConfigIgn(cc, ctrl.templatesDir, "master", testCase.apiserver) + kubeletConfig, err := generateOriginalKubeletConfigIgn(cc, ctrl.templatesDir, "master", testCase.apiserver, nil) if err != nil { t.Errorf("could not generate kubelet config from templates %v", err) } diff --git a/pkg/controller/kubelet-config/kubelet_config_failswapon_test.go b/pkg/controller/kubelet-config/kubelet_config_failswapon_test.go index 509cb93911..e55d71fe72 100644 --- a/pkg/controller/kubelet-config/kubelet_config_failswapon_test.go +++ b/pkg/controller/kubelet-config/kubelet_config_failswapon_test.go @@ -35,7 +35,7 @@ func TestFailSwapOnConfiguration(t *testing.T) { for _, tc := range testCases { t.Run(tc.nodeRole, func(t *testing.T) { - kubeletConfig, err := generateOriginalKubeletConfigIgn(cc, ctrl.templatesDir, tc.nodeRole, &osev1.APIServer{}) + kubeletConfig, err := generateOriginalKubeletConfigIgn(cc, ctrl.templatesDir, tc.nodeRole, &osev1.APIServer{}, nil) require.NoError(t, err, "Failed to generate kubelet config for %s", tc.nodeRole) contents, err := ctrlcommon.DecodeIgnitionFileContents(kubeletConfig.Contents.Source, kubeletConfig.Contents.Compression) diff --git a/pkg/controller/kubelet-config/kubelet_config_features.go b/pkg/controller/kubelet-config/kubelet_config_features.go index 4dc62d4dca..e31ad18beb 100644 --- a/pkg/controller/kubelet-config/kubelet_config_features.go +++ b/pkg/controller/kubelet-config/kubelet_config_features.go @@ -8,6 +8,8 @@ import ( "github.com/clarketm/json" osev1 "github.com/openshift/api/config/v1" + "github.com/openshift/api/machineconfiguration/v1alpha1" + "github.com/openshift/machine-config-operator/pkg/controller/osimagestream" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -60,6 +62,15 @@ func (ctrl *Controller) syncFeatureHandler(key string) error { return fmt.Errorf("could not get ControllerConfig: %w", err) } + var osImageStream *v1alpha1.OSImageStream + if ctrl.osImageStreamLister != nil { + osImageStream, err = ctrl.osImageStreamLister.Get(ctrlcommon.ClusterInstanceNameOSImageStream) + // TODO @pablintino For now consider the situation where no OSImageStreams are available + if err != nil && !errors.IsNotFound(err) { + return fmt.Errorf("could not get OSImageStream, err: %w", err) + } + } + // Find all MachineConfigPools mcpPools, err := ctrl.mcpLister.List(labels.Everything()) if err != nil { @@ -98,7 +109,14 @@ func (ctrl *Controller) syncFeatureHandler(key string) error { } } - rawCfgIgn, err := generateKubeConfigIgnFromFeatures(cc, ctrl.templatesDir, role, ctrl.fgHandler, nodeConfig, apiServer) + rawCfgIgn, err := generateKubeConfigIgnFromFeatures( + cc, + ctrl.templatesDir, + role, + ctrl.fgHandler, + nodeConfig, + apiServer, osimagestream.TryGetOSImageStreamFromPoolListByPoolName(osImageStream, mcpPools, role), + ) if err != nil { return err } @@ -190,8 +208,8 @@ func generateFeatureMap(fgHandler ctrlcommon.FeatureGatesHandler, exclusions ... return &rv } -func generateKubeConfigIgnFromFeatures(cc *mcfgv1.ControllerConfig, templatesDir, role string, fgHandler ctrlcommon.FeatureGatesHandler, nodeConfig *osev1.Node, apiServer *osev1.APIServer) ([]byte, error) { - originalKubeConfig, err := generateOriginalKubeletConfigWithFeatureGates(cc, templatesDir, role, fgHandler, apiServer) +func generateKubeConfigIgnFromFeatures(cc *mcfgv1.ControllerConfig, templatesDir, role string, fgHandler ctrlcommon.FeatureGatesHandler, nodeConfig *osev1.Node, apiServer *osev1.APIServer, imageStream *v1alpha1.OSImageStreamSet) ([]byte, error) { + originalKubeConfig, err := generateOriginalKubeletConfigWithFeatureGates(cc, templatesDir, role, fgHandler, apiServer, imageStream) if err != nil { return nil, err } @@ -214,7 +232,7 @@ func generateKubeConfigIgnFromFeatures(cc *mcfgv1.ControllerConfig, templatesDir return rawCfgIgn, nil } -func RunFeatureGateBootstrap(templateDir string, fgHandler ctrlcommon.FeatureGatesHandler, nodeConfig *osev1.Node, controllerConfig *mcfgv1.ControllerConfig, mcpPools []*mcfgv1.MachineConfigPool, apiServer *osev1.APIServer) ([]*mcfgv1.MachineConfig, error) { +func RunFeatureGateBootstrap(templateDir string, fgHandler ctrlcommon.FeatureGatesHandler, nodeConfig *osev1.Node, controllerConfig *mcfgv1.ControllerConfig, mcpPools []*mcfgv1.MachineConfigPool, apiServer *osev1.APIServer, osImageStream *v1alpha1.OSImageStream) ([]*mcfgv1.MachineConfig, error) { machineConfigs := []*mcfgv1.MachineConfig{} for _, pool := range mcpPools { @@ -222,7 +240,15 @@ func RunFeatureGateBootstrap(templateDir string, fgHandler ctrlcommon.FeatureGat if nodeConfig == nil { nodeConfig = createNewDefaultNodeconfig() } - rawCfgIgn, err := generateKubeConfigIgnFromFeatures(controllerConfig, templateDir, role, fgHandler, nodeConfig, apiServer) + rawCfgIgn, err := generateKubeConfigIgnFromFeatures( + controllerConfig, + templateDir, + role, + fgHandler, + nodeConfig, + apiServer, + osimagestream.TryGetOSImageStreamFromPoolListByPoolName(osImageStream, mcpPools, role), + ) if err != nil { return nil, err } diff --git a/pkg/controller/kubelet-config/kubelet_config_features_test.go b/pkg/controller/kubelet-config/kubelet_config_features_test.go index 2f8ea4c21f..e2b5993db0 100644 --- a/pkg/controller/kubelet-config/kubelet_config_features_test.go +++ b/pkg/controller/kubelet-config/kubelet_config_features_test.go @@ -43,7 +43,7 @@ func TestFeatureGateDrift(t *testing.T) { ctrl := f.newController(fgHandler) // Generate kubelet config with feature gates applied - kubeletConfig, err := generateOriginalKubeletConfigWithFeatureGates(cc, ctrl.templatesDir, "master", fgHandler, nil) + kubeletConfig, err := generateOriginalKubeletConfigWithFeatureGates(cc, ctrl.templatesDir, "master", fgHandler, nil, nil) require.NoError(t, err) t.Logf("Generated Kubelet Config Feature Gates: %v", kubeletConfig.FeatureGates) @@ -172,7 +172,7 @@ func TestBootstrapFeaturesDefault(t *testing.T) { fgHandler := ctrlcommon.NewFeatureGatesHardcodedHandler(nil, nil) - mcs, err := RunFeatureGateBootstrap("../../../templates", fgHandler, nil, cc, mcps, nil) + mcs, err := RunFeatureGateBootstrap("../../../templates", fgHandler, nil, cc, mcps, nil, nil) if err != nil { t.Errorf("could not run feature gate bootstrap: %v", err) } @@ -193,7 +193,7 @@ func TestBootstrapFeaturesCustomNoUpgrade(t *testing.T) { fgHandler := ctrlcommon.NewFeatureGatesHardcodedHandler([]osev1.FeatureGateName{"CSIMigration"}, nil) - mcs, err := RunFeatureGateBootstrap("../../../templates", fgHandler, nil, cc, mcps, nil) + mcs, err := RunFeatureGateBootstrap("../../../templates", fgHandler, nil, cc, mcps, nil, nil) if err != nil { t.Errorf("could not run feature gate bootstrap: %v", err) } diff --git a/pkg/controller/kubelet-config/kubelet_config_nodes.go b/pkg/controller/kubelet-config/kubelet_config_nodes.go index 4e4ccc37d8..ed29fa7123 100644 --- a/pkg/controller/kubelet-config/kubelet_config_nodes.go +++ b/pkg/controller/kubelet-config/kubelet_config_nodes.go @@ -9,7 +9,9 @@ import ( "github.com/clarketm/json" osev1 "github.com/openshift/api/config/v1" mcfgv1 "github.com/openshift/api/machineconfiguration/v1" + "github.com/openshift/api/machineconfiguration/v1alpha1" ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common" + "github.com/openshift/machine-config-operator/pkg/controller/osimagestream" "github.com/openshift/machine-config-operator/pkg/version" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -93,6 +95,15 @@ func (ctrl *Controller) syncNodeConfigHandler(key string) error { return fmt.Errorf("could not get the TLSSecurityProfile from %v: %v", ctrlcommon.APIServerInstanceName, err) } + var osImageStream *v1alpha1.OSImageStream + if ctrl.osImageStreamLister != nil { + osImageStream, err = ctrl.osImageStreamLister.Get(ctrlcommon.ClusterInstanceNameOSImageStream) + // TODO @pablintino For now consider the situation where no OSImageStreams are available + if err != nil && !errors.IsNotFound(err) { + return fmt.Errorf("could not get OSImageStream, err: %w", err) + } + } + for _, pool := range mcpPools { role := pool.Name // Get MachineConfig @@ -112,7 +123,15 @@ func (ctrl *Controller) syncNodeConfigHandler(key string) error { return err } } - originalKubeConfig, err := generateOriginalKubeletConfigWithFeatureGates(cc, ctrl.templatesDir, role, ctrl.fgHandler, apiServer) + + originalKubeConfig, err := generateOriginalKubeletConfigWithFeatureGates( + cc, + ctrl.templatesDir, + role, + ctrl.fgHandler, + apiServer, + osimagestream.TryGetOSImageStreamFromPoolListByPoolName(osImageStream, mcpPools, pool.Name), + ) if err != nil { return err } @@ -271,7 +290,7 @@ func (ctrl *Controller) deleteNodeConfig(obj interface{}) { klog.V(4).Infof("Deleted node config %s and restored default config", nodeConfig.Name) } -func RunNodeConfigBootstrap(templateDir string, fgHandler ctrlcommon.FeatureGatesHandler, cconfig *mcfgv1.ControllerConfig, nodeConfig *osev1.Node, mcpPools []*mcfgv1.MachineConfigPool, apiServer *osev1.APIServer) ([]*mcfgv1.MachineConfig, error) { +func RunNodeConfigBootstrap(templateDir string, fgHandler ctrlcommon.FeatureGatesHandler, cconfig *mcfgv1.ControllerConfig, nodeConfig *osev1.Node, mcpPools []*mcfgv1.MachineConfigPool, apiServer *osev1.APIServer, osImageStream *v1alpha1.OSImageStream) ([]*mcfgv1.MachineConfig, error) { if nodeConfig == nil { return nil, fmt.Errorf("nodes.config.openshift.io resource not found") } @@ -290,7 +309,10 @@ func RunNodeConfigBootstrap(templateDir string, fgHandler ctrlcommon.FeatureGate if err != nil { return nil, err } - originalKubeConfig, err := generateOriginalKubeletConfigWithFeatureGates(cconfig, templateDir, role, fgHandler, apiServer) + originalKubeConfig, err := generateOriginalKubeletConfigWithFeatureGates( + cconfig, templateDir, role, fgHandler, apiServer, + osimagestream.TryGetOSImageStreamFromPoolListByPoolName(osImageStream, mcpPools, pool.Name), + ) if err != nil { return nil, err } diff --git a/pkg/controller/kubelet-config/kubelet_config_nodes_test.go b/pkg/controller/kubelet-config/kubelet_config_nodes_test.go index eccd460dd7..cad020ccee 100644 --- a/pkg/controller/kubelet-config/kubelet_config_nodes_test.go +++ b/pkg/controller/kubelet-config/kubelet_config_nodes_test.go @@ -28,7 +28,7 @@ func TestOriginalKubeletConfigDefaultNodeConfig(t *testing.T) { fgHandler := ctrlcommon.NewFeatureGatesHardcodedHandler([]osev1.FeatureGateName{"Example"}, nil) ctrl := f.newController(fgHandler) - kubeletConfig, err := generateOriginalKubeletConfigIgn(cc, ctrl.templatesDir, "master", nil) + kubeletConfig, err := generateOriginalKubeletConfigIgn(cc, ctrl.templatesDir, "master", nil, nil) if err != nil { t.Errorf("could not generate kubelet config from templates %v", err) } @@ -112,7 +112,7 @@ func TestBootstrapNodeConfigDefault(t *testing.T) { for _, configNode := range []*osev1.Node{configNodeCgroupDefault, configNodeCgroupV2, configNodeCgroupV1} { expect := expected[configNode] t.Run(fmt.Sprintf("Testing %v", expect.Name), func(t *testing.T) { - mcs, err := RunNodeConfigBootstrap("../../../templates", fgHandler, cc, configNode, mcps, nil) + mcs, err := RunNodeConfigBootstrap("../../../templates", fgHandler, cc, configNode, mcps, nil, nil) if configNode == configNodeCgroupV1 { require.Error(t, err) } else { @@ -138,7 +138,7 @@ func TestBootstrapNoNodeConfig(t *testing.T) { mcp := helpers.NewMachineConfigPool("worker", nil, helpers.WorkerSelector, "v0") mcps := []*mcfgv1.MachineConfigPool{mcp} - mcs, err := RunNodeConfigBootstrap("../../../templates", nil, cc, nil, mcps, nil) + mcs, err := RunNodeConfigBootstrap("../../../templates", nil, cc, nil, mcps, nil, nil) if err == nil { t.Errorf("expected an error while generating the kubelet config with no node config") } diff --git a/pkg/controller/osimagestream/helpers.go b/pkg/controller/osimagestream/helpers.go index 0cd7b552ac..872f2321cc 100644 --- a/pkg/controller/osimagestream/helpers.go +++ b/pkg/controller/osimagestream/helpers.go @@ -3,10 +3,12 @@ package osimagestream import ( "fmt" + "github.com/openshift/api/features" v1 "github.com/openshift/api/machineconfiguration/v1" "github.com/openshift/api/machineconfiguration/v1alpha1" "github.com/openshift/machine-config-operator/pkg/controller/common" "github.com/openshift/machine-config-operator/pkg/helpers" + "github.com/openshift/machine-config-operator/pkg/version" k8serrors "k8s.io/apimachinery/pkg/api/errors" ) @@ -58,3 +60,7 @@ func TryGetOSImageStreamFromPoolListByPoolName(osImageStream *v1alpha1.OSImageSt return TryGetOSImageStreamSetByName(osImageStream, targetPool.Spec.OSImageStream.Name) } + +func IsFeatureEnabled(fgHandler common.FeatureGatesHandler) bool { + return fgHandler.Enabled(features.FeatureGateOSStreams) && !version.IsSCOS() && !version.IsFCOS() +} diff --git a/pkg/controller/osimagestream/osimagestream_controller.go b/pkg/controller/osimagestream/osimagestream_controller.go index abeeb9d530..88d45e46ae 100644 --- a/pkg/controller/osimagestream/osimagestream_controller.go +++ b/pkg/controller/osimagestream/osimagestream_controller.go @@ -8,7 +8,6 @@ import ( mcfgv1 "github.com/openshift/api/machineconfiguration/v1" "github.com/openshift/api/machineconfiguration/v1alpha1" "github.com/openshift/machine-config-operator/pkg/version" - "k8s.io/apimachinery/pkg/api/errors" corelisterv1 "k8s.io/client-go/listers/core/v1" configinformersv1 "github.com/openshift/client-go/config/informers/externalversions/config/v1" @@ -24,6 +23,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" utilruntime "k8s.io/apimachinery/pkg/util/runtime" coreinformersv1 "k8s.io/client-go/informers/core/v1" @@ -150,22 +150,33 @@ func (ctrl *Controller) boot() error { return fmt.Errorf("error building the OSImageStream at runtime: %w", err) } + var updateOSImageStream *v1alpha1.OSImageStream if existingOSImageStream == nil { klog.V(4).Infof("Creating OSImageStream singleton instance as it doesn't exist") - if _, err = ctrl.client.MachineconfigurationV1alpha1().OSImageStreams().Create(context.TODO(), osImageStream, metav1.CreateOptions{}); err != nil { + updateOSImageStream, err = ctrl.client.MachineconfigurationV1alpha1().OSImageStreams().Create(context.TODO(), osImageStream, metav1.CreateOptions{}) + if err != nil { return fmt.Errorf("error creating the OSImageStream at runtime: %w", err) } } else { oldVersion := existingOSImageStream.Annotations[ctrlcommon.ReleaseImageVersionAnnotationKey] klog.V(4).Infof("Updating the OSImageStream singleton as it was created by a previous version (%s). New version: %s", oldVersion, version.Hash) - if _, err = ctrl.client. - MachineconfigurationV1alpha1(). - OSImageStreams(). - UpdateStatus(context.TODO(), osImageStream, metav1.UpdateOptions{}); err != nil { + // Update metadata/spec first (mainly for annotations) + existingOSImageStream.ObjectMeta.Annotations = osImageStream.ObjectMeta.Annotations + updateOSImageStream, err = ctrl.client.MachineconfigurationV1alpha1().OSImageStreams().Update(context.TODO(), existingOSImageStream, metav1.UpdateOptions{}) + if err != nil { return fmt.Errorf("error updating the OSImageStream at runtime: %w", err) } } - ctrl.osImageStream = osImageStream + + // Update the status subresource (both for newly created and updated resources) + updateOSImageStream.Status = osImageStream.Status + if _, err = ctrl.client. + MachineconfigurationV1alpha1(). + OSImageStreams(). + UpdateStatus(context.TODO(), updateOSImageStream, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("error updating the OSImageStream status at runtime: %w", err) + } + ctrl.osImageStream = updateOSImageStream return nil } @@ -174,7 +185,7 @@ func (ctrl *Controller) boot() error { func (ctrl *Controller) getExistingOSImageStream() (*v1alpha1.OSImageStream, error) { osImageStream, err := ctrl.osImageStreamLister.Get(ctrlcommon.ClusterInstanceNameOSImageStream) if err != nil { - if !errors.IsNotFound(err) { + if !apierrors.IsNotFound(err) { return nil, fmt.Errorf("failed to retrieve existing OSImageStream: %v", err) } return nil, nil diff --git a/pkg/controller/render/render_controller.go b/pkg/controller/render/render_controller.go index 055534a4a6..5d8144e0b5 100644 --- a/pkg/controller/render/render_controller.go +++ b/pkg/controller/render/render_controller.go @@ -10,16 +10,20 @@ import ( "github.com/openshift/api/features" mcfgv1 "github.com/openshift/api/machineconfiguration/v1" + "github.com/openshift/api/machineconfiguration/v1alpha1" opv1 "github.com/openshift/api/operator/v1" mcfgclientset "github.com/openshift/client-go/machineconfiguration/clientset/versioned" "github.com/openshift/client-go/machineconfiguration/clientset/versioned/scheme" mcfginformersv1 "github.com/openshift/client-go/machineconfiguration/informers/externalversions/machineconfiguration/v1" + mcfginformersv1alpha1 "github.com/openshift/client-go/machineconfiguration/informers/externalversions/machineconfiguration/v1alpha1" mcfglistersv1 "github.com/openshift/client-go/machineconfiguration/listers/machineconfiguration/v1" + mcfglistersv1alpha1 "github.com/openshift/client-go/machineconfiguration/listers/machineconfiguration/v1alpha1" mcopinformersv1 "github.com/openshift/client-go/operator/informers/externalversions/operator/v1" mcoplistersv1 "github.com/openshift/client-go/operator/listers/operator/v1" mcoResourceApply "github.com/openshift/machine-config-operator/lib/resourceapply" "github.com/openshift/machine-config-operator/pkg/apihelpers" ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common" + "github.com/openshift/machine-config-operator/pkg/controller/osimagestream" daemonconsts "github.com/openshift/machine-config-operator/pkg/daemon/constants" "github.com/openshift/machine-config-operator/pkg/version" corev1 "k8s.io/api/core/v1" @@ -65,11 +69,13 @@ type Controller struct { syncHandler func(mcp string) error enqueueMachineConfigPool func(*mcfgv1.MachineConfigPool) - mcpLister mcfglistersv1.MachineConfigPoolLister - mcLister mcfglistersv1.MachineConfigLister + mcpLister mcfglistersv1.MachineConfigPoolLister + mcLister mcfglistersv1.MachineConfigLister + osImageStreamLister mcfglistersv1alpha1.OSImageStreamLister - mcpListerSynced cache.InformerSynced - mcListerSynced cache.InformerSynced + mcpListerSynced cache.InformerSynced + mcListerSynced cache.InformerSynced + osImageStreamListerSynced cache.InformerSynced ccLister mcfglistersv1.ControllerConfigLister ccListerSynced cache.InformerSynced @@ -96,6 +102,7 @@ func New( crcInformer mcfginformersv1.ContainerRuntimeConfigInformer, mckInformer mcfginformersv1.KubeletConfigInformer, mcopInformer mcopinformersv1.MachineConfigurationInformer, + osImageStreamInformer mcfginformersv1alpha1.OSImageStreamInformer, kubeClient clientset.Interface, mcfgClient mcfgclientset.Interface, featureGatesHandler ctrlcommon.FeatureGatesHandler, @@ -140,6 +147,10 @@ func New( ctrl.mcopLister = mcopInformer.Lister() ctrl.mcopListerSynced = mcopInformer.Informer().HasSynced + if osImageStreamInformer != nil && osimagestream.IsFeatureEnabled(ctrl.fgHandler) { + ctrl.osImageStreamLister = osImageStreamInformer.Lister() + ctrl.osImageStreamListerSynced = osImageStreamInformer.Informer().HasSynced + } return ctrl } @@ -148,7 +159,13 @@ func (ctrl *Controller) Run(workers int, stopCh <-chan struct{}) { defer utilruntime.HandleCrash() defer ctrl.queue.ShutDown() - if !cache.WaitForCacheSync(stopCh, ctrl.mcpListerSynced, ctrl.mcListerSynced, ctrl.ccListerSynced) { + listerCaches := []cache.InformerSynced{ctrl.mcpListerSynced, ctrl.mcListerSynced, ctrl.ccListerSynced} + + // OSImageStreams and MCPs fetched only if FeatureGateOSStreams active + if ctrl.osImageStreamListerSynced != nil { + listerCaches = append(listerCaches, ctrl.osImageStreamListerSynced) + } + if !cache.WaitForCacheSync(stopCh, listerCaches...) { return } @@ -509,12 +526,12 @@ func (ctrl *Controller) garbageCollectRenderedConfigs(_ *mcfgv1.MachineConfigPoo return nil } -func (ctrl *Controller) getRenderedMachineConfig(pool *mcfgv1.MachineConfigPool, configs []*mcfgv1.MachineConfig, cc *mcfgv1.ControllerConfig) (*mcfgv1.MachineConfig, error) { +func (ctrl *Controller) getRenderedMachineConfig(pool *mcfgv1.MachineConfigPool, configs []*mcfgv1.MachineConfig, cc *mcfgv1.ControllerConfig, osImageStreamSet *v1alpha1.OSImageStreamSet) (*mcfgv1.MachineConfig, error) { // If we don't yet have a rendered MachineConfig on the pool, we cannot // perform reconciliation. So we must solely generate the rendered // MachineConfig. if pool.Spec.Configuration.Name == "" { - return generateRenderedMachineConfig(pool, configs, cc) + return generateRenderedMachineConfig(pool, configs, cc, osImageStreamSet) } // The pool has a rendered MachineConfig, so we can do more advanced @@ -523,7 +540,7 @@ func (ctrl *Controller) getRenderedMachineConfig(pool *mcfgv1.MachineConfigPool, // Degenerate case: When the renderedMC that the MCP is currently pointing to is deleted if apierrors.IsNotFound(err) { - generated, err := generateRenderedMachineConfig(pool, configs, cc) + generated, err := generateRenderedMachineConfig(pool, configs, cc, osImageStreamSet) if err != nil { return nil, err } @@ -542,7 +559,25 @@ func (ctrl *Controller) getRenderedMachineConfig(pool *mcfgv1.MachineConfigPool, mcop = *mcopPtr } - return generateAndValidateRenderedMachineConfig(currentMC, pool, configs, cc, &mcop.Spec.IrreconcilableValidationOverrides) + return generateAndValidateRenderedMachineConfig(currentMC, pool, configs, cc, &mcop.Spec.IrreconcilableValidationOverrides, osImageStreamSet) +} + +func (ctrl *Controller) getOSImageStreamForPool(pool *mcfgv1.MachineConfigPool) (*v1alpha1.OSImageStreamSet, error) { + if !osimagestream.IsFeatureEnabled(ctrl.fgHandler) || ctrl.osImageStreamLister == nil { + return nil, nil + } + + imageStream, err := ctrl.osImageStreamLister.Get(ctrlcommon.ClusterInstanceNameOSImageStream) + // TODO @pablintino For now consider the situation where no OSImageStreams are available + if err != nil && !errors.IsNotFound(err) { + return nil, fmt.Errorf("could not get OSImageStream for pool %s: %w", pool.Name, err) + } + + imageStreamSet, err := osimagestream.GetOSImageStreamSetByName(imageStream, pool.Spec.OSImageStream.Name) + if err != nil { + return nil, fmt.Errorf("could not get OSImageStreamSet for pool %s: %w", pool.Name, err) + } + return imageStreamSet, nil } func (ctrl *Controller) syncGeneratedMachineConfig(pool *mcfgv1.MachineConfigPool, configs []*mcfgv1.MachineConfig) error { @@ -555,14 +590,19 @@ func (ctrl *Controller) syncGeneratedMachineConfig(pool *mcfgv1.MachineConfigPoo return err } - generated, err := ctrl.getRenderedMachineConfig(pool, configs, cc) + osImageStreamSet, err := ctrl.getOSImageStreamForPool(pool) + if err != nil { + return err + } + + generated, err := ctrl.getRenderedMachineConfig(pool, configs, cc, osImageStreamSet) if err != nil { return fmt.Errorf("could not generate rendered MachineConfig: %w", err) } // Collect metric when OSImageURL was overridden var isOSImageURLOverridden bool - if generated.Spec.OSImageURL != ctrlcommon.GetDefaultBaseImageContainer(&cc.Spec) { + if generated.Spec.OSImageURL != ctrlcommon.GetBaseImageContainer(&cc.Spec, osImageStreamSet) { ctrlcommon.OSImageURLOverride.WithLabelValues(pool.Name).Set(1) isOSImageURLOverridden = true } else { @@ -613,7 +653,7 @@ func (ctrl *Controller) syncGeneratedMachineConfig(pool *mcfgv1.MachineConfigPoo } // generateRenderedMachineConfig takes all MCs for a given pool and returns a single rendered MC. For ex master-XXXX or worker-XXXX -func generateRenderedMachineConfig(pool *mcfgv1.MachineConfigPool, configs []*mcfgv1.MachineConfig, cconfig *mcfgv1.ControllerConfig) (*mcfgv1.MachineConfig, error) { +func generateRenderedMachineConfig(pool *mcfgv1.MachineConfigPool, configs []*mcfgv1.MachineConfig, cconfig *mcfgv1.ControllerConfig, osImageStreamSet *v1alpha1.OSImageStreamSet) (*mcfgv1.MachineConfig, error) { // Suppress rendered config generation until a corresponding new controller can roll out too. // https://bugzilla.redhat.com/show_bug.cgi?id=1879099 if genver, ok := cconfig.Annotations[daemonconsts.GeneratedByVersionAnnotationKey]; ok { @@ -644,7 +684,7 @@ func generateRenderedMachineConfig(pool *mcfgv1.MachineConfigPool, configs []*mc } } - merged, err := ctrlcommon.MergeMachineConfigs(configs, cconfig) + merged, err := ctrlcommon.MergeMachineConfigs(configs, cconfig, osImageStreamSet) if err != nil { return nil, err @@ -666,7 +706,7 @@ func generateRenderedMachineConfig(pool *mcfgv1.MachineConfigPool, configs []*mc // The operator needs to know the user overrode this, so it knows if it needs to skip the // OSImageURL check during upgrade -- if the user took over managing OS upgrades this way, // the operator shouldn't stop the rest of the upgrade from progressing/completing. - if merged.Spec.OSImageURL != ctrlcommon.GetDefaultBaseImageContainer(&cconfig.Spec) { + if merged.Spec.OSImageURL != ctrlcommon.GetBaseImageContainer(&cconfig.Spec, osImageStreamSet) { merged.Annotations[ctrlcommon.OSImageURLOverriddenKey] = "true" // Log a warning if the osImageURL is set using a tag instead of a digest if !strings.Contains(merged.Spec.OSImageURL, "sha256:") { @@ -689,11 +729,12 @@ func generateAndValidateRenderedMachineConfig( pool *mcfgv1.MachineConfigPool, configs []*mcfgv1.MachineConfig, cconfig *mcfgv1.ControllerConfig, - validationOverrides *opv1.IrreconcilableValidationOverrides) (*mcfgv1.MachineConfig, error) { + validationOverrides *opv1.IrreconcilableValidationOverrides, + osImageStreamSet *v1alpha1.OSImageStreamSet) (*mcfgv1.MachineConfig, error) { source := getMachineConfigRefs(configs) klog.V(4).Infof("Considering %d configs %s for MachineConfig generation", len(source), source) - generated, err := generateRenderedMachineConfig(pool, configs, cconfig) + generated, err := generateRenderedMachineConfig(pool, configs, cconfig, osImageStreamSet) if err != nil { return nil, err } @@ -738,7 +779,7 @@ func generateAndValidateRenderedMachineConfig( // RunBootstrap runs the render controller in bootstrap mode. // For each pool, it matches the machineconfigs based on label selector and // returns the generated machineconfigs and pool with CurrentMachineConfig status field set. -func RunBootstrap(pools []*mcfgv1.MachineConfigPool, configs []*mcfgv1.MachineConfig, cconfig *mcfgv1.ControllerConfig) ([]*mcfgv1.MachineConfigPool, []*mcfgv1.MachineConfig, error) { +func RunBootstrap(pools []*mcfgv1.MachineConfigPool, configs []*mcfgv1.MachineConfig, cconfig *mcfgv1.ControllerConfig, osImageStream *v1alpha1.OSImageStream) ([]*mcfgv1.MachineConfigPool, []*mcfgv1.MachineConfig, error) { var ( opools []*mcfgv1.MachineConfigPool oconfigs []*mcfgv1.MachineConfig @@ -748,8 +789,15 @@ func RunBootstrap(pools []*mcfgv1.MachineConfigPool, configs []*mcfgv1.MachineCo if err != nil { return nil, nil, err } + var osImageStreamSet *v1alpha1.OSImageStreamSet + if osImageStream != nil { + osImageStreamSet, err = osimagestream.GetOSImageStreamSetByName(osImageStream, pool.Spec.OSImageStream.Name) + if err != nil { + return nil, nil, fmt.Errorf("couldn't get the OSImageStream for pool %s %w", pool.Name, err) + } + } - generated, err := generateRenderedMachineConfig(pool, pcs, cconfig) + generated, err := generateRenderedMachineConfig(pool, pcs, cconfig, osImageStreamSet) if err != nil { return nil, nil, err } diff --git a/pkg/controller/render/render_controller_test.go b/pkg/controller/render/render_controller_test.go index 1d567da40e..b17c4a9575 100644 --- a/pkg/controller/render/render_controller_test.go +++ b/pkg/controller/render/render_controller_test.go @@ -8,6 +8,7 @@ import ( "github.com/clarketm/json" ign3types "github.com/coreos/ignition/v2/config/v3_5/types" + apicfgv1 "github.com/openshift/api/config/v1" configv1 "github.com/openshift/api/config/v1" mcopfake "github.com/openshift/client-go/operator/clientset/versioned/fake" operatorinformer "github.com/openshift/client-go/operator/informers/externalversions" @@ -64,6 +65,10 @@ func newFixture(t *testing.T) *fixture { f.t = t f.objects = []runtime.Object{} f.oObjects = []runtime.Object{} + f.fgHandler = ctrlcommon.NewFeatureGatesHardcodedHandler( + []apicfgv1.FeatureGateName{}, + []apicfgv1.FeatureGateName{}, + ) return f } @@ -76,7 +81,7 @@ func (f *fixture) newController() *Controller { c := New(i.Machineconfiguration().V1().MachineConfigPools(), i.Machineconfiguration().V1().MachineConfigs(), i.Machineconfiguration().V1().ControllerConfigs(), i.Machineconfiguration().V1().ContainerRuntimeConfigs(), i.Machineconfiguration().V1().KubeletConfigs(), oi.Operator().V1().MachineConfigurations(), - k8sfake.NewSimpleClientset(), f.client, f.fgHandler) + i.Machineconfiguration().V1alpha1().OSImageStreams(), k8sfake.NewSimpleClientset(), f.client, f.fgHandler) c.mcpListerSynced = alwaysReady c.mcListerSynced = alwaysReady @@ -299,7 +304,7 @@ func TestCreatesGeneratedMachineConfig(t *testing.T) { f.objects = append(f.objects, mcs[idx]) } - gmc, err := generateRenderedMachineConfig(mcp, mcs, cc) + gmc, err := generateRenderedMachineConfig(mcp, mcs, cc, nil) assert.NoError(t, err) mcpNew := mcp.DeepCopy() @@ -331,7 +336,7 @@ func TestIgnValidationGenerateRenderedMachineConfig(t *testing.T) { } cc := newControllerConfig(ctrlcommon.ControllerConfigName) - _, err := generateRenderedMachineConfig(mcp, mcs, cc) + _, err := generateRenderedMachineConfig(mcp, mcs, cc, nil) require.Nil(t, err) // verify that an invalid ignition config (here a config with content and an empty version, @@ -343,7 +348,7 @@ func TestIgnValidationGenerateRenderedMachineConfig(t *testing.T) { require.Nil(t, err) mcs[1].Spec.Config.Raw = rawIgnCfg - _, err = generateRenderedMachineConfig(mcp, mcs, cc) + _, err = generateRenderedMachineConfig(mcp, mcs, cc, nil) require.NotNil(t, err) // verify that a machine config with no ignition content will not fail validation @@ -352,7 +357,7 @@ func TestIgnValidationGenerateRenderedMachineConfig(t *testing.T) { require.Nil(t, err) mcs[1].Spec.Config.Raw = rawEmptyIgnCfg mcs[1].Spec.KernelArguments = append(mcs[1].Spec.KernelArguments, "test1") - _, err = generateRenderedMachineConfig(mcp, mcs, cc) + _, err = generateRenderedMachineConfig(mcp, mcs, cc, nil) require.Nil(t, err) } @@ -377,7 +382,7 @@ func TestUpdatesGeneratedMachineConfig(t *testing.T) { } cc := newControllerConfig(ctrlcommon.ControllerConfigName) - gmc, err := generateRenderedMachineConfig(mcp, mcs, cc) + gmc, err := generateRenderedMachineConfig(mcp, mcs, cc, nil) if err != nil { t.Fatal(err) } @@ -400,7 +405,7 @@ func TestUpdatesGeneratedMachineConfig(t *testing.T) { f.mcLister = append(f.mcLister, gmc) f.objects = append(f.objects, gmc) - expmc, err := generateRenderedMachineConfig(mcp, mcs, cc) + expmc, err := generateRenderedMachineConfig(mcp, mcs, cc, nil) if err != nil { t.Fatal(err) } @@ -424,7 +429,7 @@ func TestGenerateMachineConfigOverrideOSImageURL(t *testing.T) { cc := newControllerConfig(ctrlcommon.ControllerConfigName) - gmc, err := generateRenderedMachineConfig(mcp, mcs, cc) + gmc, err := generateRenderedMachineConfig(mcp, mcs, cc, nil) if err != nil { t.Fatal(err) } @@ -432,7 +437,7 @@ func TestGenerateMachineConfigOverrideOSImageURL(t *testing.T) { mcs = append(mcs, helpers.NewMachineConfig("00-test-cluster-master-1", map[string]string{"node-role/master": ""}, "dummy-change-2", []ign3types.File{})) - gmc, err = generateAndValidateRenderedMachineConfig(gmc, mcp, mcs, cc, nil) + gmc, err = generateAndValidateRenderedMachineConfig(gmc, mcp, mcs, cc, nil, nil) assert.NoError(t, err) assert.Equal(t, "dummy-change-2", gmc.Spec.OSImageURL) } @@ -446,12 +451,12 @@ func TestVersionSkew(t *testing.T) { cc := newControllerConfig(ctrlcommon.ControllerConfigName) cc.Annotations[daemonconsts.GeneratedByVersionAnnotationKey] = "different-version" - _, err := generateRenderedMachineConfig(mcp, mcs, cc) + _, err := generateRenderedMachineConfig(mcp, mcs, cc, nil) require.NotNil(t, err) // Now the same thing without overriding the version cc = newControllerConfig(ctrlcommon.ControllerConfigName) - gmc, err := generateRenderedMachineConfig(mcp, mcs, cc) + gmc, err := generateRenderedMachineConfig(mcp, mcs, cc, nil) require.Nil(t, err) require.NotNil(t, gmc) } @@ -464,14 +469,14 @@ func TestGenerateRenderedConfigOnLatestControllerVersionOnly(t *testing.T) { } version.Hash = "2" cc := newControllerConfig(ctrlcommon.ControllerConfigName) - _, err := generateRenderedMachineConfig(mcp, mcs, cc) + _, err := generateRenderedMachineConfig(mcp, mcs, cc, nil) require.NotNil(t, err) mcs = []*mcfgv1.MachineConfig{ helpers.NewMachineConfigWithAnnotation("00-updated-conf", map[string]string{"node-role/master": ""}, map[string]string{ctrlcommon.GeneratedByControllerVersionAnnotationKey: "2"}, "dummy-test-1", []ign3types.File{}), helpers.NewMachineConfigWithAnnotation("99-user-conf", map[string]string{"node-role/master": ""}, map[string]string{ctrlcommon.GeneratedByControllerVersionAnnotationKey: ""}, "user-data", []ign3types.File{}), } - _, err = generateRenderedMachineConfig(mcp, mcs, cc) + _, err = generateRenderedMachineConfig(mcp, mcs, cc, nil) require.Nil(t, err) } @@ -495,7 +500,7 @@ func TestDoNothing(t *testing.T) { } cc := newControllerConfig(ctrlcommon.ControllerConfigName) - gmc, err := generateRenderedMachineConfig(mcp, mcs, cc) + gmc, err := generateRenderedMachineConfig(mcp, mcs, cc, nil) if err != nil { t.Fatal(err) } @@ -604,7 +609,7 @@ func TestGenerateMachineConfigValidation(t *testing.T) { cc := newControllerConfig(ctrlcommon.ControllerConfigName) - gmc, err := generateAndValidateRenderedMachineConfig(currentMC, mcp, mcs, cc, nil) + gmc, err := generateAndValidateRenderedMachineConfig(currentMC, mcp, mcs, cc, nil, nil) assert.Error(t, err) assert.Nil(t, gmc) } diff --git a/pkg/controller/template/kubelet_config_dir_test.go b/pkg/controller/template/kubelet_config_dir_test.go index 826c2aa8db..d504651d61 100644 --- a/pkg/controller/template/kubelet_config_dir_test.go +++ b/pkg/controller/template/kubelet_config_dir_test.go @@ -62,7 +62,7 @@ func TestKubeletConfigDirParameter(t *testing.T) { controllerConfig, err := controllerConfigFromFile(tc.controllerConfig) require.NoError(t, err, "Failed to load controller config for %s", tc.name) - cfgs, err := generateTemplateMachineConfigs(&RenderConfig{&controllerConfig.Spec, `{"dummy":"dummy"}`, "dummy", nil, nil}, templateDir) + cfgs, err := generateTemplateMachineConfigs(&RenderConfig{&controllerConfig.Spec, `{"dummy":"dummy"}`, "dummy", nil, nil}, templateDir, nil, nil) require.NoError(t, err, "Failed to generate machine configs for %s", tc.name) kubeletConfigs := make(map[string]*string) @@ -127,7 +127,7 @@ func TestKubeletConfigDirParameterSpecific(t *testing.T) { controllerConfig, err := controllerConfigFromFile("./test_data/controller_config_aws.yaml") require.NoError(t, err, "Failed to load AWS controller config") - cfgs, err := generateTemplateMachineConfigs(&RenderConfig{&controllerConfig.Spec, `{"dummy":"dummy"}`, "dummy", nil, nil}, templateDir) + cfgs, err := generateTemplateMachineConfigs(&RenderConfig{&controllerConfig.Spec, `{"dummy":"dummy"}`, "dummy", nil, nil}, templateDir, nil, nil) require.NoError(t, err, "Failed to generate machine configs") var kubeletUnit *string diff --git a/pkg/controller/template/render.go b/pkg/controller/template/render.go index 35268a8d01..ca7b09d8c1 100644 --- a/pkg/controller/template/render.go +++ b/pkg/controller/template/render.go @@ -13,6 +13,8 @@ import ( "strings" "text/template" + "github.com/openshift/api/machineconfiguration/v1alpha1" + "github.com/openshift/machine-config-operator/pkg/controller/osimagestream" "k8s.io/klog/v2" configv1 "github.com/openshift/api/config/v1" @@ -78,7 +80,7 @@ const ( // /01-worker-kubelet/_base/files/random.conf.tmpl // /master/00-master/_base/units/kubelet.tmpl // /files/hostname.tmpl -func generateTemplateMachineConfigs(config *RenderConfig, templateDir string) ([]*mcfgv1.MachineConfig, error) { +func generateTemplateMachineConfigs(config *RenderConfig, templateDir string, pools []*mcfgv1.MachineConfigPool, osImageStream *v1alpha1.OSImageStream) ([]*mcfgv1.MachineConfig, error) { infos, err := ctrlcommon.ReadDir(templateDir) if err != nil { return nil, err @@ -101,7 +103,12 @@ func generateTemplateMachineConfigs(config *RenderConfig, templateDir string) ([ continue } - roleConfigs, err := GenerateMachineConfigsForRole(config, role, templateDir) + roleConfigs, err := GenerateMachineConfigsForRole( + config, + role, + templateDir, + osimagestream.TryGetOSImageStreamFromPoolListByPoolName(osImageStream, pools, role), + ) if err != nil { return nil, fmt.Errorf("failed to create MachineConfig for role %s: %w", role, err) } @@ -120,7 +127,7 @@ func generateTemplateMachineConfigs(config *RenderConfig, templateDir string) ([ } // GenerateMachineConfigsForRole creates MachineConfigs for the role provided -func GenerateMachineConfigsForRole(config *RenderConfig, role, templateDir string) ([]*mcfgv1.MachineConfig, error) { +func GenerateMachineConfigsForRole(config *RenderConfig, role, templateDir string, imageStream *v1alpha1.OSImageStreamSet) ([]*mcfgv1.MachineConfig, error) { rolePath := role //nolint:goconst if role != workerRole && role != masterRole && role != arbiterRole { @@ -147,7 +154,7 @@ func GenerateMachineConfigsForRole(config *RenderConfig, role, templateDir strin } name := info.Name() namePath := filepath.Join(path, name) - nameConfig, err := generateMachineConfigForName(config, role, name, templateDir, namePath, &commonAdded) + nameConfig, err := generateMachineConfigForName(config, role, name, templateDir, namePath, imageStream, &commonAdded) if err != nil { return nil, err } @@ -257,7 +264,7 @@ func getPaths(config *RenderConfig, platformString string) []string { return platformBasedPaths } -func generateMachineConfigForName(config *RenderConfig, role, name, templateDir, path string, commonAdded *bool) (*mcfgv1.MachineConfig, error) { +func generateMachineConfigForName(config *RenderConfig, role, name, templateDir, path string, imageStream *v1alpha1.OSImageStreamSet, commonAdded *bool) (*mcfgv1.MachineConfig, error) { platformString, err := platformStringFromControllerConfigSpec(config.ControllerConfigSpec) if err != nil { return nil, err @@ -364,10 +371,11 @@ func generateMachineConfigForName(config *RenderConfig, role, name, templateDir, // TODO(jkyros): you might think you can remove this since we override later when we merge // config, but resourcemerge doesn't blank this field out once it's populated // so if you end up on a cluster where it was ever populated in this machineconfig, it - // will keep that last value forever once you upgrade...which is a problen now that we allow OSImageURL overrides + // will keep that last value forever once you upgrade...which is a problem now that we allow OSImageURL overrides // because it will look like an override when it shouldn't be. So don't take this out until you've solved that. // And inject the osimageurl here - mcfg.Spec.OSImageURL = ctrlcommon.GetDefaultBaseImageContainer(config.ControllerConfigSpec) + // TODO: @pablintino do we need to do the same with the extensions? + mcfg.Spec.OSImageURL = ctrlcommon.GetBaseImageContainer(config.ControllerConfigSpec, imageStream) return mcfg, nil } diff --git a/pkg/controller/template/render_test.go b/pkg/controller/template/render_test.go index 8fd1f8f0ee..415d5e1cba 100644 --- a/pkg/controller/template/render_test.go +++ b/pkg/controller/template/render_test.go @@ -239,14 +239,14 @@ func TestInvalidPlatform(t *testing.T) { // we must treat unrecognized constants as "none" controllerConfig.Spec.Infra.Status.PlatformStatus.Type = "_bad_" - _, err = generateTemplateMachineConfigs(&RenderConfig{&controllerConfig.Spec, `{"dummy":"dummy"}`, "dummy", nil, nil}, templateDir) + _, err = generateTemplateMachineConfigs(&RenderConfig{&controllerConfig.Spec, `{"dummy":"dummy"}`, "dummy", nil, nil}, templateDir, nil, nil) if err != nil { t.Errorf("expect nil error, got: %v", err) } // explicitly blocked controllerConfig.Spec.Infra.Status.PlatformStatus.Type = "_base" - _, err = generateTemplateMachineConfigs(&RenderConfig{&controllerConfig.Spec, `{"dummy":"dummy"}`, "dummy", nil, nil}, templateDir) + _, err = generateTemplateMachineConfigs(&RenderConfig{&controllerConfig.Spec, `{"dummy":"dummy"}`, "dummy", nil, nil}, templateDir, nil, nil) expectErr(err, "failed to create MachineConfig for role master: platform _base unsupported") } @@ -257,7 +257,7 @@ func TestGenerateMachineConfigs(t *testing.T) { t.Fatalf("failed to get controllerconfig config: %v", err) } - cfgs, err := generateTemplateMachineConfigs(&RenderConfig{&controllerConfig.Spec, `{"dummy":"dummy"}`, "dummy", nil, nil}, templateDir) + cfgs, err := generateTemplateMachineConfigs(&RenderConfig{&controllerConfig.Spec, `{"dummy":"dummy"}`, "dummy", nil, nil}, templateDir, nil, nil) if err != nil { t.Fatalf("failed to generate machine configs: %v", err) } diff --git a/pkg/controller/template/template_controller.go b/pkg/controller/template/template_controller.go index 4fda59c9ec..c478c21376 100644 --- a/pkg/controller/template/template_controller.go +++ b/pkg/controller/template/template_controller.go @@ -14,12 +14,17 @@ import ( "time" mcfgv1 "github.com/openshift/api/machineconfiguration/v1" + "github.com/openshift/api/machineconfiguration/v1alpha1" mcfgclientset "github.com/openshift/client-go/machineconfiguration/clientset/versioned" "github.com/openshift/client-go/machineconfiguration/clientset/versioned/scheme" mcfginformersv1 "github.com/openshift/client-go/machineconfiguration/informers/externalversions/machineconfiguration/v1" + mcfginformersv1alpha1 "github.com/openshift/client-go/machineconfiguration/informers/externalversions/machineconfiguration/v1alpha1" mcfglistersv1 "github.com/openshift/client-go/machineconfiguration/listers/machineconfiguration/v1" + mcfglistersv1alpha1 "github.com/openshift/client-go/machineconfiguration/listers/machineconfiguration/v1alpha1" mcoResourceApply "github.com/openshift/machine-config-operator/lib/resourceapply" ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common" + "github.com/openshift/machine-config-operator/pkg/controller/osimagestream" + "k8s.io/apimachinery/pkg/labels" "k8s.io/klog/v2" configv1 "github.com/openshift/api/config/v1" @@ -63,15 +68,21 @@ type Controller struct { syncHandler func(ccKey string) error enqueueControllerConfig func(*mcfgv1.ControllerConfig) - ccLister mcfglistersv1.ControllerConfigLister - mcLister mcfglistersv1.MachineConfigLister + ccLister mcfglistersv1.ControllerConfigLister + mcLister mcfglistersv1.MachineConfigLister + mcpLister mcfglistersv1.MachineConfigPoolLister + osImageStreamLister mcfglistersv1alpha1.OSImageStreamLister apiserverLister configlistersv1.APIServerLister apiserverListerSynced cache.InformerSynced - ccListerSynced cache.InformerSynced - mcListerSynced cache.InformerSynced - secretsInformerSynced cache.InformerSynced + ccListerSynced cache.InformerSynced + mcListerSynced cache.InformerSynced + mcpListerSynced cache.InformerSynced + secretsInformerSynced cache.InformerSynced + osImageStreamListerSynced cache.InformerSynced + + fgHandler ctrlcommon.FeatureGatesHandler queue workqueue.TypedRateLimitingInterface[string] } @@ -81,10 +92,13 @@ func New( templatesDir string, ccInformer mcfginformersv1.ControllerConfigInformer, mcInformer mcfginformersv1.MachineConfigInformer, + mcpInformer mcfginformersv1.MachineConfigPoolInformer, + osImageStreamInformer mcfginformersv1alpha1.OSImageStreamInformer, secretsInformer coreinformersv1.SecretInformer, apiserverInformer configinformersv1.APIServerInformer, kubeClient clientset.Interface, mcfgClient mcfgclientset.Interface, + fgHandler ctrlcommon.FeatureGatesHandler, ) *Controller { eventBroadcaster := record.NewBroadcaster() eventBroadcaster.StartLogging(klog.Infof) @@ -98,6 +112,7 @@ func New( queue: workqueue.NewTypedRateLimitingQueueWithConfig[string]( workqueue.DefaultTypedControllerRateLimiter[string](), workqueue.TypedRateLimitingQueueConfig[string]{Name: "machineconfigcontroller-templatecontroller"}), + fgHandler: fgHandler, } ccInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ @@ -130,6 +145,7 @@ func New( ctrl.ccLister = ccInformer.Lister() ctrl.mcLister = mcInformer.Lister() + ctrl.ccListerSynced = ccInformer.Informer().HasSynced ctrl.mcListerSynced = mcInformer.Informer().HasSynced ctrl.secretsInformerSynced = secretsInformer.Informer().HasSynced @@ -137,6 +153,17 @@ func New( ctrl.apiserverLister = apiserverInformer.Lister() ctrl.apiserverListerSynced = apiserverInformer.Informer().HasSynced + if osimagestream.IsFeatureEnabled(ctrl.fgHandler) { + if osImageStreamInformer != nil { + ctrl.osImageStreamLister = osImageStreamInformer.Lister() + ctrl.osImageStreamListerSynced = osImageStreamInformer.Informer().HasSynced + } + if mcpInformer != nil { + ctrl.mcpLister = mcpInformer.Lister() + ctrl.mcpListerSynced = mcpInformer.Informer().HasSynced + } + } + return ctrl } @@ -287,7 +314,16 @@ func (ctrl *Controller) Run(workers int, stopCh <-chan struct{}) { defer utilruntime.HandleCrash() defer ctrl.queue.ShutDown() - if !cache.WaitForCacheSync(stopCh, ctrl.ccListerSynced, ctrl.mcListerSynced, ctrl.secretsInformerSynced) { + listerCaches := []cache.InformerSynced{ctrl.ccListerSynced, ctrl.mcListerSynced, ctrl.secretsInformerSynced} + + // OSImageStreams and MCPs fetched only if FeatureGateOSStreams active + if ctrl.osImageStreamListerSynced != nil { + listerCaches = append(listerCaches, ctrl.osImageStreamListerSynced) + } + if ctrl.mcpListerSynced != nil { + listerCaches = append(listerCaches, ctrl.mcpListerSynced) + } + if !cache.WaitForCacheSync(stopCh, listerCaches...) { return } @@ -633,7 +669,23 @@ func (ctrl *Controller) syncControllerConfig(key string) error { return ctrl.syncFailingStatus(cfg, err) } - mcs, err := getMachineConfigsForControllerConfig(ctrl.templatesDir, cfg, clusterPullSecretRaw, apiServer) + var osImageStream *v1alpha1.OSImageStream + var mcps []*mcfgv1.MachineConfigPool + if ctrl.osImageStreamLister != nil { + // Get all MachineConfigPools + mcps, err = ctrl.mcpLister.List(labels.Everything()) + if err != nil { + return fmt.Errorf("error listing MachineConfigPools: %w", err) + } + + osImageStream, err = ctrl.osImageStreamLister.Get(ctrlcommon.ClusterInstanceNameOSImageStream) + // TODO @pablintino For now consider the situation where no OSImageStreams are available + if err != nil && !errors.IsNotFound(err) { + return fmt.Errorf("could not get OSImageStream, err: %w", err) + } + } + + mcs, err := getMachineConfigsForControllerConfig(ctrl.templatesDir, cfg, mcps, clusterPullSecretRaw, apiServer, osImageStream) if err != nil { return ctrl.syncFailingStatus(cfg, err) } @@ -652,7 +704,7 @@ func (ctrl *Controller) syncControllerConfig(key string) error { return ctrl.syncCompletedStatus(cfg) } -func getMachineConfigsForControllerConfig(templatesDir string, config *mcfgv1.ControllerConfig, clusterPullSecretRaw []byte, apiServer *configv1.APIServer) ([]*mcfgv1.MachineConfig, error) { +func getMachineConfigsForControllerConfig(templatesDir string, config *mcfgv1.ControllerConfig, pools []*mcfgv1.MachineConfigPool, clusterPullSecretRaw []byte, apiServer *configv1.APIServer, osImageStream *v1alpha1.OSImageStream) ([]*mcfgv1.MachineConfig, error) { buf := &bytes.Buffer{} if err := json.Compact(buf, clusterPullSecretRaw); err != nil { return nil, fmt.Errorf("couldn't compact pullsecret %q: %w", string(clusterPullSecretRaw), err) @@ -664,7 +716,7 @@ func getMachineConfigsForControllerConfig(templatesDir string, config *mcfgv1.Co TLSMinVersion: tlsMinVersion, TLSCipherSuites: tlsCipherSuites, } - mcs, err := generateTemplateMachineConfigs(rc, templatesDir) + mcs, err := generateTemplateMachineConfigs(rc, templatesDir, pools, osImageStream) if err != nil { return nil, err } @@ -678,7 +730,7 @@ func getMachineConfigsForControllerConfig(templatesDir string, config *mcfgv1.Co return mcs, nil } -// RunBootstrap runs the tempate controller in boostrap mode. -func RunBootstrap(templatesDir string, config *mcfgv1.ControllerConfig, pullSecretRaw []byte, apiServer *configv1.APIServer) ([]*mcfgv1.MachineConfig, error) { - return getMachineConfigsForControllerConfig(templatesDir, config, pullSecretRaw, apiServer) +// RunBootstrap runs the template controller in boostrap mode. +func RunBootstrap(templatesDir string, config *mcfgv1.ControllerConfig, pools []*mcfgv1.MachineConfigPool, pullSecretRaw []byte, apiServer *configv1.APIServer, osImageStream *v1alpha1.OSImageStream) ([]*mcfgv1.MachineConfig, error) { + return getMachineConfigsForControllerConfig(templatesDir, config, pools, pullSecretRaw, apiServer, osImageStream) } diff --git a/pkg/controller/template/template_controller_test.go b/pkg/controller/template/template_controller_test.go index 0c0373f70c..e52fc09588 100644 --- a/pkg/controller/template/template_controller_test.go +++ b/pkg/controller/template/template_controller_test.go @@ -8,6 +8,7 @@ import ( "time" "github.com/clarketm/json" + apicfgv1 "github.com/openshift/api/config/v1" configv1 "github.com/openshift/api/config/v1" fakeconfigv1client "github.com/openshift/client-go/config/clientset/versioned/fake" oseconfigfake "github.com/openshift/client-go/config/clientset/versioned/fake" @@ -56,6 +57,8 @@ type fixture struct { objects []runtime.Object oseobjects []runtime.Object apiservers []runtime.Object + + fgHandler ctrlcommon.FeatureGatesHandler } func newFixture(t *testing.T) *fixture { @@ -64,6 +67,10 @@ func newFixture(t *testing.T) *fixture { f.objects = []runtime.Object{} f.kubeobjects = []runtime.Object{} f.oseobjects = []runtime.Object{} + f.fgHandler = ctrlcommon.NewFeatureGatesHardcodedHandler( + []apicfgv1.FeatureGateName{}, + []apicfgv1.FeatureGateName{}, + ) return f } @@ -112,8 +119,10 @@ func (f *fixture) newController() *Controller { i := informers.NewSharedInformerFactory(f.client, noResyncPeriodFunc()) c := New(templateDir, - i.Machineconfiguration().V1().ControllerConfigs(), i.Machineconfiguration().V1().MachineConfigs(), cinformer.Core().V1().Secrets(), - apiserverinformer.Config().V1().APIServers(), f.kubeclient, f.client) + i.Machineconfiguration().V1().ControllerConfigs(), i.Machineconfiguration().V1().MachineConfigs(), + i.Machineconfiguration().V1().MachineConfigPools(), i.Machineconfiguration().V1alpha1().OSImageStreams(), + cinformer.Core().V1().Secrets(), apiserverinformer.Config().V1().APIServers(), f.kubeclient, + f.client, f.fgHandler) c.ccListerSynced = alwaysReady c.mcListerSynced = alwaysReady @@ -289,7 +298,7 @@ func TestCreatesMachineConfigs(t *testing.T) { f.objects = append(f.objects, cc) f.kubeobjects = append(f.kubeobjects, ps) - expMCs, err := getMachineConfigsForControllerConfig(templateDir, cc, []byte(`{"dummy": "dummy"}`), nil) + expMCs, err := getMachineConfigsForControllerConfig(templateDir, cc, nil, []byte(`{"dummy": "dummy"}`), nil, nil) if err != nil { t.Fatal(err) } @@ -320,7 +329,7 @@ func TestDoNothing(t *testing.T) { cc := newControllerConfig("test-cluster") ps := newPullSecret("coreos-pull-secret", []byte(`{"dummy": "dummy"}`)) - mcs, err := getMachineConfigsForControllerConfig(templateDir, cc, []byte(`{"dummy": "dummy"}`), nil) + mcs, err := getMachineConfigsForControllerConfig(templateDir, cc, nil, []byte(`{"dummy": "dummy"}`), nil, nil) if err != nil { t.Fatal(err) } @@ -358,7 +367,7 @@ func TestRecreateMachineConfig(t *testing.T) { cc := newControllerConfig("test-cluster") ps := newPullSecret("coreos-pull-secret", []byte(`{"dummy": "dummy"}`)) - mcs, err := getMachineConfigsForControllerConfig(templateDir, cc, []byte(`{"dummy": "dummy"}`), nil) + mcs, err := getMachineConfigsForControllerConfig(templateDir, cc, nil, []byte(`{"dummy": "dummy"}`), nil, nil) if err != nil { t.Fatal(err) } @@ -397,7 +406,7 @@ func TestUpdateMachineConfig(t *testing.T) { cc := newControllerConfig("test-cluster") ps := newPullSecret("coreos-pull-secret", []byte(`{"dummy": "dummy"}`)) - mcs, err := getMachineConfigsForControllerConfig(templateDir, cc, []byte(`{"dummy": "dummy"}`), nil) + mcs, err := getMachineConfigsForControllerConfig(templateDir, cc, nil, []byte(`{"dummy": "dummy"}`), nil, nil) if err != nil { t.Fatal(err) } @@ -417,7 +426,7 @@ func TestUpdateMachineConfig(t *testing.T) { f.objects = append(f.objects, mcs[idx]) } - expmcs, err := getMachineConfigsForControllerConfig(templateDir, cc, []byte(`{"dummy": "dummy"}`), nil) + expmcs, err := getMachineConfigsForControllerConfig(templateDir, cc, nil, []byte(`{"dummy": "dummy"}`), nil, nil) if err != nil { t.Fatal(err) } @@ -454,7 +463,7 @@ func TestKubeletAutoNodeSizingEnabled(t *testing.T) { cc := newControllerConfig("test-cluster") ps := []byte(`{"dummy": "dummy"}`) - mcs, err := getMachineConfigsForControllerConfig(templateDir, cc, ps, nil) + mcs, err := getMachineConfigsForControllerConfig(templateDir, cc, nil, ps, nil, nil) if err != nil { t.Fatal(err) } @@ -513,7 +522,7 @@ func TestKubeletAutoNodeSizingDisabledForHypershift(t *testing.T) { cc.Spec.Infra.Status.ControlPlaneTopology = configv1.ExternalTopologyMode ps := []byte(`{"dummy": "dummy"}`) - mcs, err := getMachineConfigsForControllerConfig(templateDir, cc, ps, nil) + mcs, err := getMachineConfigsForControllerConfig(templateDir, cc, nil, ps, nil, nil) if err != nil { t.Fatal(err) } diff --git a/test/e2e-bootstrap/bootstrap_test.go b/test/e2e-bootstrap/bootstrap_test.go index 8412dd05e9..b51709f03d 100644 --- a/test/e2e-bootstrap/bootstrap_test.go +++ b/test/e2e-bootstrap/bootstrap_test.go @@ -559,10 +559,13 @@ func createControllers(ctx *ctrlcommon.ControllerContext) []ctrlcommon.Controlle templatesDir, ctx.InformerFactory.Machineconfiguration().V1().ControllerConfigs(), ctx.InformerFactory.Machineconfiguration().V1().MachineConfigs(), + ctx.InformerFactory.Machineconfiguration().V1().MachineConfigPools(), + ctx.InformerFactory.Machineconfiguration().V1alpha1().OSImageStreams(), ctx.OpenShiftConfigKubeNamespacedInformerFactory.Core().V1().Secrets(), ctx.ConfigInformerFactory.Config().V1().APIServers(), ctx.ClientBuilder.KubeClientOrDie("template-controller"), ctx.ClientBuilder.MachineConfigClientOrDie("template-controller"), + ctx.FeatureGatesHandler, ), // Add all "sub-renderers here" kubeletconfig.New( @@ -570,6 +573,7 @@ func createControllers(ctx *ctrlcommon.ControllerContext) []ctrlcommon.Controlle ctx.InformerFactory.Machineconfiguration().V1().MachineConfigPools(), ctx.InformerFactory.Machineconfiguration().V1().ControllerConfigs(), ctx.InformerFactory.Machineconfiguration().V1().KubeletConfigs(), + ctx.InformerFactory.Machineconfiguration().V1alpha1().OSImageStreams(), ctx.ConfigInformerFactory.Config().V1().FeatureGates(), ctx.ConfigInformerFactory.Config().V1().Nodes(), ctx.ConfigInformerFactory.Config().V1().APIServers(), @@ -583,6 +587,7 @@ func createControllers(ctx *ctrlcommon.ControllerContext) []ctrlcommon.Controlle ctx.InformerFactory.Machineconfiguration().V1().MachineConfigPools(), ctx.InformerFactory.Machineconfiguration().V1().ControllerConfigs(), ctx.InformerFactory.Machineconfiguration().V1().ContainerRuntimeConfigs(), + ctx.InformerFactory.Machineconfiguration().V1alpha1().OSImageStreams(), ctx.ConfigInformerFactory.Config().V1().Images(), ctx.ConfigInformerFactory.Config().V1().ImageDigestMirrorSets(), ctx.ConfigInformerFactory.Config().V1().ImageTagMirrorSets(), @@ -603,6 +608,7 @@ func createControllers(ctx *ctrlcommon.ControllerContext) []ctrlcommon.Controlle ctx.InformerFactory.Machineconfiguration().V1().ContainerRuntimeConfigs(), ctx.InformerFactory.Machineconfiguration().V1().KubeletConfigs(), ctx.OperatorInformerFactory.Operator().V1().MachineConfigurations(), + ctx.InformerFactory.Machineconfiguration().V1alpha1().OSImageStreams(), ctx.ClientBuilder.KubeClientOrDie("render-controller"), ctx.ClientBuilder.MachineConfigClientOrDie("render-controller"), ctx.FeatureGatesHandler,