Skip to content

MCO-1961: Introduce the OSImageStream controller #5454

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
pablintino wants to merge 1 commit into from
Closed

MCO-1961: Introduce the OSImageStream controller #5454

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 46 additions & 4 deletions cmd/machine-config-controller/start.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,13 @@ import (
kubeletconfig "github.com/openshift/machine-config-operator/pkg/controller/kubelet-config"
machinesetbootimage "github.com/openshift/machine-config-operator/pkg/controller/machine-set-boot-image"
"github.com/openshift/machine-config-operator/pkg/controller/node"
"github.com/openshift/machine-config-operator/pkg/controller/osimagestream"
"github.com/openshift/machine-config-operator/pkg/controller/pinnedimageset"
"github.com/openshift/machine-config-operator/pkg/controller/render"
"github.com/openshift/machine-config-operator/pkg/controller/template"
"github.com/openshift/machine-config-operator/pkg/version"
"github.com/spf13/cobra"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/tools/leaderelection"
"k8s.io/klog/v2"
Expand Down Expand Up @@ -73,6 +75,12 @@ func runStartCmd(_ *cobra.Command, _ []string) {

ctrlctx := ctrlcommon.CreateControllerContext(ctx, cb)

// Early start the config informer because feature gate depends on it
ctrlctx.ConfigInformerFactory.Start(ctrlctx.Stop)
if fgErr := ctrlctx.FeatureGatesHandler.Connect(ctx); fgErr != nil {
klog.Fatal(fmt.Errorf("failed to connect to feature gates %w", fgErr))
}

go ctrlcommon.StartMetricsListener(startOpts.promMetricsListenAddress, ctrlctx.Stop, ctrlcommon.RegisterMCCMetrics)

controllers := createControllers(ctrlctx)
Expand Down Expand Up @@ -111,10 +119,6 @@ func runStartCmd(_ *cobra.Command, _ []string) {

close(ctrlctx.InformersStarted)

if fgErr := ctrlctx.FeatureGatesHandler.Connect(ctx); fgErr != nil {
klog.Fatal(fmt.Errorf("failed to connect to feature gates %w", fgErr))
}

if ctrlctx.FeatureGatesHandler.Enabled(features.FeatureGatePinnedImages) && ctrlctx.FeatureGatesHandler.Enabled(features.FeatureGateMachineConfigNodes) {
pinnedImageSet := pinnedimageset.New(
ctrlctx.InformerFactory.Machineconfiguration().V1().PinnedImageSets(),
Expand Down Expand Up @@ -164,6 +168,38 @@ func runStartCmd(_ *cobra.Command, _ []string) {
ctrlctx.OperatorInformerFactory.Start(ctrlctx.Stop)
}

// Enable OSImageStreams if the FeatureGate is active and the deployment is not OKD
if osimagestream.IsFeatureEnabled(ctrlctx.FeatureGatesHandler) {
osImageStreamController := osimagestream.NewController(
ctrlctx.ClientBuilder.KubeClientOrDie("osimagestream-controller"),
ctrlctx.ClientBuilder.MachineConfigClientOrDie("osimagestream-controller"),
ctrlctx.InformerFactory.Machineconfiguration().V1().ControllerConfigs(),
ctrlctx.KubeNamespacedInformerFactory.Core().V1().ConfigMaps(),
ctrlctx.InformerFactory.Machineconfiguration().V1alpha1().OSImageStreams(),
ctrlctx.ConfigInformerFactory.Config().V1().ClusterVersions(),
nil,
)

go osImageStreamController.Run(ctrlctx.Stop)
// start the informers again to enable feature gated types.
// see comments in SharedInformerFactory interface.
ctrlctx.KubeNamespacedInformerFactory.Start(ctrlctx.Stop)
ctrlctx.InformerFactory.Start(ctrlctx.Stop)
ctrlctx.ConfigInformerFactory.Start(ctrlctx.Stop)

if err = osImageStreamController.WaitBoot(); err != nil {
if !errors.IsNotFound(err) {
klog.Fatal(fmt.Errorf("failed to initialize the OSImageStream controller %w", err))
}

// Note: Till the feature is stable and the CoreOS images with the streams embedded have been
// running a for a few versions/cycles it's not a bad idea to ignore the situation where
// no streams are available and let the cluster start with zero available streams.
// TODO @pablintino: Track with a Jira card
klog.Warningf("Boot of the OSImageStreams Controller failed due to no available streams.")
}
}

for _, c := range controllers {
go c.Run(2, ctrlctx.Stop)
}
Expand Down Expand Up @@ -202,17 +238,21 @@ func createControllers(ctx *ctrlcommon.ControllerContext) []ctrlcommon.Controlle
rootOpts.templates,
ctx.InformerFactory.Machineconfiguration().V1().ControllerConfigs(),
ctx.InformerFactory.Machineconfiguration().V1().MachineConfigs(),
ctx.InformerFactory.Machineconfiguration().V1().MachineConfigPools(),
ctx.InformerFactory.Machineconfiguration().V1alpha1().OSImageStreams(),
ctx.OpenShiftConfigKubeNamespacedInformerFactory.Core().V1().Secrets(),
ctx.ConfigInformerFactory.Config().V1().APIServers(),
ctx.ClientBuilder.KubeClientOrDie("template-controller"),
ctx.ClientBuilder.MachineConfigClientOrDie("template-controller"),
ctx.FeatureGatesHandler,
),
// Add all "sub-renderers here"
kubeletconfig.New(
rootOpts.templates,
ctx.InformerFactory.Machineconfiguration().V1().MachineConfigPools(),
ctx.InformerFactory.Machineconfiguration().V1().ControllerConfigs(),
ctx.InformerFactory.Machineconfiguration().V1().KubeletConfigs(),
ctx.InformerFactory.Machineconfiguration().V1alpha1().OSImageStreams(),
ctx.ConfigInformerFactory.Config().V1().FeatureGates(),
ctx.ConfigInformerFactory.Config().V1().Nodes(),
ctx.ConfigInformerFactory.Config().V1().APIServers(),
Expand All @@ -226,6 +266,7 @@ func createControllers(ctx *ctrlcommon.ControllerContext) []ctrlcommon.Controlle
ctx.InformerFactory.Machineconfiguration().V1().MachineConfigPools(),
ctx.InformerFactory.Machineconfiguration().V1().ControllerConfigs(),
ctx.InformerFactory.Machineconfiguration().V1().ContainerRuntimeConfigs(),
ctx.InformerFactory.Machineconfiguration().V1alpha1().OSImageStreams(),
ctx.ConfigInformerFactory.Config().V1().Images(),
ctx.ConfigInformerFactory.Config().V1().ImageDigestMirrorSets(),
ctx.ConfigInformerFactory.Config().V1().ImageTagMirrorSets(),
Expand All @@ -246,6 +287,7 @@ func createControllers(ctx *ctrlcommon.ControllerContext) []ctrlcommon.Controlle
ctx.InformerFactory.Machineconfiguration().V1().ContainerRuntimeConfigs(),
ctx.InformerFactory.Machineconfiguration().V1().KubeletConfigs(),
ctx.OperatorInformerFactory.Operator().V1().MachineConfigurations(),
ctx.InformerFactory.Machineconfiguration().V1alpha1().OSImageStreams(),
ctx.ClientBuilder.KubeClientOrDie("render-controller"),
ctx.ClientBuilder.MachineConfigClientOrDie("render-controller"),
ctx.FeatureGatesHandler,
Expand Down
77 changes: 66 additions & 11 deletions pkg/controller/bootstrap/bootstrap.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,13 @@ package bootstrap

import (
"bytes"
"context"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"time"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

Expand All @@ -23,6 +25,7 @@ import (
apicfgv1 "github.com/openshift/api/config/v1"
apicfgv1alpha1 "github.com/openshift/api/config/v1alpha1"
"github.com/openshift/api/features"
imagev1 "github.com/openshift/api/image/v1"
mcfgv1 "github.com/openshift/api/machineconfiguration/v1"
mcfgv1alpha1 "github.com/openshift/api/machineconfiguration/v1alpha1"
apioperatorsv1alpha1 "github.com/openshift/api/operator/v1alpha1"
Expand All @@ -32,6 +35,7 @@ import (
containerruntimeconfig "github.com/openshift/machine-config-operator/pkg/controller/container-runtime-config"
"github.com/openshift/machine-config-operator/pkg/controller/internalreleaseimage"
kubeletconfig "github.com/openshift/machine-config-operator/pkg/controller/kubelet-config"
"github.com/openshift/machine-config-operator/pkg/controller/osimagestream"
"github.com/openshift/machine-config-operator/pkg/controller/render"
"github.com/openshift/machine-config-operator/pkg/controller/template"
"github.com/openshift/machine-config-operator/pkg/version"
Expand Down Expand Up @@ -70,7 +74,7 @@ func (b *Bootstrap) Run(destDir string) error {
return err
}

psraw, err := getPullSecretFromSecret(psfraw)
pullSecret, err := getValidatePullSecretFromBytes(psfraw)
if err != nil {
return err
}
Expand All @@ -82,8 +86,13 @@ func (b *Bootstrap) Run(destDir string) error {
apicfgv1.Install(scheme)
apicfgv1alpha1.Install(scheme)
corev1.AddToScheme(scheme)
imagev1.AddToScheme(scheme)
codecFactory := serializer.NewCodecFactory(scheme)
decoder := codecFactory.UniversalDecoder(mcfgv1.GroupVersion, apioperatorsv1alpha1.GroupVersion, apicfgv1.GroupVersion, apicfgv1alpha1.GroupVersion, corev1.SchemeGroupVersion, mcfgv1alpha1.GroupVersion)
decoder := codecFactory.UniversalDecoder(
mcfgv1.GroupVersion, apioperatorsv1alpha1.GroupVersion,
apicfgv1.GroupVersion, apicfgv1alpha1.GroupVersion,
corev1.SchemeGroupVersion, mcfgv1alpha1.GroupVersion,
imagev1.SchemeGroupVersion)

var (
cconfig *mcfgv1.ControllerConfig
Expand All @@ -101,6 +110,7 @@ func (b *Bootstrap) Run(destDir string) error {
imagePolicies []*apicfgv1.ImagePolicy
imgCfg *apicfgv1.Image
apiServer *apicfgv1.APIServer
imageStream *imagev1.ImageStream
iri *mcfgv1alpha1.InternalReleaseImage
)
for _, info := range infos {
Expand Down Expand Up @@ -171,6 +181,17 @@ func (b *Bootstrap) Run(destDir string) error {
if obj.GetName() == ctrlcommon.InternalReleaseImageInstanceName {
iri = obj
}
case *imagev1.ImageStream:
for _, tag := range obj.Spec.Tags {
if tag.Name == "machine-config-operator" {
if imageStream != nil {
klog.Infof("multiple ImageStream found. Previous ImageStream %s replaced by %s", imageStream.Name, obj.Name)
}
imageStream = obj

}
}
// It's an ImageStream that doesn't look like the Release one (doesn't have our tag)
default:
klog.Infof("skipping %q [%d] manifest because of unhandled %T", file.Name(), idx+1, obji)
}
Expand All @@ -191,15 +212,49 @@ func (b *Bootstrap) Run(destDir string) error {
return fmt.Errorf("error creating feature gates handler: %w", err)
}

iconfigs, err := template.RunBootstrap(b.templatesDir, cconfig, psraw, apiServer)
var osImageStream *mcfgv1alpha1.OSImageStream
// Enable OSImageStreams if the FeatureGate is active and the deployment is not OKD
if osimagestream.IsFeatureEnabled(fgHandler) {
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
defer cancel()

// TODO @pablintino we need to change the factory API to avoid passing that cmLister at bootstrap
osImageStream, err = osimagestream.BuildOsImageStreamBootstrap(ctx,
pullSecret,
cconfig,
imageStream,
&osimagestream.OSImageTuple{
OSImage: cconfig.Spec.BaseOSContainerImage,
OSExtensionsImage: cconfig.Spec.BaseOSExtensionsContainerImage,
},
osimagestream.NewDefaultStreamSourceFactory(nil, &osimagestream.DefaultImagesInspectorFactory{}),
)
if err != nil {
return fmt.Errorf("error inspecting available OSImageStreams: %w", err)
}

// If no error happened override the ControllerConfig URLs with the default stream ones
if err == nil {
defaultStreamSet, err := osimagestream.GetOSImageStreamSetByName(osImageStream, "")
if err != nil {
// Should never happen
return fmt.Errorf("error getting default OSImageStreamSet: %w", err)
}
cconfig.Spec.BaseOSContainerImage = string(defaultStreamSet.OSImage)
cconfig.Spec.BaseOSExtensionsContainerImage = string(defaultStreamSet.OSExtensionsImage)
}
}

pullSecretBytes := pullSecret.Data[corev1.DockerConfigJsonKey]
iconfigs, err := template.RunBootstrap(b.templatesDir, cconfig, pools, pullSecretBytes, apiServer, osImageStream)
if err != nil {
return err
}
klog.Infof("Successfully generated MachineConfigs from templates.")

configs = append(configs, iconfigs...)

rconfigs, err := containerruntimeconfig.RunImageBootstrap(b.templatesDir, cconfig, pools, icspRules, idmsRules, itmsRules, imgCfg, clusterImagePolicies, imagePolicies, fgHandler)
rconfigs, err := containerruntimeconfig.RunImageBootstrap(b.templatesDir, cconfig, pools, icspRules, idmsRules, itmsRules, imgCfg, clusterImagePolicies, imagePolicies, fgHandler, osImageStream)
if err != nil {
return err
}
Expand All @@ -208,7 +263,7 @@ func (b *Bootstrap) Run(destDir string) error {
configs = append(configs, rconfigs...)

if len(crconfigs) > 0 {
containerRuntimeConfigs, err := containerruntimeconfig.RunContainerRuntimeBootstrap(b.templatesDir, crconfigs, cconfig, pools)
containerRuntimeConfigs, err := containerruntimeconfig.RunContainerRuntimeBootstrap(b.templatesDir, crconfigs, cconfig, pools, osImageStream)
if err != nil {
return err
}
Expand All @@ -217,7 +272,7 @@ func (b *Bootstrap) Run(destDir string) error {
klog.Infof("Successfully generated MachineConfigs from containerruntime.")

if featureGate != nil {
featureConfigs, err := kubeletconfig.RunFeatureGateBootstrap(b.templatesDir, fgHandler, nodeConfig, cconfig, pools, apiServer)
featureConfigs, err := kubeletconfig.RunFeatureGateBootstrap(b.templatesDir, fgHandler, nodeConfig, cconfig, pools, apiServer, osImageStream)
if err != nil {
return err
}
Expand All @@ -234,7 +289,7 @@ func (b *Bootstrap) Run(destDir string) error {
}
}
if nodeConfig != nil {
nodeConfigs, err := kubeletconfig.RunNodeConfigBootstrap(b.templatesDir, fgHandler, cconfig, nodeConfig, pools, apiServer)
nodeConfigs, err := kubeletconfig.RunNodeConfigBootstrap(b.templatesDir, fgHandler, cconfig, nodeConfig, pools, apiServer, osImageStream)
if err != nil {
return err
}
Expand All @@ -243,7 +298,7 @@ func (b *Bootstrap) Run(destDir string) error {
klog.Infof("Successfully generated MachineConfigs from node.Configs.")

if len(kconfigs) > 0 {
kconfigs, err := kubeletconfig.RunKubeletBootstrap(b.templatesDir, kconfigs, cconfig, fgHandler, nodeConfig, pools, apiServer)
kconfigs, err := kubeletconfig.RunKubeletBootstrap(b.templatesDir, kconfigs, cconfig, fgHandler, nodeConfig, pools, apiServer, osImageStream)
if err != nil {
return err
}
Expand Down Expand Up @@ -274,7 +329,7 @@ func (b *Bootstrap) Run(destDir string) error {
klog.Infof("Successfully created %d pre-built image component MachineConfigs for hybrid OCL.", len(preBuiltImageMCs))
}

fpools, gconfigs, err := render.RunBootstrap(pools, configs, cconfig)
fpools, gconfigs, err := render.RunBootstrap(pools, configs, cconfig, osImageStream)
if err != nil {
return err
}
Expand Down Expand Up @@ -357,7 +412,7 @@ func (b *Bootstrap) Run(destDir string) error {

}

func getPullSecretFromSecret(sData []byte) ([]byte, error) {
func getValidatePullSecretFromBytes(sData []byte) (*corev1.Secret, error) {
obji, err := runtime.Decode(kscheme.Codecs.UniversalDecoder(corev1.SchemeGroupVersion), sData)
if err != nil {
return nil, err
Expand All @@ -369,7 +424,7 @@ func getPullSecretFromSecret(sData []byte) ([]byte, error) {
if s.Type != corev1.SecretTypeDockerConfigJson {
return nil, fmt.Errorf("expected secret type %s found %s", corev1.SecretTypeDockerConfigJson, s.Type)
}
return s.Data[corev1.DockerConfigJsonKey], nil
return s, nil
}

type manifest struct {
Expand Down
24 changes: 18 additions & 6 deletions pkg/controller/common/helpers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
ign2types "github.com/coreos/ignition/config/v2_2/types"
validate2 "github.com/coreos/ignition/config/validate"
ign3error "github.com/coreos/ignition/v2/config/shared/errors"
"github.com/openshift/api/machineconfiguration/v1alpha1"

ign3 "github.com/coreos/ignition/v2/config/v3_5"
ign3types "github.com/coreos/ignition/v2/config/v3_5/types"
Expand Down Expand Up @@ -68,7 +69,7 @@ func boolToPtr(b bool) *bool {
// It uses the Ignition config from first object as base and appends all the rest.
// Kernel arguments are concatenated.
// It defaults to the OSImageURL provided by the CVO but allows a MC provided OSImageURL to take precedence.
func MergeMachineConfigs(configs []*mcfgv1.MachineConfig, cconfig *mcfgv1.ControllerConfig) (*mcfgv1.MachineConfig, error) {
func MergeMachineConfigs(configs []*mcfgv1.MachineConfig, cconfig *mcfgv1.ControllerConfig, imageStream *v1alpha1.OSImageStreamSet) (*mcfgv1.MachineConfig, error) {
if len(configs) == 0 {
return nil, nil
}
Expand Down Expand Up @@ -187,15 +188,15 @@ func MergeMachineConfigs(configs []*mcfgv1.MachineConfig, cconfig *mcfgv1.Contro
// For layering, we want to let the user override OSImageURL again
// The template configs always match what's in controllerconfig because they get rendered from there,
// so the only way we get an override here is if the user adds something different
osImageURL := GetDefaultBaseImageContainer(&cconfig.Spec)
osImageURL := GetBaseImageContainer(&cconfig.Spec, imageStream)
for _, cfg := range configs {
if cfg.Spec.OSImageURL != "" {
osImageURL = cfg.Spec.OSImageURL
}
}

// Allow overriding the extensions container
baseOSExtensionsContainerImage := cconfig.Spec.BaseOSExtensionsContainerImage
baseOSExtensionsContainerImage := GetBaseExtensionsImageContainer(&cconfig.Spec, imageStream)
for _, cfg := range configs {
if cfg.Spec.BaseOSExtensionsContainerImage != "" {
baseOSExtensionsContainerImage = cfg.Spec.BaseOSExtensionsContainerImage
Expand Down Expand Up @@ -1035,9 +1036,20 @@ func GetIgnitionFileDataByPath(config *ign3types.Config, path string) ([]byte, e
return nil, nil
}

// GetDefaultBaseImageContainer returns the default bootable host base image.
func GetDefaultBaseImageContainer(cconfigspec *mcfgv1.ControllerConfigSpec) string {
return cconfigspec.BaseOSContainerImage
// GetBaseImageContainer returns the default bootable host base image.
func GetBaseImageContainer(cconfigspec *mcfgv1.ControllerConfigSpec, imageStream *v1alpha1.OSImageStreamSet) string {
if imageStream == nil {
return cconfigspec.BaseOSContainerImage
}
return string(imageStream.OSImage)
}

// GetBaseExtensionsImageContainer returns the default bootable host base image.
func GetBaseExtensionsImageContainer(cconfigspec *mcfgv1.ControllerConfigSpec, imageStream *v1alpha1.OSImageStreamSet) string {
if imageStream == nil {
return cconfigspec.BaseOSExtensionsContainerImage
}
return string(imageStream.OSExtensionsImage)
}

// Configures common template FuncMaps used across all renderers.
Expand Down
Loading