Merge pull request #1754 from haircommander/min-kubelet-version

OCPNODE-2940: add support for minimumKubeletVersion

bindata/assets/config/defaultconfig.yaml

@@ -34,11 +34,6 @@ apiServerArguments:
     - "true"
   anonymous-auth:
     - "true"
-  authorization-mode:
-    - Scope
-    - SystemMasters
-    - RBAC
-    - Node
   audit-log-format:
     - json
   audit-log-maxbackup:

pkg/cmd/render/render.go

@@ -22,6 +22,7 @@ import (
 	"github.com/openshift/cluster-kube-apiserver-operator/bindata"
 	"github.com/openshift/cluster-kube-apiserver-operator/pkg/operator/configobservation/apienablement"
 	"github.com/openshift/cluster-kube-apiserver-operator/pkg/operator/configobservation/auth"
+	"github.com/openshift/cluster-kube-apiserver-operator/pkg/operator/configobservation/node"
 	libgoaudit "github.com/openshift/library-go/pkg/operator/apiserver/audit"
 	"github.com/openshift/library-go/pkg/operator/configobserver/featuregates"
 	genericrender "github.com/openshift/library-go/pkg/operator/render"
@@ -357,6 +358,10 @@ func bootstrapDefaultConfig(featureGates featuregates.FeatureGate) ([]byte, erro
 		}
 	}
 
+	if err := node.AddAuthorizationModes(defaultConfig, featureGates.Enabled(features.FeatureGateMinimumKubeletVersion)); err != nil {
+		return nil, err
+	}
+
 	defaultConfigRaw, err := json.Marshal(defaultConfig)
 	if err != nil {
 		return nil, fmt.Errorf("failed to marshal default config - %s", err)

pkg/cmd/render/render_test.go

@@ -264,7 +264,7 @@ func TestRenderCommand(t *testing.T) {
 				if !ok {
 					return fmt.Errorf("missing \"feature-gates\" entry in APIServerArguments")
 				}
-				expectedGates := []string{"Bar=false", "Foo=true", "OpenShiftPodSecurityAdmission=true"}
+				expectedGates := []string{"Bar=false", "Foo=true", "OpenShiftPodSecurityAdmission=true", "MinimumKubeletVersion=true"}
 				if len(actualGates) != len(expectedGates) {
 					return fmt.Errorf("expected to get exactly %d feature gates but found %d: expected=%v got=%v", len(expectedGates), len(actualGates), expectedGates, actualGates)
 				}
@@ -675,7 +675,7 @@ spec:
 }
 
 func TestGetDefaultConfigWithAuditPolicy(t *testing.T) {
-	raw, err := bootstrapDefaultConfig(featuregates.NewFeatureGate([]configv1.FeatureGateName{features.FeatureGateOpenShiftPodSecurityAdmission}, nil))
+	raw, err := bootstrapDefaultConfig(featuregates.NewFeatureGate([]configv1.FeatureGateName{features.FeatureGateOpenShiftPodSecurityAdmission, features.FeatureGateMinimumKubeletVersion}, nil))
 	require.NoError(t, err)
 	require.True(t, len(raw) > 0)
 

pkg/cmd/render/testdata/rendered/default-fg/featuregate.yaml

@@ -8,5 +8,6 @@ status:
     enabled:
     - name: Foo
     - name: OpenShiftPodSecurityAdmission
+    - name: MinimumKubeletVersion
     disabled:
     - name: Bar

pkg/operator/configobservation/configobservercontroller/observe_config_controller.go

@@ -157,6 +157,8 @@ func NewConfigObserver(operatorClient v1helpers.StaticPodOperatorClient, kubeInf
 					),
 				},
 			),
+			node.NewMinimumKubeletVersionObserver(featureGateAccessor),
+			node.NewAuthorizationModeObserver(featureGateAccessor),
 			proxy.NewProxyObserveFunc([]string{"targetconfigcontroller", "proxy"}),
 			images.ObserveInternalRegistryHostname,
 			images.ObserveExternalRegistryHostnames,

pkg/operator/configobservation/node/observe_authorization_mode.go

@@ -0,0 +1,81 @@
+package node
+
+import (
+	"github.com/openshift/api/features"
+	"github.com/openshift/library-go/pkg/operator/configobserver"
+	"github.com/openshift/library-go/pkg/operator/configobserver/featuregates"
+	"github.com/openshift/library-go/pkg/operator/events"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+)
+
+// There are scopes authorizer tests that fail if this order is changed.
+// So this should not be sorted
+var defaultAuthenticationModes = []string{
+	"Scope",
+	"SystemMasters",
+	"RBAC",
+	"Node",
+}
+
+var authenticationModesWithMinimumKubeletVersion = []string{
+	"Scope",
+	"SystemMasters",
+	"RBAC",
+	ModeMinimumKubeletVersion, // before "Node" to have a chance to deny a node
+	"Node",
+}
+var (
+	authModeFlag  = "authorization-mode"
+	apiServerArgs = "apiServerArguments"
+	authModePath  = []string{apiServerArgs, authModeFlag}
+)
+
+type authorizationModeObserver struct {
+	featureGateAccessor featuregates.FeatureGateAccess
+	authModes           []string
+}
+
+func NewAuthorizationModeObserver(featureGateAccessor featuregates.FeatureGateAccess) configobserver.ObserveConfigFunc {
+	return (&authorizationModeObserver{
+		featureGateAccessor: featureGateAccessor,
+	}).ObserveAuthorizationMode
+}
+
+// ObserveAuthorizationMode watches the featuregate configuration and generates the apiServerArguments.authorization-mode
+// It currently hardcodes the default set and adds MinimumKubeletVersion if the feature is set to on.
+func (o *authorizationModeObserver) ObserveAuthorizationMode(genericListers configobserver.Listers, _ events.Recorder, existingConfig map[string]interface{}) (ret map[string]interface{}, errs []error) {
+	defer func() {
+		// Prune the observed config so that it only contains minimumKubeletVersion field.
+		ret = configobserver.Pruned(ret, authModePath)
+	}()
+
+	if !o.featureGateAccessor.AreInitialFeatureGatesObserved() {
+		return existingConfig, nil
+	}
+
+	featureGates, err := o.featureGateAccessor.CurrentFeatureGates()
+	if err != nil {
+		return existingConfig, append(errs, err)
+	}
+
+	ret = map[string]interface{}{}
+	if err := AddAuthorizationModes(ret, featureGates.Enabled(features.FeatureGateMinimumKubeletVersion)); err != nil {
+		return existingConfig, append(errs, err)
+	}
+	return ret, nil
+}
+
+// AddAuthorizationModes modifies the passed in config
+// to add the "authorization-mode": "MinimumKubeletVersion" if the feature is on. If it's off, it
+// removes it instead.
+// This function assumes MinimumKubeletVersion auth mode isn't present by default,
+// and should likely be removed when it is.
+func AddAuthorizationModes(observedConfig map[string]interface{}, isMinimumKubeletVersionEnabled bool) error {
+	modes := defaultAuthenticationModes
+	if isMinimumKubeletVersionEnabled {
+		modes = authenticationModesWithMinimumKubeletVersion
+	}
+
+	unstructured.RemoveNestedField(observedConfig, authModePath...)
+	return unstructured.SetNestedStringSlice(observedConfig, modes, authModePath...)
+}

pkg/operator/configobservation/node/observe_authorization_mode_test.go

@@ -0,0 +1,85 @@
+package node
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+)
+
+func TestAddAuthorizationModes(t *testing.T) {
+	for _, on := range []bool{false, true} {
+		expectedSet := []any{"Scope", "SystemMasters", "RBAC", "Node"}
+		if on {
+			expectedSet = []any{"Scope", "SystemMasters", "RBAC", ModeMinimumKubeletVersion, "Node"}
+		}
+		for _, tc := range []struct {
+			name           string
+			existingConfig map[string]interface{}
+			expectedConfig map[string]interface{}
+		}{
+			{
+				name: "should not fail if apiServerArguments not present",
+				existingConfig: map[string]interface{}{
+					"fakeconfig": "fake",
+				},
+				expectedConfig: map[string]interface{}{
+					"fakeconfig":         "fake",
+					"apiServerArguments": map[string]any{"authorization-mode": expectedSet},
+				},
+			},
+			{
+				name: "should not fail if authorization-mode not present",
+				existingConfig: map[string]interface{}{
+					"apiServerArguments": map[string]any{"fake": []any{"fake"}},
+				},
+				expectedConfig: map[string]interface{}{
+					"apiServerArguments": map[string]any{"fake": []any{"fake"}, "authorization-mode": expectedSet},
+				},
+			},
+			{
+				name: "should clobber value if not expected",
+				existingConfig: map[string]interface{}{
+					"apiServerArguments": map[string]any{"authorization-mode": []any{"fake"}},
+				},
+				expectedConfig: map[string]interface{}{
+					"apiServerArguments": map[string]any{"authorization-mode": expectedSet},
+				},
+			},
+			{
+				name: "should not fail if MinimumKubeletVersion already present",
+				existingConfig: map[string]interface{}{
+					"apiServerArguments": map[string]any{"authorization-mode": []any{"MinimumKubeletVersion"}},
+				},
+				expectedConfig: map[string]interface{}{
+					"apiServerArguments": map[string]any{"authorization-mode": expectedSet},
+				},
+			},
+			{
+				name: "should not fail if apiServerArguments not present",
+				existingConfig: map[string]interface{}{
+					"fakeconfig": "fake",
+				},
+				expectedConfig: map[string]interface{}{
+					"fakeconfig":         "fake",
+					"apiServerArguments": map[string]any{"authorization-mode": expectedSet},
+				},
+			},
+		} {
+			name := tc.name + " when feature is "
+			if on {
+				name += "on"
+			} else {
+				name += "off"
+			}
+			t.Run(name, func(t *testing.T) {
+				if err := AddAuthorizationModes(tc.existingConfig, on); err != nil {
+					t.Fatal(err)
+				}
+
+				if diff := cmp.Diff(tc.expectedConfig, tc.existingConfig); diff != "" {
+					t.Errorf("unexpected config:\n%s", diff)
+				}
+			})
+		}
+	}
+}

pkg/operator/configobservation/node/observe_minimum_kubelet_version.go

@@ -0,0 +1,77 @@
+package node
+
+import (
+	configv1 "github.com/openshift/api/config/v1"
+	"github.com/openshift/api/features"
+	"github.com/openshift/cluster-kube-apiserver-operator/pkg/operator/configobservation"
+	"github.com/openshift/library-go/pkg/operator/configobserver"
+	"github.com/openshift/library-go/pkg/operator/configobserver/featuregates"
+	"github.com/openshift/library-go/pkg/operator/events"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/klog/v2"
+)
+
+var (
+	ModeMinimumKubeletVersion       = "MinimumKubeletVersion"
+	minimumKubeletVersionConfigPath = "minimumKubeletVersion"
+)
+
+type minimumKubeletVersionObserver struct {
+	featureGateAccessor featuregates.FeatureGateAccess
+}
+
+func NewMinimumKubeletVersionObserver(featureGateAccessor featuregates.FeatureGateAccess) configobserver.ObserveConfigFunc {
+	return (&minimumKubeletVersionObserver{
+		featureGateAccessor: featureGateAccessor,
+	}).ObserveMinimumKubeletVersion
+}
+
+// ObserveKubeletMinimumVersion watches the node configuration and generates the minimumKubeletVersion
+func (o *minimumKubeletVersionObserver) ObserveMinimumKubeletVersion(genericListers configobserver.Listers, _ events.Recorder, existingConfig map[string]interface{}) (ret map[string]interface{}, errs []error) {
+	defer func() {
+		// Prune the observed config so that it only contains minimumKubeletVersion field.
+		ret = configobserver.Pruned(ret, []string{minimumKubeletVersionConfigPath})
+	}()
+
+	if !o.featureGateAccessor.AreInitialFeatureGatesObserved() {
+		return existingConfig, nil
+	}
+
+	featureGates, err := o.featureGateAccessor.CurrentFeatureGates()
+	if err != nil {
+		return existingConfig, append(errs, err)
+	}
+
+	if !featureGates.Enabled(features.FeatureGateMinimumKubeletVersion) {
+		return existingConfig, nil
+	}
+
+	nodeLister := genericListers.(configobservation.Listers)
+	configNode, err := nodeLister.NodeLister().Get("cluster")
+	// we got an error so without the node object we are not able to determine minimumKubeletVersion
+	if err != nil {
+		// if config/v1/node/cluster object is not found, that can be treated as a non-error case, but raise a warning
+		if apierrors.IsNotFound(err) {
+			klog.Warningf("ObserveMinimumKubeletVersion: nodes.%s/cluster not found", configv1.GroupName)
+		} else {
+			errs = append(errs, err)
+		}
+		return existingConfig, errs
+	}
+
+	ret = map[string]interface{}{}
+	if configNode.Spec.MinimumKubeletVersion == "" {
+		// in case minimum kubelet version is not set on cluster
+		// return empty set of configs, this helps to unset the config
+		// values related to the minimumKubeletVersion.
+		// Also, ensures that this observer doesn't break cluster upgrades/downgrades
+		return ret, errs
+	}
+
+	if err := unstructured.SetNestedField(ret, configNode.Spec.MinimumKubeletVersion, minimumKubeletVersionConfigPath); err != nil {
+		return existingConfig, append(errs, err)
+	}
+
+	return ret, errs
+}