OLS-1775: Postgres reconciliation on certificate rotation.

internal/controller/constants.go

@@ -169,6 +169,12 @@ const (
 	// PostgresSecretHashKey is the key of the hash value of OLS Postgres secret
 	// #nosec G101
 	PostgresSecretHashKey = "hash/postgres-secret"
+	// PostgresCAHashKey is the key of the hash value of the OLS Postgres CA certificate
+	PostgresCAHashKey = "hash/postgres-ca"
+	// PostgresServiceCACertKeyName is the data key name for the service CA certificate in the ConfigMap
+	PostgresServiceCACertKeyName = "service-ca.crt"
+	// PostgresTLSCertKeyName is the data key name for the TLS certificate in the Secret
+	PostgresTLSCertKeyName = "tls.crt"
 	// PostgresServiceName is the name of OLS application Postgres server service
 	PostgresServiceName = "lightspeed-postgres-server"
 	// PostgresSecretName is the name of OLS application Postgres secret
@@ -255,6 +261,7 @@ ssl_ca_file = '/etc/certs/cm-olspostgresca/service-ca.crt'
 	OperatorDeploymentName          = "lightspeed-operator-controller-manager"
 	OLSDefaultCacheType             = "postgres"
 	PostgresConfigHashStateCacheKey = "olspostgresconfig-hash"
+	PostgresCAHashStateCacheKey     = "olspostgresca-hash"
 	// #nosec G101
 	PostgresSecretHashStateCacheKey = "olspostgressecret-hash"
 	// OperatorNetworkPolicyName is the name of the network policy for the operator

internal/controller/ols_app_postgres_assets.go

@@ -256,16 +256,16 @@ func (r *OLSConfigReconciler) updatePostgresDeployment(ctx context.Context, exis
 	// Validate deployment annotations.
 	if existingDeployment.Annotations == nil ||
 		existingDeployment.Annotations[PostgresConfigHashKey] != r.stateCache[PostgresConfigHashStateCacheKey] ||
-		existingDeployment.Annotations[PostgresSecretHashKey] != r.stateCache[PostgresSecretHashStateCacheKey] {
-		updateDeploymentAnnotations(existingDeployment, map[string]string{
+		existingDeployment.Annotations[PostgresSecretHashKey] != r.stateCache[PostgresSecretHashStateCacheKey] ||
+		existingDeployment.Annotations[PostgresCAHashKey] != r.stateCache[PostgresCAHashStateCacheKey] {
+		annotations := map[string]string{
 			PostgresConfigHashKey: r.stateCache[PostgresConfigHashStateCacheKey],
 			PostgresSecretHashKey: r.stateCache[PostgresSecretHashStateCacheKey],
-		})
+			PostgresCAHashKey:     r.stateCache[PostgresCAHashStateCacheKey],
+		}
+		updateDeploymentAnnotations(existingDeployment, annotations)
 		// update the deployment template annotation triggers the rolling update
-		updateDeploymentTemplateAnnotations(existingDeployment, map[string]string{
-			PostgresConfigHashKey: r.stateCache[PostgresConfigHashStateCacheKey],
-			PostgresSecretHashKey: r.stateCache[PostgresSecretHashStateCacheKey],
-		})
+		updateDeploymentTemplateAnnotations(existingDeployment, annotations)
 
 		if _, err := setDeploymentContainerEnvs(existingDeployment, desiredDeployment.Spec.Template.Spec.Containers[0].Env, PostgresDeploymentName); err != nil {
 			return err

internal/controller/ols_app_postgres_reconciliator.go

@@ -30,6 +30,10 @@ func (r *OLSConfigReconciler) reconcilePostgresServer(ctx context.Context, olsco
 			Name: "reconcile Postgres Secret",
 			Task: r.reconcilePostgresSecret,
 		},
+		{
+			Name: "reconcile Postgres CA Secret",
+			Task: r.reconcilePostgresCA,
+		},
 		{
 			Name: "reconcile Postgres Service",
 			Task: r.reconcilePostgresService,
@@ -70,14 +74,13 @@ func (r *OLSConfigReconciler) reconcilePostgresDeployment(ctx context.Context, c
 	existingDeployment := &appsv1.Deployment{}
 	err = r.Get(ctx, client.ObjectKey{Name: PostgresDeploymentName, Namespace: r.Options.Namespace}, existingDeployment)
 	if err != nil && errors.IsNotFound(err) {
-		updateDeploymentAnnotations(desiredDeployment, map[string]string{
-			PostgresConfigHashKey: r.stateCache[PostgresConfigHashStateCacheKey],
-			PostgresSecretHashKey: r.stateCache[PostgresSecretHashStateCacheKey],
-		})
-		updateDeploymentTemplateAnnotations(desiredDeployment, map[string]string{
+		annotations := map[string]string{
 			PostgresConfigHashKey: r.stateCache[PostgresConfigHashStateCacheKey],
 			PostgresSecretHashKey: r.stateCache[PostgresSecretHashStateCacheKey],
-		})
+			PostgresCAHashKey:     r.stateCache[PostgresCAHashStateCacheKey],
+		}
+		updateDeploymentAnnotations(desiredDeployment, annotations)
+		updateDeploymentTemplateAnnotations(desiredDeployment, annotations)
 		r.logger.Info("creating a new OLS postgres deployment", "deployment", desiredDeployment.Name)
 		err = r.Create(ctx, desiredDeployment)
 		if err != nil {
@@ -273,3 +276,61 @@ func (r *OLSConfigReconciler) reconcilePostgresNetworkPolicy(ctx context.Context
 	r.logger.Info("OLS postgres network policy reconciled", "network policy", networkPolicy.Name)
 	return nil
 }
+
+func (r *OLSConfigReconciler) reconcilePostgresCA(ctx context.Context, cr *olsv1alpha1.OLSConfig) error {
+	certBytes := []byte{}
+
+	// Get service CA certificate from ConfigMap
+	tmpCM := &corev1.ConfigMap{}
+	err := r.Client.Get(ctx, client.ObjectKey{Name: OLSCAConfigMap, Namespace: r.Options.Namespace}, tmpCM)
+	if err != nil {
+		if !errors.IsNotFound(err) {
+			return fmt.Errorf("failed to get %s ConfigMap: %w", OLSCAConfigMap, err)
+		}
+		r.logger.Info("CA ConfigMap not found, skipping CA bundle", "configmap", OLSCAConfigMap)
+	} else {
+		if caCert, exists := tmpCM.Data[PostgresServiceCACertKeyName]; exists {
+			certBytes = append(certBytes, []byte(PostgresServiceCACertKeyName)...)
+			certBytes = append(certBytes, []byte(caCert)...)
+		}
+	}
+
+	// Get serving cert from Secret
+	tmpSec := &corev1.Secret{}
+	err = r.Client.Get(ctx, client.ObjectKey{Name: PostgresCertsSecretName, Namespace: r.Options.Namespace}, tmpSec)
+	if err != nil {
+		if !errors.IsNotFound(err) {
+			return fmt.Errorf("failed to get %s Secret: %w", PostgresCertsSecretName, err)
+		}
+		r.logger.Info("serving cert Secret not found, skipping server certificate", "secret", PostgresCertsSecretName)
+	} else {
+		if tlsCert, exists := tmpSec.Data[PostgresTLSCertKeyName]; exists {
+			certBytes = append(certBytes, []byte(PostgresTLSCertKeyName)...)
+			certBytes = append(certBytes, tlsCert...)
+		}
+	}
+
+	// Calculate hash based on available inputs
+	combinedHash := ""
+	if len(certBytes) > 0 {
+		var err error
+		if combinedHash, err = hashBytes(certBytes); err != nil {
+			return fmt.Errorf("failed to generate Postgres CA hash: %w", err)
+		}
+	}
+
+	// Store existing hash before updating
+	existingHash := r.stateCache[PostgresCAHashStateCacheKey]
+
+	// Always update state cache to ensure it's set, even if value hasn't changed
+	r.stateCache[PostgresCAHashStateCacheKey] = combinedHash
+
+	// Check if hash changed (including changes to/from empty string)
+	if combinedHash == existingHash {
+		return nil
+	}
+
+	r.logger.Info("Postgres CA hash updated, deployment will be updated via updatePostgresDeployment")
+
+	return nil
+}

internal/controller/ols_app_server_deployment.go

@@ -8,7 +8,7 @@ import (
 
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
@@ -461,22 +461,39 @@ func (r *OLSConfigReconciler) updateOLSDeployment(ctx context.Context, existingD
 		existingDeployment.Annotations[OLSConfigHashKey] != r.stateCache[OLSConfigHashStateCacheKey] ||
 		existingDeployment.Annotations[OLSAppTLSHashKey] != r.stateCache[OLSAppTLSHashStateCacheKey] ||
 		existingDeployment.Annotations[LLMProviderHashKey] != r.stateCache[LLMProviderHashStateCacheKey] ||
-		existingDeployment.Annotations[PostgresSecretHashKey] != r.stateCache[PostgresSecretHashStateCacheKey] {
-		updateDeploymentAnnotations(existingDeployment, map[string]string{
+		existingDeployment.Annotations[PostgresSecretHashKey] != r.stateCache[PostgresSecretHashStateCacheKey] ||
+		existingDeployment.Annotations[PostgresCAHashKey] != r.stateCache[PostgresCAHashStateCacheKey] {
+
+		// Check if PostgreSQL CA hash changed - if so, verify PostgreSQL deployment is ready before restarting app-server
+		postgresCAHashChanged := existingDeployment.Annotations[PostgresCAHashKey] != r.stateCache[PostgresCAHashStateCacheKey]
+		if postgresCAHashChanged {
+			postgresDeployment := &appsv1.Deployment{}
+			err := r.Get(ctx, client.ObjectKey{Name: PostgresDeploymentName, Namespace: r.Options.Namespace}, postgresDeployment)
+			if err == nil {
+				// PostgreSQL deployment exists, check if it's ready
+				_, checkErr := r.checkDeploymentStatus(postgresDeployment)
+				if checkErr != nil {
+					// PostgreSQL is not ready yet, skip app-server update to avoid readiness failures
+					r.logger.Info("PostgreSQL deployment is not ready yet, skipping app-server update to prevent readiness failures",
+						"postgres_ready_replicas", postgresDeployment.Status.ReadyReplicas,
+						"postgres_desired_replicas", *postgresDeployment.Spec.Replicas)
+					return nil
+				}
+				r.logger.Info("PostgreSQL deployment is ready, proceeding with app-server update")
+			}
+		}
+
+		annotations := map[string]string{
 			OLSConfigHashKey:      r.stateCache[OLSConfigHashStateCacheKey],
 			OLSAppTLSHashKey:      r.stateCache[OLSAppTLSHashStateCacheKey],
 			LLMProviderHashKey:    r.stateCache[LLMProviderHashStateCacheKey],
 			AdditionalCAHashKey:   r.stateCache[AdditionalCAHashStateCacheKey],
 			PostgresSecretHashKey: r.stateCache[PostgresSecretHashStateCacheKey],
-		})
+			PostgresCAHashKey:     r.stateCache[PostgresCAHashStateCacheKey],
+		}
+		updateDeploymentAnnotations(existingDeployment, annotations)
 		// update the deployment template annotation triggers the rolling update
-		updateDeploymentTemplateAnnotations(existingDeployment, map[string]string{
-			OLSConfigHashKey:      r.stateCache[OLSConfigHashStateCacheKey],
-			OLSAppTLSHashKey:      r.stateCache[OLSAppTLSHashStateCacheKey],
-			LLMProviderHashKey:    r.stateCache[LLMProviderHashStateCacheKey],
-			AdditionalCAHashKey:   r.stateCache[AdditionalCAHashStateCacheKey],
-			PostgresSecretHashKey: r.stateCache[PostgresSecretHashStateCacheKey],
-		})
+		updateDeploymentTemplateAnnotations(existingDeployment, annotations)
 		changed = true
 	}
 
@@ -557,17 +574,18 @@ func (r *OLSConfigReconciler) telemetryEnabled() (bool, error) {
 
 	pullSecret := &corev1.Secret{}
 	err := r.Get(context.Background(), client.ObjectKey{Namespace: pullSecretNamespace, Name: pullSecretName}, pullSecret)
-
 	if err != nil {
-		if apierrors.IsNotFound(err) {
+		if errors.IsNotFound(err) {
+			// Secret doesn't exist - telemetry is not enabled (normal in test environments)
 			return false, nil
 		}
 		return false, err
 	}
 
 	dockerconfigjson, ok := pullSecret.Data[".dockerconfigjson"]
 	if !ok {
-		return false, fmt.Errorf("pull secret does not contain .dockerconfigjson")
+		// Secret exists but doesn't have the expected key - telemetry not properly configured
+		return false, nil
 	}
 
 	dockerconfigjsonDecoded := map[string]interface{}{}

internal/controller/ols_app_server_reconciliator.go

@@ -140,8 +140,9 @@ func (r *OLSConfigReconciler) reconcileOLSConfigMap(ctx context.Context, cr *ols
 
 func (r *OLSConfigReconciler) reconcileOLSAdditionalCAConfigMap(ctx context.Context, cr *olsv1alpha1.OLSConfig) error {
 	if cr.Spec.OLSConfig.AdditionalCAConfigMapRef == nil {
-		// no additional CA certs, skip
-		r.logger.Info("Additional CA not configured, reconciliation skipped")
+		// no additional CA certs, set empty hash
+		r.logger.Info("Additional CA not configured, setting empty hash")
+		r.stateCache[AdditionalCAHashStateCacheKey] = ""
 		return nil
 	}
 
@@ -278,18 +279,15 @@ func (r *OLSConfigReconciler) reconcileDeployment(ctx context.Context, cr *olsv1
 	existingDeployment := &appsv1.Deployment{}
 	err = r.Get(ctx, client.ObjectKey{Name: OLSAppServerDeploymentName, Namespace: r.Options.Namespace}, existingDeployment)
 	if err != nil && errors.IsNotFound(err) {
-		updateDeploymentAnnotations(desiredDeployment, map[string]string{
+		annotations := map[string]string{
 			OLSConfigHashKey:      r.stateCache[OLSConfigHashStateCacheKey],
 			OLSAppTLSHashKey:      r.stateCache[OLSAppTLSHashStateCacheKey],
 			LLMProviderHashKey:    r.stateCache[LLMProviderHashStateCacheKey],
 			PostgresSecretHashKey: r.stateCache[PostgresSecretHashStateCacheKey],
-		})
-		updateDeploymentTemplateAnnotations(desiredDeployment, map[string]string{
-			OLSConfigHashKey:      r.stateCache[OLSConfigHashStateCacheKey],
-			OLSAppTLSHashKey:      r.stateCache[OLSAppTLSHashStateCacheKey],
-			LLMProviderHashKey:    r.stateCache[LLMProviderHashStateCacheKey],
-			PostgresSecretHashKey: r.stateCache[PostgresSecretHashStateCacheKey],
-		})
+			PostgresCAHashKey:     r.stateCache[PostgresCAHashStateCacheKey],
+		}
+		updateDeploymentAnnotations(desiredDeployment, annotations)
+		updateDeploymentTemplateAnnotations(desiredDeployment, annotations)
 		r.logger.Info("creating a new deployment", "deployment", desiredDeployment.Name)
 		err = r.Create(ctx, desiredDeployment)
 		if err != nil {

internal/controller/olsconfig_controller.go

@@ -351,9 +351,15 @@ func (r *OLSConfigReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		Owns(&corev1.PersistentVolumeClaim{}).
 		Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(secretWatcherFilter)).
 		Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(telemetryPullSecretWatcherFilter)).
+		Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, obj client.Object) []reconcile.Request {
+			return r.postgresCAWatcherFilter(ctx, obj)
+		})).
 		Watches(&corev1.ConfigMap{}, handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, obj client.Object) []reconcile.Request {
 			return r.configMapWatcherFilter(ctx, obj)
 		})).
+		Watches(&corev1.ConfigMap{}, handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, obj client.Object) []reconcile.Request {
+			return r.postgresCAWatcherFilter(ctx, obj)
+		})).
 		Owns(&consolev1.ConsolePlugin{}).
 		Owns(&monv1.ServiceMonitor{}).
 		Owns(&monv1.PrometheusRule{}).

internal/controller/resource_watchers.go

@@ -124,3 +124,29 @@ func (r *OLSConfigReconciler) restartAppServer(ctx context.Context, inCluster bo
 	}
 	return nil
 }
+
+// postgresCAWatcherFilter watches for changes to PostgreSQL CA certificate resources
+func (r *OLSConfigReconciler) postgresCAWatcherFilter(ctx context.Context, obj client.Object) []reconcile.Request {
+	// Only watch resources in the operator's namespace
+	if obj.GetNamespace() != r.Options.Namespace {
+		return nil
+	}
+
+	// Watch the openshift-service-ca.crt ConfigMap
+	if obj.GetName() == OLSCAConfigMap {
+		return []reconcile.Request{
+			{NamespacedName: types.NamespacedName{
+				Name: OLSConfigName,
+			}},
+		}
+	}
+	// Watch the PostgreSQL serving certificate Secret
+	if obj.GetName() == PostgresCertsSecretName {
+		return []reconcile.Request{
+			{NamespacedName: types.NamespacedName{
+				Name: OLSConfigName,
+			}},
+		}
+	}
+	return nil
+}