diff --git a/internal/controller/constants.go b/internal/controller/constants.go index e5f59059f..a4326ba0d 100644 --- a/internal/controller/constants.go +++ b/internal/controller/constants.go @@ -169,6 +169,12 @@ const ( // PostgresSecretHashKey is the key of the hash value of OLS Postgres secret // #nosec G101 PostgresSecretHashKey = "hash/postgres-secret" + // PostgresCAHashKey is the key of the hash value of the OLS Postgres CA certificate + PostgresCAHashKey = "hash/postgres-ca" + // PostgresServiceCACertKeyName is the data key name for the service CA certificate in the ConfigMap + PostgresServiceCACertKeyName = "service-ca.crt" + // PostgresTLSCertKeyName is the data key name for the TLS certificate in the Secret + PostgresTLSCertKeyName = "tls.crt" // PostgresServiceName is the name of OLS application Postgres server service PostgresServiceName = "lightspeed-postgres-server" // PostgresSecretName is the name of OLS application Postgres secret @@ -255,6 +261,7 @@ ssl_ca_file = '/etc/certs/cm-olspostgresca/service-ca.crt' OperatorDeploymentName = "lightspeed-operator-controller-manager" OLSDefaultCacheType = "postgres" PostgresConfigHashStateCacheKey = "olspostgresconfig-hash" + PostgresCAHashStateCacheKey = "olspostgresca-hash" // #nosec G101 PostgresSecretHashStateCacheKey = "olspostgressecret-hash" // OperatorNetworkPolicyName is the name of the network policy for the operator diff --git a/internal/controller/ols_app_postgres_assets.go b/internal/controller/ols_app_postgres_assets.go index 4e90abbe3..fed732fef 100644 --- a/internal/controller/ols_app_postgres_assets.go +++ b/internal/controller/ols_app_postgres_assets.go @@ -256,16 +256,16 @@ func (r *OLSConfigReconciler) updatePostgresDeployment(ctx context.Context, exis // Validate deployment annotations. if existingDeployment.Annotations == nil || existingDeployment.Annotations[PostgresConfigHashKey] != r.stateCache[PostgresConfigHashStateCacheKey] || - existingDeployment.Annotations[PostgresSecretHashKey] != r.stateCache[PostgresSecretHashStateCacheKey] { - updateDeploymentAnnotations(existingDeployment, map[string]string{ + existingDeployment.Annotations[PostgresSecretHashKey] != r.stateCache[PostgresSecretHashStateCacheKey] || + existingDeployment.Annotations[PostgresCAHashKey] != r.stateCache[PostgresCAHashStateCacheKey] { + annotations := map[string]string{ PostgresConfigHashKey: r.stateCache[PostgresConfigHashStateCacheKey], PostgresSecretHashKey: r.stateCache[PostgresSecretHashStateCacheKey], - }) + PostgresCAHashKey: r.stateCache[PostgresCAHashStateCacheKey], + } + updateDeploymentAnnotations(existingDeployment, annotations) // update the deployment template annotation triggers the rolling update - updateDeploymentTemplateAnnotations(existingDeployment, map[string]string{ - PostgresConfigHashKey: r.stateCache[PostgresConfigHashStateCacheKey], - PostgresSecretHashKey: r.stateCache[PostgresSecretHashStateCacheKey], - }) + updateDeploymentTemplateAnnotations(existingDeployment, annotations) if _, err := setDeploymentContainerEnvs(existingDeployment, desiredDeployment.Spec.Template.Spec.Containers[0].Env, PostgresDeploymentName); err != nil { return err diff --git a/internal/controller/ols_app_postgres_reconciliator.go b/internal/controller/ols_app_postgres_reconciliator.go index 21552dd38..5a0de0561 100644 --- a/internal/controller/ols_app_postgres_reconciliator.go +++ b/internal/controller/ols_app_postgres_reconciliator.go @@ -30,6 +30,10 @@ func (r *OLSConfigReconciler) reconcilePostgresServer(ctx context.Context, olsco Name: "reconcile Postgres Secret", Task: r.reconcilePostgresSecret, }, + { + Name: "reconcile Postgres CA Secret", + Task: r.reconcilePostgresCA, + }, { Name: "reconcile Postgres Service", Task: r.reconcilePostgresService, @@ -70,14 +74,13 @@ func (r *OLSConfigReconciler) reconcilePostgresDeployment(ctx context.Context, c existingDeployment := &appsv1.Deployment{} err = r.Get(ctx, client.ObjectKey{Name: PostgresDeploymentName, Namespace: r.Options.Namespace}, existingDeployment) if err != nil && errors.IsNotFound(err) { - updateDeploymentAnnotations(desiredDeployment, map[string]string{ - PostgresConfigHashKey: r.stateCache[PostgresConfigHashStateCacheKey], - PostgresSecretHashKey: r.stateCache[PostgresSecretHashStateCacheKey], - }) - updateDeploymentTemplateAnnotations(desiredDeployment, map[string]string{ + annotations := map[string]string{ PostgresConfigHashKey: r.stateCache[PostgresConfigHashStateCacheKey], PostgresSecretHashKey: r.stateCache[PostgresSecretHashStateCacheKey], - }) + PostgresCAHashKey: r.stateCache[PostgresCAHashStateCacheKey], + } + updateDeploymentAnnotations(desiredDeployment, annotations) + updateDeploymentTemplateAnnotations(desiredDeployment, annotations) r.logger.Info("creating a new OLS postgres deployment", "deployment", desiredDeployment.Name) err = r.Create(ctx, desiredDeployment) if err != nil { @@ -273,3 +276,61 @@ func (r *OLSConfigReconciler) reconcilePostgresNetworkPolicy(ctx context.Context r.logger.Info("OLS postgres network policy reconciled", "network policy", networkPolicy.Name) return nil } + +func (r *OLSConfigReconciler) reconcilePostgresCA(ctx context.Context, cr *olsv1alpha1.OLSConfig) error { + certBytes := []byte{} + + // Get service CA certificate from ConfigMap + tmpCM := &corev1.ConfigMap{} + err := r.Client.Get(ctx, client.ObjectKey{Name: OLSCAConfigMap, Namespace: r.Options.Namespace}, tmpCM) + if err != nil { + if !errors.IsNotFound(err) { + return fmt.Errorf("failed to get %s ConfigMap: %w", OLSCAConfigMap, err) + } + r.logger.Info("CA ConfigMap not found, skipping CA bundle", "configmap", OLSCAConfigMap) + } else { + if caCert, exists := tmpCM.Data[PostgresServiceCACertKeyName]; exists { + certBytes = append(certBytes, []byte(PostgresServiceCACertKeyName)...) + certBytes = append(certBytes, []byte(caCert)...) + } + } + + // Get serving cert from Secret + tmpSec := &corev1.Secret{} + err = r.Client.Get(ctx, client.ObjectKey{Name: PostgresCertsSecretName, Namespace: r.Options.Namespace}, tmpSec) + if err != nil { + if !errors.IsNotFound(err) { + return fmt.Errorf("failed to get %s Secret: %w", PostgresCertsSecretName, err) + } + r.logger.Info("serving cert Secret not found, skipping server certificate", "secret", PostgresCertsSecretName) + } else { + if tlsCert, exists := tmpSec.Data[PostgresTLSCertKeyName]; exists { + certBytes = append(certBytes, []byte(PostgresTLSCertKeyName)...) + certBytes = append(certBytes, tlsCert...) + } + } + + // Calculate hash based on available inputs + combinedHash := "" + if len(certBytes) > 0 { + var err error + if combinedHash, err = hashBytes(certBytes); err != nil { + return fmt.Errorf("failed to generate Postgres CA hash: %w", err) + } + } + + // Store existing hash before updating + existingHash := r.stateCache[PostgresCAHashStateCacheKey] + + // Always update state cache to ensure it's set, even if value hasn't changed + r.stateCache[PostgresCAHashStateCacheKey] = combinedHash + + // Check if hash changed (including changes to/from empty string) + if combinedHash == existingHash { + return nil + } + + r.logger.Info("Postgres CA hash updated, deployment will be updated via updatePostgresDeployment") + + return nil +} diff --git a/internal/controller/ols_app_server_deployment.go b/internal/controller/ols_app_server_deployment.go index 194fc2ea1..397677d17 100644 --- a/internal/controller/ols_app_server_deployment.go +++ b/internal/controller/ols_app_server_deployment.go @@ -8,7 +8,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -461,22 +461,39 @@ func (r *OLSConfigReconciler) updateOLSDeployment(ctx context.Context, existingD existingDeployment.Annotations[OLSConfigHashKey] != r.stateCache[OLSConfigHashStateCacheKey] || existingDeployment.Annotations[OLSAppTLSHashKey] != r.stateCache[OLSAppTLSHashStateCacheKey] || existingDeployment.Annotations[LLMProviderHashKey] != r.stateCache[LLMProviderHashStateCacheKey] || - existingDeployment.Annotations[PostgresSecretHashKey] != r.stateCache[PostgresSecretHashStateCacheKey] { - updateDeploymentAnnotations(existingDeployment, map[string]string{ + existingDeployment.Annotations[PostgresSecretHashKey] != r.stateCache[PostgresSecretHashStateCacheKey] || + existingDeployment.Annotations[PostgresCAHashKey] != r.stateCache[PostgresCAHashStateCacheKey] { + + // Check if PostgreSQL CA hash changed - if so, verify PostgreSQL deployment is ready before restarting app-server + postgresCAHashChanged := existingDeployment.Annotations[PostgresCAHashKey] != r.stateCache[PostgresCAHashStateCacheKey] + if postgresCAHashChanged { + postgresDeployment := &appsv1.Deployment{} + err := r.Get(ctx, client.ObjectKey{Name: PostgresDeploymentName, Namespace: r.Options.Namespace}, postgresDeployment) + if err == nil { + // PostgreSQL deployment exists, check if it's ready + _, checkErr := r.checkDeploymentStatus(postgresDeployment) + if checkErr != nil { + // PostgreSQL is not ready yet, skip app-server update to avoid readiness failures + r.logger.Info("PostgreSQL deployment is not ready yet, skipping app-server update to prevent readiness failures", + "postgres_ready_replicas", postgresDeployment.Status.ReadyReplicas, + "postgres_desired_replicas", *postgresDeployment.Spec.Replicas) + return nil + } + r.logger.Info("PostgreSQL deployment is ready, proceeding with app-server update") + } + } + + annotations := map[string]string{ OLSConfigHashKey: r.stateCache[OLSConfigHashStateCacheKey], OLSAppTLSHashKey: r.stateCache[OLSAppTLSHashStateCacheKey], LLMProviderHashKey: r.stateCache[LLMProviderHashStateCacheKey], AdditionalCAHashKey: r.stateCache[AdditionalCAHashStateCacheKey], PostgresSecretHashKey: r.stateCache[PostgresSecretHashStateCacheKey], - }) + PostgresCAHashKey: r.stateCache[PostgresCAHashStateCacheKey], + } + updateDeploymentAnnotations(existingDeployment, annotations) // update the deployment template annotation triggers the rolling update - updateDeploymentTemplateAnnotations(existingDeployment, map[string]string{ - OLSConfigHashKey: r.stateCache[OLSConfigHashStateCacheKey], - OLSAppTLSHashKey: r.stateCache[OLSAppTLSHashStateCacheKey], - LLMProviderHashKey: r.stateCache[LLMProviderHashStateCacheKey], - AdditionalCAHashKey: r.stateCache[AdditionalCAHashStateCacheKey], - PostgresSecretHashKey: r.stateCache[PostgresSecretHashStateCacheKey], - }) + updateDeploymentTemplateAnnotations(existingDeployment, annotations) changed = true } @@ -557,9 +574,9 @@ func (r *OLSConfigReconciler) telemetryEnabled() (bool, error) { pullSecret := &corev1.Secret{} err := r.Get(context.Background(), client.ObjectKey{Namespace: pullSecretNamespace, Name: pullSecretName}, pullSecret) - if err != nil { - if apierrors.IsNotFound(err) { + if errors.IsNotFound(err) { + // Secret doesn't exist - telemetry is not enabled (normal in test environments) return false, nil } return false, err @@ -567,7 +584,8 @@ func (r *OLSConfigReconciler) telemetryEnabled() (bool, error) { dockerconfigjson, ok := pullSecret.Data[".dockerconfigjson"] if !ok { - return false, fmt.Errorf("pull secret does not contain .dockerconfigjson") + // Secret exists but doesn't have the expected key - telemetry not properly configured + return false, nil } dockerconfigjsonDecoded := map[string]interface{}{} diff --git a/internal/controller/ols_app_server_reconciliator.go b/internal/controller/ols_app_server_reconciliator.go index 77e0795ff..738b453b0 100644 --- a/internal/controller/ols_app_server_reconciliator.go +++ b/internal/controller/ols_app_server_reconciliator.go @@ -140,8 +140,9 @@ func (r *OLSConfigReconciler) reconcileOLSConfigMap(ctx context.Context, cr *ols func (r *OLSConfigReconciler) reconcileOLSAdditionalCAConfigMap(ctx context.Context, cr *olsv1alpha1.OLSConfig) error { if cr.Spec.OLSConfig.AdditionalCAConfigMapRef == nil { - // no additional CA certs, skip - r.logger.Info("Additional CA not configured, reconciliation skipped") + // no additional CA certs, set empty hash + r.logger.Info("Additional CA not configured, setting empty hash") + r.stateCache[AdditionalCAHashStateCacheKey] = "" return nil } @@ -278,18 +279,15 @@ func (r *OLSConfigReconciler) reconcileDeployment(ctx context.Context, cr *olsv1 existingDeployment := &appsv1.Deployment{} err = r.Get(ctx, client.ObjectKey{Name: OLSAppServerDeploymentName, Namespace: r.Options.Namespace}, existingDeployment) if err != nil && errors.IsNotFound(err) { - updateDeploymentAnnotations(desiredDeployment, map[string]string{ + annotations := map[string]string{ OLSConfigHashKey: r.stateCache[OLSConfigHashStateCacheKey], OLSAppTLSHashKey: r.stateCache[OLSAppTLSHashStateCacheKey], LLMProviderHashKey: r.stateCache[LLMProviderHashStateCacheKey], PostgresSecretHashKey: r.stateCache[PostgresSecretHashStateCacheKey], - }) - updateDeploymentTemplateAnnotations(desiredDeployment, map[string]string{ - OLSConfigHashKey: r.stateCache[OLSConfigHashStateCacheKey], - OLSAppTLSHashKey: r.stateCache[OLSAppTLSHashStateCacheKey], - LLMProviderHashKey: r.stateCache[LLMProviderHashStateCacheKey], - PostgresSecretHashKey: r.stateCache[PostgresSecretHashStateCacheKey], - }) + PostgresCAHashKey: r.stateCache[PostgresCAHashStateCacheKey], + } + updateDeploymentAnnotations(desiredDeployment, annotations) + updateDeploymentTemplateAnnotations(desiredDeployment, annotations) r.logger.Info("creating a new deployment", "deployment", desiredDeployment.Name) err = r.Create(ctx, desiredDeployment) if err != nil { diff --git a/internal/controller/olsconfig_controller.go b/internal/controller/olsconfig_controller.go index 764ec4566..7fe62570c 100644 --- a/internal/controller/olsconfig_controller.go +++ b/internal/controller/olsconfig_controller.go @@ -351,9 +351,15 @@ func (r *OLSConfigReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&corev1.PersistentVolumeClaim{}). Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(secretWatcherFilter)). Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(telemetryPullSecretWatcherFilter)). + Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, obj client.Object) []reconcile.Request { + return r.postgresCAWatcherFilter(ctx, obj) + })). Watches(&corev1.ConfigMap{}, handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, obj client.Object) []reconcile.Request { return r.configMapWatcherFilter(ctx, obj) })). + Watches(&corev1.ConfigMap{}, handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, obj client.Object) []reconcile.Request { + return r.postgresCAWatcherFilter(ctx, obj) + })). Owns(&consolev1.ConsolePlugin{}). Owns(&monv1.ServiceMonitor{}). Owns(&monv1.PrometheusRule{}). diff --git a/internal/controller/resource_watchers.go b/internal/controller/resource_watchers.go index 493ae8ab5..4ae84635f 100644 --- a/internal/controller/resource_watchers.go +++ b/internal/controller/resource_watchers.go @@ -124,3 +124,29 @@ func (r *OLSConfigReconciler) restartAppServer(ctx context.Context, inCluster bo } return nil } + +// postgresCAWatcherFilter watches for changes to PostgreSQL CA certificate resources +func (r *OLSConfigReconciler) postgresCAWatcherFilter(ctx context.Context, obj client.Object) []reconcile.Request { + // Only watch resources in the operator's namespace + if obj.GetNamespace() != r.Options.Namespace { + return nil + } + + // Watch the openshift-service-ca.crt ConfigMap + if obj.GetName() == OLSCAConfigMap { + return []reconcile.Request{ + {NamespacedName: types.NamespacedName{ + Name: OLSConfigName, + }}, + } + } + // Watch the PostgreSQL serving certificate Secret + if obj.GetName() == PostgresCertsSecretName { + return []reconcile.Request{ + {NamespacedName: types.NamespacedName{ + Name: OLSConfigName, + }}, + } + } + return nil +}