Add Spring Boot sample application for testing failover scenarios

README.md

@@ -9,6 +9,7 @@ Postgres-Operator provides PostgreSQL as a service on Kubernetes and OpenShift.
 * * `./charts/patroni-services` - directory with HELM chart for Postgres Services.
 * `./pkg` - directory with operator source code, which is used for running Postgres Operator.
 * `./tests` - directory with robot test source code, `Dockerfile`.
+* * `./tests/examples` - example projects demonstrating various use cases.
 
 ## How to start
 
@@ -25,6 +26,11 @@ There are no well-defined rules for troubleshooting, as each task is unique, but
 * Deploy parameters.
 * Logs from all Postgres Service pods: operator, postgres db and others.
 
+## Examples
+
+* **[Spring Boot Failover Testing](tests/examples/spring-boot-failover-test/)** - Test PostgreSQL failover behavior with Spring Boot applications
+* [More Examples](tests/examples/) - Additional example projects
+
 ## Useful links
 
 * [Installation Guide](/docs/public/installation.md)

docs/public/installation.md

@@ -813,7 +813,7 @@ For more information on how to do the Major Upgrade of PostgreSQL, please, follo
 ```yaml
     pgbouncer:
       listen_port: '6432'
-      listen_addr: '0.0.0.0'
+      listen_addr: '*'
       auth_type: 'md5'
       auth_file: '/etc/pgbouncer/userlist.txt'
       auth_user: 'pgbouncer'

helmfile.yaml.gotmpl

@@ -0,0 +1,173 @@
+# pgskipper-operator Helmfile Configuration
+#
+# This helmfile manages the deployment of pgskipper-operator components:
+# - Patroni-Core Operator (PostgreSQL core functionality)
+# - Patroni-Services Operator (PostgreSQL services)
+#
+# Usage:
+#   helmfile sync                      - Deploy with official images
+#   helmfile -e orbstack sync          - Deploy to OrbStack
+#   helmfile -e rancher sync           - Deploy to Rancher Desktop
+#   helmfile -e k3d-v4only sync        - Deploy to k3d with IPv4 only
+#   helmfile destroy                   - Uninstall releases (see cleanup note below)
+#
+# Environment Variables (optional):
+#   USE_LOCAL_IMAGES=true              - Build and use local operator images
+#   PGSKIPPER_IMAGE=<image>            - Override operator image (default: ghcr.io/netcracker/pgskipper-operator)
+#   PGSKIPPER_TAG=<tag>                - Override operator tag (default: main)
+#   NAMESPACE=<namespace>              - Target namespace (default: postgres)
+#
+# Examples:
+#   USE_LOCAL_IMAGES=true helmfile -e orbstack sync  - Build local images for OrbStack
+#   PGSKIPPER_TAG=v1.2.3 helmfile sync               - Use specific version from ghcr.io
+#
+# Cleanup after destroy:
+#   After running `helmfile destroy`, some operator-created resources may remain.
+#   To fully clean up, run:
+#     kubectl delete pvc,service,configmap --all -n postgres --ignore-not-found
+#   Or delete the entire namespace:
+#     kubectl delete namespace postgres
+
+helmDefaults:
+  wait: false
+  timeout: 600
+  createNamespace: true
+  cleanupOnFail: true
+
+environments:
+  default:
+    values:
+      - environments/default.yaml
+  k3d-v4only:
+    values:
+      - environments/k3d-v4only.yaml
+  orbstack:
+    values:
+      - environments/orbstack.yaml
+  rancher:
+    values:
+      - environments/rancher.yaml
+
+---
+
+{{ $namespace := env "NAMESPACE" | default "postgres" }}
+
+{{/* Image configuration via environment variables */}}
+{{ $useLocalImages := env "USE_LOCAL_IMAGES" | default "false" }}
+
+{{ $pgskipperImage := "" }}
+{{ $pgskipperTag := "" }}
+{{ $pullPolicy := "" }}
+
+{{ if eq $useLocalImages "true" }}
+  {{ $pgskipperImage = "pgskipper-operator" }}
+  {{ $pgskipperTag = "local" }}
+  {{ $pullPolicy = "Never" }}
+{{ else }}
+  {{ $pgskipperImage = env "PGSKIPPER_IMAGE" | default "ghcr.io/netcracker/pgskipper-operator" }}
+  {{ $pgskipperTag = env "PGSKIPPER_TAG" | default "main" }}
+  {{ $pullPolicy = "IfNotPresent" }}
+{{ end }}
+
+releases:
+  #############################################################################
+  # Patroni-Core Operator
+  # Manages PostgreSQL core functionality and CRDs
+  #############################################################################
+  - name: patroni-core
+    namespace: {{ $namespace }}
+    chart: ./operator/charts/patroni-core
+    labels:
+      component: postgres-operator
+      type: core
+    values:
+      - ./tests/examples/spring-boot-failover-test/helm-charts/postgresql/patroni-core-simple.yaml
+      - operator:
+          image: {{ $pgskipperImage }}:{{ $pgskipperTag }}
+          imagePullPolicy: {{ $pullPolicy }}
+    hooks:
+      # Build custom operator images if requested
+      - events: ["presync"]
+        showlogs: true
+        command: "bash"
+        args:
+          - "-c"
+          # language=bash
+          - |
+            if [ "{{ $useLocalImages }}" != "true" ]; then
+              echo "Using official pgskipper-operator images: {{ $pgskipperImage }}:{{ $pgskipperTag }}"
+            else
+              echo "Building local pgskipper-operator images..."
+
+              # Switch to configured Docker context (for local k8s like OrbStack/Rancher)
+              DOCKER_CONTEXT="{{ .Values.dockerContext | default "default" }}"
+              CURRENT_CONTEXT=$(docker context show)
+              echo "Current Docker context: $CURRENT_CONTEXT"
+              echo "Target Docker context: $DOCKER_CONTEXT"
+
+              if [ "$CURRENT_CONTEXT" != "$DOCKER_CONTEXT" ]; then
+                echo "Switching to $DOCKER_CONTEXT context..."
+                docker context use "$DOCKER_CONTEXT"
+              fi
+
+              TAG_ENV="{{ $pgskipperTag }}" DOCKER_NAMES="{{ $pgskipperImage }}:{{ $pgskipperTag }}" make docker-build
+
+              echo "✓ Local operator images built successfully: {{ $pgskipperImage }}:{{ $pgskipperTag }}"
+            fi
+      # Validate storage before deployment
+      - events: ["presync"]
+        showlogs: true
+        command: "bash"
+        args:
+          - "-c"
+          # language=bash
+          - |
+            cd tests/examples/spring-boot-failover-test
+            ./scripts/configure-storage.sh --auto
+      # Wait for operator to be ready
+      - events: ["postsync"]
+        showlogs: true
+        command: "bash"
+        args:
+          - "-c"
+          # language=bash
+          - |
+            echo "Waiting for Patroni-Core Operator to be ready..."
+            kubectl wait --for=condition=available --timeout=300s \
+              deployment -l name=patroni-core-operator -n {{ $namespace }} 2>/dev/null || true
+
+  #############################################################################
+  # Patroni-Services Operator
+  # Manages PostgreSQL services and high availability
+  #############################################################################
+  - name: patroni-services
+    namespace: {{ $namespace }}
+    chart: ./operator/charts/patroni-services
+    labels:
+      component: postgres-operator
+      type: services
+    values:
+      - ./tests/examples/spring-boot-failover-test/helm-charts/postgresql/patroni-services-simple.yaml
+      - operator:
+          image: {{ $pgskipperImage }}:{{ $pgskipperTag }}
+          imagePullPolicy: {{ $pullPolicy }}
+    hooks:
+      # Wait for PostgreSQL cluster to be ready
+      - events: ["postsync"]
+        showlogs: true
+        command: "bash"
+        args:
+          - "-c"
+          # language=bash
+          - |
+            echo "Waiting for PostgreSQL cluster to initialize..."
+            sleep 30
+
+            echo "Waiting for primary PostgreSQL pod..."
+            timeout 600 bash -c 'until kubectl get pods -n {{ $namespace }} --selector=pgtype=master 2>/dev/null | grep -q Running; do sleep 5; done' || true
+
+            echo "Waiting for all PostgreSQL pods to be ready..."
+            kubectl wait --for=condition=ready --timeout=600s \
+              pods -l app=postgres -n {{ $namespace }} 2>/dev/null || true
+
+            echo "PostgreSQL cluster is ready!"

operator/build/configs/patroni.config.yaml

@@ -51,7 +51,7 @@ kubernetes:
     app: ${PATRONI_CLUSTER_NAME}
   role_label: pgtype
   scope_label: app
-  pod_ip: ${LISTEN_ADDR}
+  pod_ip: ${POD_DNS_NAME}
 postgresql:
   authentication:
     replication:
@@ -67,15 +67,15 @@ postgresql:
     on_role_change: /setup_endpoint_callback.py
     on_start: /setup_endpoint_callback.py
     on_stop: /setup_endpoint_callback.py
-  connect_address: ${LISTEN_ADDR}:5432
+  connect_address: ${POD_DNS_NAME}:5432
   data_dir: /var/lib/pgsql/data/postgresql_${NODE_NAME}
-  listen: '0.0.0.0, ::0:5432'
+  listen: '*:5432'
   parameters:
     unix_socket_directories: /var/run/postgresql, /tmp
   pgpass: /tmp/pgpass0
 restapi:
-  connect_address: ${LISTEN_ADDR}:8008
-  listen: ${LISTEN_ADDR}:8008
+  connect_address: ${POD_DNS_NAME}:8008
+  listen: '*:8008'
 tags:
   clonefrom: false
   nofailover: ${DR_MODE}

operator/charts/patroni-services/values.yaml

@@ -415,7 +415,7 @@ connectionPooler:
       '*': "host=pg-patroni-direct port=5432"
     pgbouncer:
       listen_port: '6432'
-      listen_addr: '0.0.0.0'
+      listen_addr: '*'
       auth_type: 'md5'
       auth_file: '/etc/pgbouncer/userlist.txt'
       auth_user: 'pgbouncer'

operator/controllers/patroni_core_controller.go

@@ -196,14 +196,6 @@ func (pr *PatroniCoreReconciler) Reconcile(ctx context.Context, request ctrl.Req
 	pr.logger.Info("Reconcile will be started...")
 	time.Sleep(30 * time.Second)
 
-	if err := credentials.ProcessCreds(pr.helper.GetOwnerReferences()); err != nil {
-		return pr.handleReconcileError(maxReconcileAttempts,
-			"CanNotActualizeCredsOnCluster",
-			newCrHash,
-			"Error during actualization of creds on cluster",
-			err)
-	}
-
 	if len(cr.RunTestsTime) > 0 {
 		pr.logger.Info("runTestsOnly : true")
 		if err := pr.createTestsPods(cr); err != nil {
@@ -274,6 +266,15 @@ func (pr *PatroniCoreReconciler) Reconcile(ctx context.Context, request ctrl.Req
 		return reconcile.Result{RequeueAfter: time.Minute}, err
 	}
 
+	// Process credentials after cluster is created
+	if err := credentials.ProcessCreds(pr.helper.GetOwnerReferences()); err != nil {
+		return pr.handleReconcileError(maxReconcileAttempts,
+			"CanNotActualizeCredsOnCluster",
+			newCrHash,
+			"Error during actualization of creds on cluster",
+			err)
+	}
+
 	if err := pr.helper.UpdatePatroniConfigMaps(); err != nil {
 		pr.logger.Error("error during update of patroni config maps", zap.Error(err))
 		// will not return err because there is a slight chance, that

operator/pkg/deployment/patroni.go

@@ -215,6 +215,23 @@ func NewPatroniStatefulset(cr *patroniv1.PatroniCore, deploymentIdx int, cluster
 										},
 									},
 								},
+								{
+									Name: "POD_NAME",
+									ValueFrom: &corev1.EnvVarSource{
+										FieldRef: &corev1.ObjectFieldSelector{
+											APIVersion: "v1",
+											FieldPath:  "metadata.name",
+										},
+									},
+								},
+								{
+									Name:  "HEADLESS_SERVICE",
+									Value: "patroni-headless",
+								},
+								{
+									Name:  "POD_DNS_NAME",
+									Value: "$(POD_NAME).$(HEADLESS_SERVICE).$(POD_NAMESPACE).svc.cluster.local",
+								},
 								{
 									Name:  "PATRONI_CLUSTER_NAME",
 									Value: clusterName,
@@ -266,7 +283,7 @@ func NewPatroniStatefulset(cr *patroniv1.PatroniCore, deploymentIdx int, cluster
 					DNSPolicy:                     corev1.DNSClusterFirst,
 				},
 			},
-			ServiceName:                          "backrest-headless",
+			ServiceName:                          "patroni-headless",
 			PodManagementPolicy:                  appsv1.OrderedReadyPodManagement,
 			UpdateStrategy:                       appsv1.StatefulSetUpdateStrategy{Type: appsv1.RollingUpdateStatefulSetStrategyType},
 			RevisionHistoryLimit:                 ptr.To[int32](10),

operator/pkg/deployment/pgbackrest.go

@@ -183,6 +183,30 @@ func GetBackrestHeadless() *corev1.Service {
 	}
 }
 
+func GetPatroniHeadless(clusterName string) *corev1.Service {
+	labels := map[string]string{"app": clusterName}
+	ports := []corev1.ServicePort{
+		{Name: "postgresql", Port: 5432},
+		{Name: "patroni-api", Port: 8008},
+	}
+	return &corev1.Service{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: "v1",
+			Kind:       "Service",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "patroni-headless",
+			Namespace: util.GetNameSpace(),
+		},
+
+		Spec: corev1.ServiceSpec{
+			Selector:  labels,
+			Ports:     ports,
+			ClusterIP: "None",
+		},
+	}
+}
+
 func getPgBackRestSettings(pgBackrestSpec *v1.PgBackRest, isStandby bool) string {
 	var listSettings []string
 	listSettings = append(listSettings, "[global]")

operator/pkg/patroni/patroni.go

@@ -336,11 +336,33 @@ func updateStandbyClusterSettings(configMap *corev1.ConfigMap, settings interfac
 	err := yaml.Unmarshal([]byte(configMap.Data[configMapKey]), &config)
 	if err != nil {
 		logger.Error("Could not unmarshal patroni config map", zap.Error(err))
+		return configMap
 	}
-	config["bootstrap"].(map[interface{}]interface{})["dcs"].(map[interface{}]interface{})["standby_cluster"] = settings
+
+	// Validate config structure exists before type assertion
+	if config == nil {
+		logger.Error("Config map is nil after unmarshal")
+		return configMap
+	}
+
+	bootstrap, ok := config["bootstrap"].(map[interface{}]interface{})
+	if !ok || bootstrap == nil {
+		logger.Error("Config map missing 'bootstrap' section or wrong type")
+		return configMap
+	}
+
+	dcs, ok := bootstrap["dcs"].(map[interface{}]interface{})
+	if !ok || dcs == nil {
+		logger.Error("Config map missing 'bootstrap.dcs' section or wrong type")
+		return configMap
+	}
+
+	dcs["standby_cluster"] = settings
+
 	result, err := yaml.Marshal(config)
 	if err != nil {
 		logger.Error("Could not marshal patroni config map", zap.Error(err))
+		return configMap
 	}
 	configMap.Data[configMapKey] = string(result)
 	return configMap

operator/pkg/reconciler/patroni.go

@@ -456,6 +456,14 @@ func (r *PatroniReconciler) processPatroniServices(cr *v1.PatroniCore, patroniSp
 			}
 		}
 	}
+
+	// Create patroni headless service for DNS-based pod discovery
+	patroniHeadless := deployment.GetPatroniHeadless(r.cluster.ClusterName)
+	if err := r.helper.ResourceManager.CreateOrUpdateService(patroniHeadless); err != nil {
+		logger.Error(fmt.Sprintf("Cannot create service %s", patroniHeadless.Name), zap.Error(err))
+		return err
+	}
+
 	return nil
 }