EVEREST-1841 MongoDB Major Version Upgrades

.github/workflows/ci.yml

@@ -105,10 +105,10 @@ jobs:
           PG_OPERATOR_VERSION: 2.5.0
           PERCONA_VERSION_SERVICE_URL: https://check-dev.percona.com/versions/v1
 
-      - name: Run upgrade tests
+      - name: Run operator upgrade tests
         if: always()
         run: |
-          kubectl kuttl test --config ./tests/integration/kuttl-upgrade.yaml
+          kubectl kuttl test --config ./tests/integration/kuttl-operator-upgrade.yaml
         env:
           PXC_OPERATOR_VERSION: 1.16.1
           PSMDB_OPERATOR_VERSION: 1.19.0
@@ -120,7 +120,7 @@ jobs:
 
   eks-e2e-tests:
     name: E2E tests (EKS)
-    if: contains(github.event.pull_request.labels.*.name, 'test/e2e') || contains(github.event.pull_request.labels.*.name, 'test/e2e-upgrade') || contains(github.event.pull_request.labels.*.name, 'test/e2e-core')
+    if: contains(github.event.pull_request.labels.*.name, 'test/e2e') || contains(github.event.pull_request.labels.*.name, 'test/e2e-operator-upgrade') || contains(github.event.pull_request.labels.*.name, 'test/e2e-core') || contains(github.event.pull_request.labels.*.name, 'test/e2e-db-upgrade')
     timeout-minutes: 80
     concurrency: eks-e2e-tests
     strategy:
@@ -186,9 +186,9 @@ jobs:
           NODEPOOL_NAME: "everest-e2e-dev-eks-workers"
 
       - name: Run upgrade tests
-        if: contains(github.event.pull_request.labels.*.name, 'test/e2e') || contains(github.event.pull_request.labels.*.name, 'test/e2e-upgrade')
+        if: contains(github.event.pull_request.labels.*.name, 'test/e2e') || contains(github.event.pull_request.labels.*.name, 'test/e2e-operator-upgrade')
         run: |
-          kubectl kuttl test --config ./tests/e2e/kuttl-upgrade.yaml
+          kubectl kuttl test --config ./tests/e2e/kuttl-operator-upgrade.yaml
         env:
           PXC_OPERATOR_VERSION: 1.16.1
           PSMDB_OPERATOR_VERSION: 1.19.0
@@ -280,6 +280,52 @@ jobs:
           kubectl delete apiservice v1.packages.operators.coreos.com --ignore-not-found=true
           kubectl get crd -o name | grep .coreos.com$ | awk -F '/' {'print $2'} | xargs --no-run-if-empty kubectl delete crd
 
+      - name: Run db upgrade tests
+        if: contains(github.event.pull_request.labels.*.name, 'test/e2e') || contains(github.event.pull_request.labels.*.name, 'test/e2e-db-upgrade')
+        run: |
+          kubectl kuttl test --config ./tests/e2e/kuttl-db-upgrade.yaml
+        env:
+          PXC_OPERATOR_VERSION: 1.16.1
+          PSMDB_OPERATOR_VERSION: 1.19.0
+          PG_OPERATOR_VERSION: 2.5.0
+          PERCONA_VERSION_SERVICE_URL: https://check-dev.percona.com/versions/v1
+
+      - name: Clean up custom resources
+        if: always()
+        run: |
+          namespaces=$(kubectl get pxc -A -o jsonpath='{.items[*].metadata.namespace}')
+          for namespace in ${namespaces[@]}
+          do
+            kubectl -n $namespace get pxc -o name | awk -F '/' {'print $2'} | xargs --no-run-if-empty kubectl patch pxc -n $namespace -p '{"metadata":{"finalizers":null}}' --type merge
+          done
+
+          namespaces=$(kubectl get psmdb -A -o jsonpath='{.items[*].metadata.namespace}')
+          for namespace in ${namespaces[@]}
+          do
+            kubectl -n $namespace get psmdb -o name | awk -F '/' {'print $2'} | xargs --no-run-if-empty kubectl patch psmdb -n $namespace -p '{"metadata":{"finalizers":null}}' --type merge
+          done
+
+          namespaces=$(kubectl get pg -A -o jsonpath='{.items[*].metadata.namespace}')
+          for namespace in ${namespaces[@]}
+          do
+            kubectl -n $namespace get pg -o name | awk -F '/' {'print $2'} | xargs --no-run-if-empty kubectl patch pg -n $namespace -p '{"metadata":{"finalizers":null}}' --type merge
+          done
+
+          namespaces=$(kubectl get db -A -o jsonpath='{.items[*].metadata.namespace}')
+          for namespace in ${namespaces[@]}
+          do
+            kubectl -n $namespace get db -o name | awk -F '/' {'print $2'} | xargs --no-run-if-empty kubectl patch db -n $namespace -p '{"metadata":{"finalizers":null}}' --type merge
+          done
+
+          kubectl delete db --all-namespaces --all --cascade=foreground
+          kubectl delete pvc --all-namespaces --all
+          kubectl delete backupstorage --all-namespaces --all
+          kubectl get ns -o name | grep kuttl  | awk -F '/' {'print $2'} | xargs --no-run-if-empty kubectl delete ns
+          kubectl delete ns operators olm --ignore-not-found=true --wait=false
+          sleep 10
+          kubectl delete apiservice v1.packages.operators.coreos.com --ignore-not-found=true
+          kubectl get crd -o name | grep .coreos.com$ | awk -F '/' {'print $2'} | xargs --no-run-if-empty kubectl delete crd
+
       - name: Scale down cluster
         if: always()
         run: |

internal/controller/providers/psmdb/provider.go

@@ -203,6 +203,7 @@ func defaultSpec() psmdbv1.PerconaServerMongoDBSpec {
 		UpgradeOptions: psmdbv1.UpgradeOptions{
 			Apply:    "disabled",
 			Schedule: "0 4 * * *",
+			SetFCV:   true,
 		},
 		PMM: psmdbv1.PMMSpec{
 			Enabled: false,

tests/e2e/db-upgrade/psmdb/01-assert.yaml

@@ -0,0 +1,55 @@
+apiVersion: kuttl.dev/v1
+kind: TestAssert
+timeout: 120
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: perconaservermongodbs.psmdb.percona.com
+spec:
+  group: psmdb.percona.com
+  names:
+    kind: PerconaServerMongoDB
+    listKind: PerconaServerMongoDBList
+    plural: perconaservermongodbs
+    shortNames:
+    - psmdb
+    singular: perconaservermongodb
+  scope: Namespaced
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: databaseclusters.everest.percona.com
+spec:
+  group: everest.percona.com
+  names:
+    kind: DatabaseCluster
+    listKind: DatabaseClusterList
+    plural: databaseclusters
+    shortNames:
+    - db
+    - dbc
+    - dbcluster
+    singular: databasecluster
+  scope: Namespaced
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: percona-server-mongodb-operator
+status:
+  availableReplicas: 1
+  observedGeneration: 1
+  readyReplicas: 1
+  replicas: 1
+  updatedReplicas: 1
+---
+apiVersion: everest.percona.com/v1alpha1
+kind: DatabaseEngine
+metadata:
+  name: percona-server-mongodb-operator
+spec:
+  type: psmdb
+status:
+  status: installed

tests/e2e/db-upgrade/psmdb/01-deploy-dbaas.yaml

@@ -0,0 +1,7 @@
+apiVersion: kuttl.dev/v1
+kind: TestStep
+timeout: 10
+commands:
+  - script: SYSTEM_NAMESPACE=$NAMESPACE MONITORING_NAMESPACE=$NAMESPACE DB_NAMESPACES=$NAMESPACE ../../../../bin/manager -metrics-bind-address :0 -health-probe-bind-address :0
+    background: true
+    skipLogOutput: true

tests/e2e/db-upgrade/psmdb/10-assert.yaml

@@ -0,0 +1,75 @@
+apiVersion: kuttl.dev/v1
+kind: TestAssert
+timeout: 300
+commands:
+  - command: kubectl wait --for=jsonpath='{.status.crVersion}'=${PSMDB_OPERATOR_VERSION} db/test-psmdb-cluster -n "${NAMESPACE}"
+---
+apiVersion: everest.percona.com/v1alpha1
+kind: DatabaseCluster
+metadata:
+  name: test-psmdb-cluster
+spec:
+  engine:
+    type: psmdb
+    version: 6.0.16-13
+    config: |
+     operationProfiling:
+        mode: slowOp
+    userSecretsName: psmdb-sample-secrets
+    replicas: 3
+    storage:
+      size: 15G
+    resources:
+      cpu: 300m
+      memory: 500M
+  proxy:
+    type: mongos
+    replicas: 3
+status:
+  ready: 3
+  size: 3
+  status: ready
+---
+apiVersion: psmdb.percona.com/v1
+kind: PerconaServerMongoDB
+metadata:
+  name: test-psmdb-cluster
+  finalizers:
+    - percona.com/delete-psmdb-pods-in-order
+    - percona.com/delete-psmdb-pvc
+spec:
+  image: percona/percona-server-mongodb:6.0.16-13
+  multiCluster:
+    enabled: false
+  replsets:
+    - configuration: |
+       operationProfiling:
+          mode: slowOp
+      expose:
+        enabled: false
+      name: rs0
+      resources:
+        limits:
+          cpu: 300m
+          memory:  500M
+      size:        3
+      volumeSpec:
+        persistentVolumeClaim:
+          resources:
+            requests:
+              storage: 15G
+  secrets:
+    users:  psmdb-sample-secrets
+  sharding:
+    enabled: false
+status:
+  mongoVersion: 6.0.16-13
+  replsets:
+    rs0:
+      initialized: true
+      ready: 3
+      size: 3
+      status: ready
+  ready: 3
+  size: 3
+  state: ready

tests/e2e/db-upgrade/psmdb/10-create-cluster.yaml

@@ -0,0 +1,27 @@
+apiVersion: kuttl.dev/v1
+kind: TestStep
+---
+apiVersion: everest.percona.com/v1alpha1
+kind: DatabaseCluster
+metadata:
+  name: test-psmdb-cluster
+  finalizers:
+  - everest.percona.com/upstream-cluster-cleanup
+  - foregroundDeletion
+spec:
+  engine:
+    type: psmdb
+    version: 6.0.16-13
+    config: |
+     operationProfiling:
+        mode: slowOp
+    userSecretsName: psmdb-sample-secrets
+    replicas: 3
+    storage:
+      size: 15G
+    resources:
+      cpu: 300m
+      memory: 500M
+  proxy:
+    type: mongos
+    replicas: 3

tests/e2e/db-upgrade/psmdb/20-assert.yaml

@@ -0,0 +1,75 @@
+apiVersion: kuttl.dev/v1
+kind: TestAssert
+timeout: 300
+commands:
+  - command: kubectl wait --for=jsonpath='{.status.crVersion}'=${PSMDB_OPERATOR_VERSION} db/test-psmdb-cluster -n "${NAMESPACE}"
+---
+apiVersion: everest.percona.com/v1alpha1
+kind: DatabaseCluster
+metadata:
+  name: test-psmdb-cluster
+spec:
+  engine:
+    type: psmdb
+    version: 6.0.18-15
+    config: |
+     operationProfiling:
+        mode: slowOp
+    userSecretsName: psmdb-sample-secrets
+    replicas: 3
+    storage:
+      size: 15G
+    resources:
+      cpu: 300m
+      memory: 500M
+  proxy:
+    type: mongos
+    replicas: 3
+status:
+  ready: 3
+  size: 3
+  status: ready
+---
+apiVersion: psmdb.percona.com/v1
+kind: PerconaServerMongoDB
+metadata:
+  name: test-psmdb-cluster
+  finalizers:
+    - percona.com/delete-psmdb-pods-in-order
+    - percona.com/delete-psmdb-pvc
+spec:
+  image: percona/percona-server-mongodb:6.0.18-15-multi
+  multiCluster:
+    enabled: false
+  replsets:
+    - configuration: |
+       operationProfiling:
+          mode: slowOp
+      expose:
+        enabled: false
+      name: rs0
+      resources:
+        limits:
+          cpu: 300m
+          memory:  500M
+      size:        3
+      volumeSpec:
+        persistentVolumeClaim:
+          resources:
+            requests:
+              storage: 15G
+  secrets:
+    users:  psmdb-sample-secrets
+  sharding:
+    enabled: false
+status:
+  mongoVersion: 6.0.18-15
+  replsets:
+    rs0:
+      initialized: true
+      ready: 3
+      size: 3
+      status: ready
+  ready: 3
+  size: 3
+  state: ready

tests/e2e/db-upgrade/psmdb/20-minor-upgrade.yaml

@@ -0,0 +1,27 @@
+apiVersion: kuttl.dev/v1
+kind: TestStep
+---
+apiVersion: everest.percona.com/v1alpha1
+kind: DatabaseCluster
+metadata:
+  name: test-psmdb-cluster
+  finalizers:
+  - everest.percona.com/upstream-cluster-cleanup
+  - foregroundDeletion
+spec:
+  engine:
+    type: psmdb
+    version: 6.0.18-15
+    config: |
+     operationProfiling:
+        mode: slowOp
+    userSecretsName: psmdb-sample-secrets
+    replicas: 3
+    storage:
+      size: 15G
+    resources:
+      cpu: 300m
+      memory: 500M
+  proxy:
+    type: mongos
+    replicas: 3

tests/e2e/db-upgrade/psmdb/30-another-minor-upgrade.yaml

@@ -0,0 +1,27 @@
+apiVersion: kuttl.dev/v1
+kind: TestStep
+---
+apiVersion: everest.percona.com/v1alpha1
+kind: DatabaseCluster
+metadata:
+  name: test-psmdb-cluster
+  finalizers:
+  - everest.percona.com/upstream-cluster-cleanup
+  - foregroundDeletion
+spec:
+  engine:
+    type: psmdb
+    version: 6.0.19-16
+    config: |
+     operationProfiling:
+        mode: slowOp
+    userSecretsName: psmdb-sample-secrets
+    replicas: 3
+    storage:
+      size: 15G
+    resources:
+      cpu: 300m
+      memory: 500M
+  proxy:
+    type: mongos
+    replicas: 3