Nightly Integration tests for self-hosted infrastructure.

.github/workflows/integration-tests.yaml

@@ -9,6 +9,10 @@ on:
 jobs:
   integration-tests-multi-region:
     runs-on: ubuntu-latest-4-core
+    strategy:
+      fail-fast: false
+      matrix:
+        provider: [k3d, gcp]
     timeout-minutes: 90
     permissions:
       contents: 'read'
@@ -35,11 +39,11 @@ jobs:
           gcloud components install gke-gcloud-auth-plugin
       - name: Run tests (multi-region)
         env:
-          isNightly: true
+          PROVIDER: ${{ matrix.provider }}
           USE_GKE_GCLOUD_AUTH_PLUGIN: True
         run: |
-          set -e
-          make test/nightly-e2e/multi-region | tee test_output.log || true
+          set -euo pipefail
+          make test/e2e/multi-region | tee test_output.log
       - name: Archive test results
         if: ${{ always() }}
         uses: actions/upload-artifact@v4
@@ -123,6 +127,10 @@ jobs:
             "${{ secrets.SLACK_WEBHOOK_URL }}"
   integration-tests-single-region:
     runs-on: ubuntu-latest-4-core
+    strategy:
+      fail-fast: false
+      matrix:
+        provider: [k3d, gcp]
     timeout-minutes: 90
     permissions:
       contents: read
@@ -150,11 +158,11 @@ jobs:
           gcloud components install gke-gcloud-auth-plugin
       - name: Run tests (single-region)
         env:
-          isNightly: true
+          PROVIDER: ${{ matrix.provider }}
           USE_GKE_GCLOUD_AUTH_PLUGIN: True
         run: |
-          set -e
-          make test/nightly-e2e/single-region | tee test_output.log || true
+          set -euo pipefail
+          make test/e2e/single-region | tee test_output.log
       - name: Archive test results
         if: ${{ always() }}
         uses: actions/upload-artifact@v4

Makefile

@@ -4,6 +4,7 @@ ifeq ($(UNAME_S),Linux)
   COCKROACH_BIN ?= https://binaries.cockroachdb.com/cockroach-v23.2.0.linux-amd64.tgz
   HELM_BIN ?= https://get.helm.sh/helm-v3.14.0-linux-amd64.tar.gz
   K3D_BIN ?=  https://github.com/k3d-io/k3d/releases/download/v5.7.4/k3d-linux-amd64
+  KIND_BIN ?= https://kind.sigs.k8s.io/dl/v0.29.0/kind-linux-amd64
   KUBECTL_BIN ?= https://dl.k8s.io/release/v1.29.1/bin/linux/amd64/kubectl
   YQ_BIN ?= https://github.com/mikefarah/yq/releases/download/v4.31.2/yq_linux_amd64
   JQ_BIN ?= https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64
@@ -14,6 +15,7 @@ ifeq ($(UNAME_S),Darwin)
   COCKROACH_BIN ?= https://binaries.cockroachdb.com/cockroach-v23.2.0.darwin-10.9-amd64.tgz
   HELM_BIN ?= https://get.helm.sh/helm-v3.14.0-darwin-amd64.tar.gz
   K3D_BIN ?=  https://github.com/k3d-io/k3d/releases/download/v5.7.4/k3d-darwin-arm64
+  KIND_BIN ?= https://kind.sigs.k8s.io/dl/v0.29.0/kind-darwin-arm64
   KUBECTL_BIN ?= https://dl.k8s.io/release/v1.29.1/bin/darwin/amd64/kubectl
   YQ_BIN ?= https://github.com/mikefarah/yq/releases/download/v4.31.2/yq_darwin_amd64
   JQ_BIN ?= https://github.com/stedolan/jq/releases/download/jq-1.6/jq-osx-amd64
@@ -118,10 +120,10 @@ test/e2e/%: bin/cockroach bin/kubectl bin/helm build/self-signer test/cluster/up
 	$(MAKE) test/cluster/down; \
 	exit $${EXIT_CODE:-0}
 
-test/e2e/multi-region: bin/cockroach bin/kubectl bin/helm  build/self-signer
+test/e2e/multi-region: bin/cockroach bin/kubectl bin/helm  build/self-signer bin/k3d
 	@PATH="$(PWD)/bin:${PATH}" go test -timeout 60m -v -test.run TestOperatorInMultiRegion ./tests/e2e/operator/multiRegion/... || (echo "Multi region tests failed with exit code $$?" && exit 1)
 
-test/e2e/single-region: bin/cockroach bin/kubectl bin/helm build/self-signer
+test/e2e/single-region: bin/cockroach bin/kubectl bin/helm build/self-signer bin/k3d
 	@PATH="$(PWD)/bin:${PATH}" go test -timeout 60m -v -test.run TestOperatorInSingleRegion ./tests/e2e/operator/singleRegion/... || (echo "Single region tests failed with exit code $$?" && exit 1)
 
 test/e2e/migrate: bin/cockroach bin/kubectl bin/helm bin/migration-helper build/self-signer test/cluster/up/3
@@ -135,13 +137,6 @@ test/single-cluster/up: bin/k3d
 test/multi-cluster/down: bin/k3d
 	 ./tests/k3d/dev-multi-cluster.sh down
 
-test/nightly-e2e/single-region: bin/cockroach bin/kubectl bin/helm build/self-signer
-	@PATH="$(PWD)/bin:${PATH}" go test -timeout 60m -v -test.run TestOperatorInSingleRegion ./tests/e2e/operator/singleRegion/... || (echo "Single region tests failed with exit code $$?" && exit 1)
-
-test/nightly-e2e/multi-region: bin/cockroach bin/kubectl bin/helm build/self-signer
-	@PATH="$(PWD)/bin:${PATH}" go test -timeout 60m -v -test.run TestOperatorInMultiRegion ./tests/e2e/operator/multiRegion/... || (echo "Multi region tests failed with exit code $$?" && exit 1)
-
-
 test/lint: bin/helm ## lint the helm chart
 	@build/lint.sh && \
 	bin/helm lint cockroachdb && \
@@ -176,6 +171,11 @@ bin/k3d: ## install k3d
 	@curl -Lo bin/k3d $(K3D_BIN)
 	@chmod +x bin/k3d
 
+bin/kind: ## install kind
+	@mkdir -p bin
+	@curl -Lo bin/kind $(KIND_BIN)
+	@chmod +x bin/kind
+
 bin/kubectl: ## install kubectl
 	@mkdir -p bin
 	@curl -Lo bin/kubectl $(KUBECTL_BIN)

cockroachdb-parent/charts/operator/templates/operator.yaml

@@ -254,6 +254,7 @@ rules:
       - admissionregistration.k8s.io
     resources:
       - validatingwebhookconfigurations
+      - mutatingwebhookconfigurations
     verbs:
       - create
   - apiGroups:
@@ -265,6 +266,15 @@ rules:
     verbs:
       - get
       - patch
+  - apiGroups:
+      - admissionregistration.k8s.io
+    resources:
+      - mutatingwebhookconfigurations
+    resourceNames:
+      - cockroach-mutating-webhook-config
+    verbs:
+      - get
+      - patch
   # The "create" verb cannot be qualified with resourceNames, so grant the
   # unqualified permission so that the operator can create new CRDs. But only
   # allow the operator to get and patch its own CRDs.

cockroachdb-parent/charts/operator/values.yaml

@@ -4,13 +4,13 @@
 # image captures the container image settings for Operator pods.
 image:
   # registry is the container registry where the image is stored.
-  registry: "us-docker.pkg.dev/releases-prod/self-hosted"
+  registry: "us-docker.pkg.dev/cockroach-cloud-images/development"
   # repository defines the image repository.
-  repository: "cockroachdb-operator@sha256"
+  repository: "cockroach-operator@sha256"
   # pullPolicy specifies the image pull policy.
   pullPolicy: IfNotPresent
   # tag is the image tag.
-  tag: "6f62639e9fee99d99b0387a9dccda84daa1a489b592b008f2f354ec57eae09ac"
+  tag: "72844b85354fd55b9a487abbd6e253b7d5081f65513c3813fde0ceb7d3ee2f70"
 # certificate defines the certificate settings for the Operator.
 certificate:
   # validForDays specifies the number of days the certificate is valid for.

tests/e2e/migrate/helm_chart_to_cockroach_enterprise_operator_test.go

@@ -112,7 +112,7 @@ func (h *HelmChartToOperator) TestDefaultMigration(t *testing.T) {
 		k8s.RunKubectl(t, kubectlOptions, "delete", "priorityclass", "crdb-critical")
 	}()
 
-	operator.InstallCockroachDBEnterpriseOperator(t, kubectlOptions)
+	operator.InstallCockroachDBEnterpriseOperator(t, kubectlOptions, nil)
 	defer func() {
 		t.Log("Uninstall the cockroachdb enterprise operator")
 		operator.UninstallCockroachDBEnterpriseOperator(t, kubectlOptions)
@@ -221,7 +221,7 @@ func (h *HelmChartToOperator) TestCertManagerMigration(t *testing.T) {
 		k8s.RunKubectl(t, kubectlOptions, "delete", "priorityclass", "crdb-critical")
 	}()
 
-	operator.InstallCockroachDBEnterpriseOperator(t, kubectlOptions)
+	operator.InstallCockroachDBEnterpriseOperator(t, kubectlOptions, nil)
 	defer func() {
 		t.Log("Uninstall the cockroachdb enterprise operator")
 		operator.UninstallCockroachDBEnterpriseOperator(t, kubectlOptions)
@@ -308,7 +308,7 @@ func (h *HelmChartToOperator) TestPCRPrimaryMigration(t *testing.T) {
 		k8s.RunKubectl(t, kubectlOptions, "delete", "priorityclass", "crdb-critical")
 	}()
 
-	operator.InstallCockroachDBEnterpriseOperator(t, kubectlOptions)
+	operator.InstallCockroachDBEnterpriseOperator(t, kubectlOptions, nil)
 	defer func() {
 		t.Log("Uninstall the cockroachdb enterprise operator")
 		operator.UninstallCockroachDBEnterpriseOperator(t, kubectlOptions)

tests/e2e/migrate/public_operator_to_cockroach_enterprise_operator_test.go

@@ -127,7 +127,7 @@ func (o *PublicOperatorToCockroachEnterpriseOperator) TestDefaultMigration(t *te
 	k8s.KubectlApply(t, kubectlOptions, filepath.Join(manifestsDirPath, "rbac.yaml"))
 
 	t.Log("Install the cockroachdb enterprise operator")
-	operator.InstallCockroachDBEnterpriseOperator(t, kubectlOptions)
+	operator.InstallCockroachDBEnterpriseOperator(t, kubectlOptions, nil)
 	defer func() {
 		t.Log("Uninstall the cockroachdb enterprise operator")
 		operator.UninstallCockroachDBEnterpriseOperator(t, kubectlOptions)

tests/e2e/operator/infra/common.go

@@ -24,8 +24,11 @@ const (
 
 // Common constants.
 const (
-	defaultRetries        = 30
-	defaultRetryInterval  = 10 * time.Second
+	defaultRetries       = 30
+	defaultRetryInterval = 10 * time.Second
+	// Load balancer specific retry settings (extended for AWS)
+	loadBalancerRetries   = 60 // 10 minutes total
+	loadBalancerInterval  = 10 * time.Second
 	coreDNSDeploymentName = "coredns"
 	coreDNSServiceName    = "crl-core-dns"
 	coreDNSNamespace      = "kube-system"
@@ -237,7 +240,7 @@ func finalizeCoreDNSDeployment(t *testing.T, kubectlOpts *k8s.KubectlOptions) er
 func WaitForCoreDNSServiceIPs(t *testing.T, kubectlOpts *k8s.KubectlOptions) ([]string, error) {
 	var ips []string
 
-	_, err := retry.DoWithRetryE(t, "waiting for CoreDNS service IPs", defaultRetries, defaultRetryInterval,
+	_, err := retry.DoWithRetryE(t, "waiting for CoreDNS service IPs", loadBalancerRetries, loadBalancerInterval,
 		func() (string, error) {
 			svc, err := k8s.GetServiceE(t, kubectlOpts, coreDNSServiceName)
 			if err != nil {