Nightly Integration tests for self-hosted infrastructure.

Author: NishanthNalluri

.github/workflows/integration-tests.yaml

@@ -9,6 +9,10 @@ on:
 jobs:
   integration-tests-multi-region:
     runs-on: ubuntu-latest-4-core
+    strategy:
+      fail-fast: false
+      matrix:
+        provider: [kind, gcp]
     timeout-minutes: 90
     permissions:
       contents: 'read'
@@ -35,11 +39,11 @@ jobs:
           gcloud components install gke-gcloud-auth-plugin
       - name: Run tests (multi-region)
         env:
-          isNightly: true
+          PROVIDER: ${{ matrix.provider }}
           USE_GKE_GCLOUD_AUTH_PLUGIN: True
         run: |
-          set -e
-          make test/nightly-e2e/multi-region | tee test_output.log || true
+          set -euo pipefail
+          make test/nightly-e2e/multi-region | tee test_output.log
       - name: Archive test results
         if: ${{ always() }}
         uses: actions/upload-artifact@v4
@@ -123,6 +127,10 @@ jobs:
             "${{ secrets.SLACK_WEBHOOK_URL }}"
   integration-tests-single-region:
     runs-on: ubuntu-latest-4-core
+    strategy:
+      fail-fast: false
+      matrix:
+        provider: [kind, gcp]
     timeout-minutes: 90
     permissions:
       contents: read
@@ -150,11 +158,11 @@ jobs:
           gcloud components install gke-gcloud-auth-plugin
       - name: Run tests (single-region)
         env:
-          isNightly: true
+          PROVIDER: ${{ matrix.provider }}
           USE_GKE_GCLOUD_AUTH_PLUGIN: True
         run: |
-          set -e
-          make test/nightly-e2e/single-region | tee test_output.log || true
+          set -euo pipefail
+          make test/nightly-e2e/single-region | tee test_output.log
       - name: Archive test results
         if: ${{ always() }}
         uses: actions/upload-artifact@v4

Makefile

@@ -4,6 +4,7 @@ ifeq ($(UNAME_S),Linux)
   COCKROACH_BIN ?= https://binaries.cockroachdb.com/cockroach-v23.2.0.linux-amd64.tgz
   HELM_BIN ?= https://get.helm.sh/helm-v3.14.0-linux-amd64.tar.gz
   K3D_BIN ?=  https://github.com/k3d-io/k3d/releases/download/v5.7.4/k3d-linux-amd64
+  KIND_BIN ?= https://kind.sigs.k8s.io/dl/v0.29.0/kind-linux-amd64
   KUBECTL_BIN ?= https://dl.k8s.io/release/v1.29.1/bin/linux/amd64/kubectl
   YQ_BIN ?= https://github.com/mikefarah/yq/releases/download/v4.31.2/yq_linux_amd64
   JQ_BIN ?= https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64
@@ -14,6 +15,7 @@ ifeq ($(UNAME_S),Darwin)
   COCKROACH_BIN ?= https://binaries.cockroachdb.com/cockroach-v23.2.0.darwin-10.9-amd64.tgz
   HELM_BIN ?= https://get.helm.sh/helm-v3.14.0-darwin-amd64.tar.gz
   K3D_BIN ?=  https://github.com/k3d-io/k3d/releases/download/v5.7.4/k3d-darwin-arm64
+  KIND_BIN ?= https://kind.sigs.k8s.io/dl/v0.29.0/kind-darwin-arm64
   KUBECTL_BIN ?= https://dl.k8s.io/release/v1.29.1/bin/darwin/amd64/kubectl
   YQ_BIN ?= https://github.com/mikefarah/yq/releases/download/v4.31.2/yq_darwin_amd64
   JQ_BIN ?= https://github.com/stedolan/jq/releases/download/jq-1.6/jq-osx-amd64
@@ -135,10 +137,10 @@ test/single-cluster/up: bin/k3d
 test/multi-cluster/down: bin/k3d
 	 ./tests/k3d/dev-multi-cluster.sh down
 
-test/nightly-e2e/single-region: bin/cockroach bin/kubectl bin/helm build/self-signer
+test/nightly-e2e/single-region: bin/cockroach bin/kubectl bin/helm build/self-signer bin/kind
 	@PATH="$(PWD)/bin:${PATH}" go test -timeout 60m -v -test.run TestOperatorInSingleRegion ./tests/e2e/operator/singleRegion/... || (echo "Single region tests failed with exit code $$?" && exit 1)
 
-test/nightly-e2e/multi-region: bin/cockroach bin/kubectl bin/helm build/self-signer
+test/nightly-e2e/multi-region: bin/cockroach bin/kubectl bin/helm build/self-signer bin/kind
 	@PATH="$(PWD)/bin:${PATH}" go test -timeout 60m -v -test.run TestOperatorInMultiRegion ./tests/e2e/operator/multiRegion/... || (echo "Multi region tests failed with exit code $$?" && exit 1)
 
 
@@ -176,6 +178,11 @@ bin/k3d: ## install k3d
 	@curl -Lo bin/k3d $(K3D_BIN)
 	@chmod +x bin/k3d
 
+bin/kind: ## install kind
+	@mkdir -p bin
+	@curl -Lo bin/kind $(KIND_BIN)
+	@chmod +x bin/kind
+
 bin/kubectl: ## install kubectl
 	@mkdir -p bin
 	@curl -Lo bin/kubectl $(KUBECTL_BIN)

cockroachdb-parent/charts/operator/templates/operator.yaml

@@ -254,6 +254,7 @@ rules:
       - admissionregistration.k8s.io
     resources:
       - validatingwebhookconfigurations
+      - mutatingwebhookconfigurations
     verbs:
       - create
   - apiGroups:
@@ -265,6 +266,15 @@ rules:
     verbs:
       - get
       - patch
+  - apiGroups:
+      - admissionregistration.k8s.io
+    resources:
+      - mutatingwebhookconfigurations
+    resourceNames:
+      - cockroach-mutating-webhook-config
+    verbs:
+      - get
+      - patch
   # The "create" verb cannot be qualified with resourceNames, so grant the
   # unqualified permission so that the operator can create new CRDs. But only
   # allow the operator to get and patch its own CRDs.

cockroachdb-parent/charts/operator/values.yaml

@@ -4,13 +4,13 @@
 # image captures the container image settings for Operator pods.
 image:
   # registry is the container registry where the image is stored.
-  registry: "us-docker.pkg.dev/releases-prod/self-hosted"
+  registry: "us-docker.pkg.dev/cockroach-cloud-images/development"
   # repository defines the image repository.
-  repository: "cockroachdb-operator@sha256"
+  repository: "cockroach-operator@sha256"
   # pullPolicy specifies the image pull policy.
   pullPolicy: IfNotPresent
   # tag is the image tag.
-  tag: "6f62639e9fee99d99b0387a9dccda84daa1a489b592b008f2f354ec57eae09ac"
+  tag: "72844b85354fd55b9a487abbd6e253b7d5081f65513c3813fde0ceb7d3ee2f70"
 # certificate defines the certificate settings for the Operator.
 certificate:
   # validForDays specifies the number of days the certificate is valid for.

tests/e2e/migrate/helm_chart_to_cockroach_enterprise_operator_test.go

@@ -112,7 +112,7 @@ func (h *HelmChartToOperator) TestDefaultMigration(t *testing.T) {
 		k8s.RunKubectl(t, kubectlOptions, "delete", "priorityclass", "crdb-critical")
 	}()
 
-	operator.InstallCockroachDBEnterpriseOperator(t, kubectlOptions)
+	operator.InstallCockroachDBEnterpriseOperator(t, kubectlOptions, nil)
 	defer func() {
 		t.Log("Uninstall the cockroachdb enterprise operator")
 		operator.UninstallCockroachDBEnterpriseOperator(t, kubectlOptions)
@@ -221,7 +221,7 @@ func (h *HelmChartToOperator) TestCertManagerMigration(t *testing.T) {
 		k8s.RunKubectl(t, kubectlOptions, "delete", "priorityclass", "crdb-critical")
 	}()
 
-	operator.InstallCockroachDBEnterpriseOperator(t, kubectlOptions)
+	operator.InstallCockroachDBEnterpriseOperator(t, kubectlOptions, nil)
 	defer func() {
 		t.Log("Uninstall the cockroachdb enterprise operator")
 		operator.UninstallCockroachDBEnterpriseOperator(t, kubectlOptions)
@@ -308,7 +308,7 @@ func (h *HelmChartToOperator) TestPCRPrimaryMigration(t *testing.T) {
 		k8s.RunKubectl(t, kubectlOptions, "delete", "priorityclass", "crdb-critical")
 	}()
 
-	operator.InstallCockroachDBEnterpriseOperator(t, kubectlOptions)
+	operator.InstallCockroachDBEnterpriseOperator(t, kubectlOptions, nil)
 	defer func() {
 		t.Log("Uninstall the cockroachdb enterprise operator")
 		operator.UninstallCockroachDBEnterpriseOperator(t, kubectlOptions)

tests/e2e/migrate/public_operator_to_cockroach_enterprise_operator_test.go

@@ -127,7 +127,7 @@ func (o *PublicOperatorToCockroachEnterpriseOperator) TestDefaultMigration(t *te
 	k8s.KubectlApply(t, kubectlOptions, filepath.Join(manifestsDirPath, "rbac.yaml"))
 
 	t.Log("Install the cockroachdb enterprise operator")
-	operator.InstallCockroachDBEnterpriseOperator(t, kubectlOptions)
+	operator.InstallCockroachDBEnterpriseOperator(t, kubectlOptions, nil)
 	defer func() {
 		t.Log("Uninstall the cockroachdb enterprise operator")
 		operator.UninstallCockroachDBEnterpriseOperator(t, kubectlOptions)

tests/e2e/operator/infra/common.go

@@ -18,14 +18,18 @@ import (
 
 // Provider types.
 const (
-	ProviderK3D = "k3d"
-	ProviderGCP = "gcp"
+	ProviderK3D  = "k3d"
+	ProviderKind = "kind"
+	ProviderGCP  = "gcp"
 )
 
 // Common constants.
 const (
-	defaultRetries        = 30
-	defaultRetryInterval  = 10 * time.Second
+	defaultRetries       = 30
+	defaultRetryInterval = 10 * time.Second
+	// Load balancer specific retry settings (extended for AWS)
+	loadBalancerRetries   = 60 // 10 minutes total
+	loadBalancerInterval  = 10 * time.Second
 	coreDNSDeploymentName = "coredns"
 	coreDNSServiceName    = "crl-core-dns"
 	coreDNSNamespace      = "kube-system"
@@ -46,8 +50,9 @@ const (
 
 // RegionCodes maps provider types to their region codes
 var RegionCodes = map[string][]string{
-	ProviderK3D: {"us-east1", "us-east2"},
-	ProviderGCP: {"us-central1", "us-east1"},
+	ProviderK3D:  {"us-east1", "us-east2"},
+	ProviderKind: {"us-east1", "us-east2"},
+	ProviderGCP:  {"us-central1", "us-east1"},
 }
 
 // LoadBalancerAnnotations contains provider-specific service annotations.
@@ -57,7 +62,8 @@ var LoadBalancerAnnotations = map[string]map[string]string{
 		"networking.gke.io/load-balancer-type":                         "Internal",
 		"cloud.google.com/load-balancer-type":                          "Internal",
 	},
-	ProviderK3D: {},
+	ProviderK3D:  {},
+	ProviderKind: {},
 }
 
 // NetworkConfigs defines standard network configurations for each provider and region.
@@ -237,7 +243,7 @@ func finalizeCoreDNSDeployment(t *testing.T, kubectlOpts *k8s.KubectlOptions) er
 func WaitForCoreDNSServiceIPs(t *testing.T, kubectlOpts *k8s.KubectlOptions) ([]string, error) {
 	var ips []string
 
-	_, err := retry.DoWithRetryE(t, "waiting for CoreDNS service IPs", defaultRetries, defaultRetryInterval,
+	_, err := retry.DoWithRetryE(t, "waiting for CoreDNS service IPs", loadBalancerRetries, loadBalancerInterval,
 		func() (string, error) {
 			svc, err := k8s.GetServiceE(t, kubectlOpts, coreDNSServiceName)
 			if err != nil {