Skip to content

Commit 2777e54

Browse files
NishanthNalluriNishanthNalluri
committed
Nightly Integration tests for self-hosted infrastructure.
1 parent 2cc6e92 commit 2777e54

File tree

10 files changed

+579
-73
lines changed

10 files changed

+579
-73
lines changed

‎Makefile‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ ifeq ($(UNAME_S),Linux)
44
COCKROACH_BIN ?= https://binaries.cockroachdb.com/cockroach-v23.2.0.linux-amd64.tgz
55
HELM_BIN ?= https://get.helm.sh/helm-v3.14.0-linux-amd64.tar.gz
66
K3D_BIN ?= https://github.com/k3d-io/k3d/releases/download/v5.7.4/k3d-linux-amd64
7+
KIND_BIN ?= https://kind.sigs.k8s.io/dl/v0.29.0/kind-linux-amd64
78
KUBECTL_BIN ?= https://dl.k8s.io/release/v1.29.1/bin/linux/amd64/kubectl
89
YQ_BIN ?= https://github.com/mikefarah/yq/releases/download/v4.31.2/yq_linux_amd64
910
JQ_BIN ?= https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64
@@ -14,6 +15,7 @@ ifeq ($(UNAME_S),Darwin)
1415
COCKROACH_BIN ?= https://binaries.cockroachdb.com/cockroach-v23.2.0.darwin-10.9-amd64.tgz
1516
HELM_BIN ?= https://get.helm.sh/helm-v3.14.0-darwin-amd64.tar.gz
1617
K3D_BIN ?= https://github.com/k3d-io/k3d/releases/download/v5.7.4/k3d-darwin-arm64
18+
KIND_BIN ?= https://kind.sigs.k8s.io/dl/v0.29.0/kind-darwin-arm64
1719
KUBECTL_BIN ?= https://dl.k8s.io/release/v1.29.1/bin/darwin/amd64/kubectl
1820
YQ_BIN ?= https://github.com/mikefarah/yq/releases/download/v4.31.2/yq_darwin_amd64
1921
JQ_BIN ?= https://github.com/stedolan/jq/releases/download/jq-1.6/jq-osx-amd64
@@ -135,10 +137,10 @@ test/single-cluster/up: bin/k3d
135137
test/multi-cluster/down: bin/k3d
136138
./tests/k3d/dev-multi-cluster.sh down
137139

138-
test/nightly-e2e/single-region: bin/cockroach bin/kubectl bin/helm build/self-signer
140+
test/nightly-e2e/single-region: bin/cockroach bin/kubectl bin/helm build/self-signer bin/kind
139141
@PATH="$(PWD)/bin:${PATH}" go test -timeout 60m -v -test.run TestOperatorInSingleRegion ./tests/e2e/operator/singleRegion/... || (echo "Single region tests failed with exit code $$?" && exit 1)
140142

141-
test/nightly-e2e/multi-region: bin/cockroach bin/kubectl bin/helm build/self-signer
143+
test/nightly-e2e/multi-region: bin/cockroach bin/kubectl bin/helm build/self-signer bin/kind
142144
@PATH="$(PWD)/bin:${PATH}" go test -timeout 60m -v -test.run TestOperatorInMultiRegion ./tests/e2e/operator/multiRegion/... || (echo "Multi region tests failed with exit code $$?" && exit 1)
143145

144146

@@ -176,6 +178,11 @@ bin/k3d: ## install k3d
176178
@curl -Lo bin/k3d $(K3D_BIN)
177179
@chmod +x bin/k3d
178180

181+
bin/kind: ## install kind
182+
@mkdir -p bin
183+
@curl -Lo bin/kind $(KIND_BIN)
184+
@chmod +x bin/kind
185+
179186
bin/kubectl: ## install kubectl
180187
@mkdir -p bin
181188
@curl -Lo bin/kubectl $(KUBECTL_BIN)

‎tests/e2e/migrate/helm_chart_to_cockroach_enterprise_operator_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,7 @@ func (h *HelmChartToOperator) TestDefaultMigration(t *testing.T) {
111111
k8s.RunKubectl(t, kubectlOptions, "delete", "priorityclass", "crdb-critical")
112112
}()
113113

114-
operator.InstallCockroachDBEnterpriseOperator(t, kubectlOptions)
114+
operator.InstallCockroachDBEnterpriseOperator(t, kubectlOptions, nil)
115115
defer func() {
116116
t.Log("Uninstall the cockroachdb enterprise operator")
117117
operator.UninstallCockroachDBEnterpriseOperator(t, kubectlOptions)
@@ -220,7 +220,7 @@ func (h *HelmChartToOperator) TestCertManagerMigration(t *testing.T) {
220220
k8s.RunKubectl(t, kubectlOptions, "delete", "priorityclass", "crdb-critical")
221221
}()
222222

223-
operator.InstallCockroachDBEnterpriseOperator(t, kubectlOptions)
223+
operator.InstallCockroachDBEnterpriseOperator(t, kubectlOptions, nil)
224224
defer func() {
225225
t.Log("Uninstall the cockroachdb enterprise operator")
226226
operator.UninstallCockroachDBEnterpriseOperator(t, kubectlOptions)

‎tests/e2e/migrate/public_operator_to_cockroach_enterprise_operator_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,7 @@ func (o *PublicOperatorToCockroachEnterpriseOperator) TestDefaultMigration(t *te
119119
k8s.KubectlApply(t, kubectlOptions, filepath.Join(manifestsDirPath, "rbac.yaml"))
120120

121121
t.Log("Install the cockroachdb enterprise operator")
122-
operator.InstallCockroachDBEnterpriseOperator(t, kubectlOptions)
122+
operator.InstallCockroachDBEnterpriseOperator(t, kubectlOptions, nil)
123123
defer func() {
124124
t.Log("Uninstall the cockroachdb enterprise operator")
125125
operator.UninstallCockroachDBEnterpriseOperator(t, kubectlOptions)

‎tests/e2e/operator/infra/common.go‎

Lines changed: 21 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -18,14 +18,19 @@ import (
1818

1919
// Provider types.
2020
const (
21-
ProviderK3D = "k3d"
22-
ProviderGCP = "gcp"
21+
ProviderK3D = "k3d"
22+
ProviderKind = "kind"
23+
ProviderGCP = "gcp"
24+
ProviderAWS = "aws"
2325
)
2426

2527
// Common constants.
2628
const (
27-
defaultRetries = 30
28-
defaultRetryInterval = 10 * time.Second
29+
defaultRetries = 30
30+
defaultRetryInterval = 10 * time.Second
31+
// Load balancer specific retry settings (extended for AWS)
32+
loadBalancerRetries = 60 // 10 minutes total
33+
loadBalancerInterval = 10 * time.Second
2934
coreDNSDeploymentName = "coredns"
3035
coreDNSServiceName = "crl-core-dns"
3136
coreDNSNamespace = "kube-system"
@@ -38,16 +43,22 @@ const (
3843
DefaultVPCCIDR = "172.28.0.0/16"
3944

4045
// Instance types for different cloud providers.
41-
gcpDefaultMachineType = "e2-standard-4"
46+
gcpDefaultMachineType = "e2-standard-4"
47+
AWSDefaultInstanceType = "m5.large"
4248

4349
// Default node counts
4450
defaultNodesPerZone = 1
51+
DefaultNodeCount = int32(3)
52+
DefaultMinNodeCount = int32(3)
53+
DefaultMaxNodeCount = int32(4)
4554
)
4655

4756
// RegionCodes maps provider types to their region codes
4857
var RegionCodes = map[string][]string{
49-
ProviderK3D: {"us-east1", "us-east2"},
50-
ProviderGCP: {"us-central1", "us-east1"},
58+
ProviderK3D: {"us-east1", "us-east2"},
59+
ProviderKind: {"us-east1", "us-east2"},
60+
ProviderGCP: {"us-central1", "us-east1"},
61+
ProviderAWS: {"us-east-1", "us-east-2", "us-west-1"},
5162
}
5263

5364
// LoadBalancerAnnotations contains provider-specific service annotations.
@@ -57,7 +68,8 @@ var LoadBalancerAnnotations = map[string]map[string]string{
5768
"networking.gke.io/load-balancer-type": "Internal",
5869
"cloud.google.com/load-balancer-type": "Internal",
5970
},
60-
ProviderK3D: {},
71+
ProviderK3D: {},
72+
ProviderKind: {},
6173
}
6274

6375
// NetworkConfigs defines standard network configurations for each provider and region.
@@ -237,7 +249,7 @@ func finalizeCoreDNSDeployment(t *testing.T, kubectlOpts *k8s.KubectlOptions) er
237249
func WaitForCoreDNSServiceIPs(t *testing.T, kubectlOpts *k8s.KubectlOptions) ([]string, error) {
238250
var ips []string
239251

240-
_, err := retry.DoWithRetryE(t, "waiting for CoreDNS service IPs", defaultRetries, defaultRetryInterval,
252+
_, err := retry.DoWithRetryE(t, "waiting for CoreDNS service IPs", loadBalancerRetries, loadBalancerInterval,
241253
func() (string, error) {
242254
svc, err := k8s.GetServiceE(t, kubectlOpts, coreDNSServiceName)
243255
if err != nil {

0 commit comments

Comments
 (0)