diff --git a/cmd/dartboard/subcommands/apply.go b/cmd/dartboard/subcommands/apply.go index d68847a3f..ecb84658f 100644 --- a/cmd/dartboard/subcommands/apply.go +++ b/cmd/dartboard/subcommands/apply.go @@ -27,6 +27,7 @@ func Apply(cli *cli.Context) error { if err = tf.PrintVersion(cli.Context); err != nil { return err } + if err = tf.Apply(cli.Context); err != nil { return err } diff --git a/cmd/dartboard/subcommands/deploy.go b/cmd/dartboard/subcommands/deploy.go index 185eaa503..bf0342baa 100644 --- a/cmd/dartboard/subcommands/deploy.go +++ b/cmd/dartboard/subcommands/deploy.go @@ -50,6 +50,7 @@ func Deploy(cli *cli.Context) error { if err = tf.PrintVersion(cli.Context); err != nil { return err } + if err = tf.Apply(cli.Context); err != nil { return err } @@ -66,25 +67,31 @@ func Deploy(cli *cli.Context) error { if err = chartInstall(tester.Kubeconfig, chart{"k6-files", "tester", "k6-files"}, nil); err != nil { return err } + if err = chartInstall(tester.Kubeconfig, chart{"mimir", "tester", "mimir"}, nil); err != nil { return err } + if err = chartInstall(tester.Kubeconfig, chart{"grafana-dashboards", "tester", "grafana-dashboards"}, nil); err != nil { return err } + if err = chartInstallGrafana(r, &tester); err != nil { return err } upstream := clusters["upstream"] rancherVersion := r.ChartVariables.RancherVersion + rancherImageTag := "v" + rancherVersion if r.ChartVariables.RancherImageTagOverride != "" { rancherImageTag = r.ChartVariables.RancherImageTagOverride + image := "rancher/rancher" if r.ChartVariables.RancherImageOverride != "" { image = r.ChartVariables.RancherImageOverride } + err = importImageIntoK3d(tf, image+":"+rancherImageTag, upstream) if err != nil { return err @@ -94,12 +101,15 @@ func Deploy(cli *cli.Context) error { if err = chartInstallCertManager(r, &upstream); err != nil { return err } + if err = chartInstallRancher(r, rancherImageTag, &upstream); err != nil { return err } + if err = chartInstallRancherIngress(&upstream); err != nil { return err } + if err = chartInstallCgroupsExporter(&upstream); err != nil { return err } @@ -108,9 +118,11 @@ func Deploy(cli *cli.Context) error { if err = kubectl.WaitRancher(upstream.Kubeconfig); err != nil { return err } + if err = chartInstallRancherMonitoring(r, &upstream); err != nil { return err } + if err = importDownstreamClusters(r, rancherImageTag, tf, clusters); err != nil { return err } @@ -135,6 +147,7 @@ func chartInstall(kubeConf string, chart chart, vals map[string]any, extraArgs . if err = helm.Install(kubeConf, path, name, namespace, vals, extraArgs...); err != nil { return fmt.Errorf("chart %s: %w", name, err) } + return nil } @@ -163,6 +176,7 @@ func chartInstallCertManager(r *dart.Dart, cluster *tofu.Cluster) error { namespace: "cert-manager", path: fmt.Sprintf("https://charts.jetstack.io/charts/cert-manager-v%s.tgz", r.ChartVariables.CertManagerVersion), } + return chartInstall(cluster.Kubeconfig, chartCertManager, map[string]any{"installCRDs": true}) } @@ -185,7 +199,6 @@ func chartInstallRancher(r *dart.Dart, rancherImageTag string, cluster *tofu.Clu if r.ChartVariables.ForcePrimeRegistry { rancherRepo = "https://charts.rancher.com/server-charts/prime" } - } chartRancher := chart{ @@ -198,10 +211,12 @@ func chartInstallRancher(r *dart.Dart, rancherImageTag string, cluster *tofu.Clu if err != nil { return fmt.Errorf("chart %s: %w", chartRancher.name, err) } + rancherClusterName := clusterAdd.Public.Name rancherClusterURL := clusterAdd.Public.HTTPSURL var extraEnv []map[string]any + extraEnv = []map[string]any{ { "name": "CATTLE_SERVER_URL", @@ -221,6 +236,7 @@ func chartInstallRancher(r *dart.Dart, rancherImageTag string, cluster *tofu.Clu chartVals := getRancherValsJSON(r.ChartVariables.RancherImageOverride, rancherImageTag, r.ChartVariables.AdminPassword, rancherClusterName, extraEnv, r.ChartVariables.RancherReplicas) var extraArgs []string + if r.ChartVariables.RancherValues != "" { p, err := writeValuesFile(r.ChartVariables.RancherValues) if err != nil { @@ -245,9 +261,11 @@ func writeValuesFile(content string) (string, error) { if err != nil { return "", err } + if _, err := io.WriteString(p, content); err != nil { return "", err } + return p.Name(), nil } @@ -267,6 +285,7 @@ func chartInstallRancherIngress(cluster *tofu.Cluster) error { if len(clusterAdd.Local.Name) > 0 { sans = append(sans, clusterAdd.Local.Name) } + if len(clusterAdd.Public.Name) > 0 { sans = append(sans, clusterAdd.Public.Name) } @@ -281,7 +300,9 @@ func chartInstallRancherIngress(cluster *tofu.Cluster) error { func chartInstallRancherMonitoring(r *dart.Dart, cluster *tofu.Cluster) error { rancherMinorVersion := strings.Join(strings.Split(r.ChartVariables.RancherVersion, ".")[0:2], ".") + const chartPrefix = "https://github.com/rancher/charts/raw/release-v" + chartPath := fmt.Sprintf("%s%s", chartPrefix, rancherMinorVersion) if len(r.ChartVariables.RancherAppsRepoOverride) > 0 { @@ -305,6 +326,7 @@ func chartInstallRancherMonitoring(r *dart.Dart, cluster *tofu.Cluster) error { }, "systemDefaultRegistry": "", } + err := chartInstall(cluster.Kubeconfig, chartRancherMonitoringCRD, chartVals) if err != nil { return err @@ -321,6 +343,7 @@ func chartInstallRancherMonitoring(r *dart.Dart, cluster *tofu.Cluster) error { if err != nil { return fmt.Errorf("chart %s: %w", chartRancherMonitoring.name, err) } + mimirURL := clusterAdd.Public.HTTPURL + "/mimir/api/v1/push" chartVals = getRancherMonitoringValsJSON(cluster.ReserveNodeForMonitoring, mimirURL) @@ -333,12 +356,13 @@ func chartInstallCgroupsExporter(cluster *tofu.Cluster) error { } func getRancherMonitoringValsJSON(reserveNodeForMonitoring bool, mimirURL string) map[string]any { - nodeSelector := map[string]any{} tolerations := []any{} monitoringRestrictions := map[string]any{} + if reserveNodeForMonitoring { nodeSelector["monitoring"] = "true" + tolerations = append(tolerations, map[string]any{"key": "monitoring", "operator": "Exists", "effect": "NoSchedule"}) monitoringRestrictions["nodeSelector"] = nodeSelector monitoringRestrictions["tolerations"] = tolerations @@ -488,7 +512,6 @@ func getRancherValsJSON(rancherImageOverride, rancherImageTag, bootPwd, hostname } func importDownstreamClusters(r *dart.Dart, rancherImageTag string, tf *tofu.Tofu, clusters map[string]tofu.Cluster) error { - log.Print("Import downstream clusters") if err := importDownstreamClustersRancherSetup(r, clusters); err != nil { @@ -504,7 +527,9 @@ func importDownstreamClusters(r *dart.Dart, rancherImageTag string, tf *tofu.Tof if !strings.HasPrefix(clusterName, "downstream") { continue } + clustersCount++ + go importDownstreamClusterDo(r, rancherImageTag, tf, clusters, clusterName, clustersChan, errorChan) } @@ -512,11 +537,13 @@ func importDownstreamClusters(r *dart.Dart, rancherImageTag string, tf *tofu.Tof if clustersCount == 0 { return nil } + select { case err := <-errorChan: return err case completed := <-clustersChan: log.Printf("Cluster %q imported successfully.\n", completed) + clustersCount-- } } @@ -524,6 +551,7 @@ func importDownstreamClusters(r *dart.Dart, rancherImageTag string, tf *tofu.Tof func importDownstreamClusterDo(r *dart.Dart, rancherImageTag string, tf *tofu.Tofu, clusters map[string]tofu.Cluster, clusterName string, ch chan<- string, errCh chan<- error) { log.Print("Import cluster " + clusterName) + yamlFile, err := os.CreateTemp("", "scli-"+clusterName+"-*.yaml") if err != nil { errCh <- fmt.Errorf("%s import failed: %w", clusterName, err) @@ -542,8 +570,10 @@ func importDownstreamClusterDo(r *dart.Dart, rancherImageTag string, tf *tofu.To if !ok { err := fmt.Errorf("error: cannot find access data for cluster %q", clusterName) errCh <- fmt.Errorf("%s import failed: %w", clusterName, err) + return } + if r.ChartVariables.RancherImageTagOverride != "" { err = importImageIntoK3d(tf, "rancher/rancher-agent:"+rancherImageTag, downstream) if err != nil { @@ -562,6 +592,7 @@ func importDownstreamClusterDo(r *dart.Dart, rancherImageTag string, tf *tofu.To errCh <- fmt.Errorf("%s import failed: %w", clusterName, err) return } + if err := kubectl.WaitForReadyCondition(clusters["upstream"].Kubeconfig, "cluster.fleet.cattle.io", clusterName, "fleet-default", "ready", 10); err != nil { errCh <- fmt.Errorf("%s import failed: %w", clusterName, err) @@ -573,29 +604,34 @@ func importDownstreamClusterDo(r *dart.Dart, rancherImageTag string, tf *tofu.To errCh <- fmt.Errorf("%s waiting for rancher-webhook failed: %w", clusterName, err) return } + if r.ChartVariables.DownstreamRancherMonitoring { if err := chartInstallRancherMonitoring(r, &downstream); err != nil { errCh <- fmt.Errorf("downstream monitoring installation on cluster %s failed: %w", clusterName, err) return } } + ch <- clusterName } func importDownstreamClustersRancherSetup(r *dart.Dart, clusters map[string]tofu.Cluster) error { tester := clusters["tester"] upstream := clusters["upstream"] + upstreamAdd, err := getAppAddressFor(upstream) if err != nil { return err } downstreamClusters := []string{} + for clusterName := range clusters { if strings.HasPrefix(clusterName, "downstream") { downstreamClusters = append(downstreamClusters, clusterName) } } + importedClusterNames := strings.Join(downstreamClusters, ",") envVars := map[string]string{ @@ -608,6 +644,7 @@ func importDownstreamClustersRancherSetup(r *dart.Dart, clusters map[string]tofu if err = kubectl.K6run(tester.Kubeconfig, "rancher/rancher_setup.js", envVars, nil, true, upstreamAdd.Local.HTTPSURL, false); err != nil { return err } + return nil } @@ -615,45 +652,50 @@ func importClustersDownstreamGetYAML(clusters map[string]tofu.Cluster, name stri var status map[string]interface{} upstream := clusters["upstream"] + upstreamAdd, err := getAppAddressFor(upstream) if err != nil { - return + return clusterID, err } if status, err = kubectl.GetStatus(upstream.Kubeconfig, "clusters.provisioning.cattle.io", name, "fleet-default"); err != nil { - return + return clusterID, err } + clusterID, ok := status["clusterName"].(string) if !ok { err = fmt.Errorf("error accessing fleet-default/%s clusters: no valid 'clusterName' in 'Status'", name) - return + return clusterID, err } if status, err = kubectl.GetStatus(upstream.Kubeconfig, "clusterregistrationtokens.management.cattle.io", "default-token", clusterID); err != nil { - return + return clusterID, err } + token, ok := status["token"].(string) if !ok { err = fmt.Errorf("error accessing %s/default-token clusterregistrationtokens: no valid 'token' in 'Status'", clusterID) - return + return clusterID, err } url := fmt.Sprintf("%s/v3/import/%s_%s.yaml", upstreamAdd.Local.HTTPSURL, token, clusterID) tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} client := &http.Client{Transport: tr} + resp, err := client.Get(url) if err != nil { - return + return clusterID, err } defer resp.Body.Close() _, err = io.Copy(yamlFile, resp.Body) if err != nil { - return + return clusterID, err } + if err = yamlFile.Sync(); err != nil { - return + return clusterID, err } - return + return clusterID, err } diff --git a/cmd/dartboard/subcommands/getaccess.go b/cmd/dartboard/subcommands/getaccess.go index 602b4b51a..0bb82b2f6 100644 --- a/cmd/dartboard/subcommands/getaccess.go +++ b/cmd/dartboard/subcommands/getaccess.go @@ -39,6 +39,7 @@ func GetAccess(cli *cli.Context) error { tester := clusters["tester"] downstreams := make(map[string]tofu.Cluster) + for k, v := range clusters { if strings.HasPrefix(k, "downstream") { downstreams[k] = v @@ -46,6 +47,7 @@ func GetAccess(cli *cli.Context) error { } upstreamAddresses, err := getAppAddressFor(upstream) + rancherURL := "" if err == nil { rancherURL = upstreamAddresses.Local.HTTPSURL @@ -57,9 +59,11 @@ func GetAccess(cli *cli.Context) error { fmt.Println() printAccessDetails(r, "UPSTREAM", upstream, rancherURL) + for name, downstream := range downstreams { printAccessDetails(r, strings.ToUpper(name), downstream, "") } + printAccessDetails(r, "TESTER", tester, "") return nil diff --git a/cmd/dartboard/subcommands/load.go b/cmd/dartboard/subcommands/load.go index 2edea9e49..7832e3fb1 100644 --- a/cmd/dartboard/subcommands/load.go +++ b/cmd/dartboard/subcommands/load.go @@ -54,6 +54,7 @@ func Load(cli *cli.Context) error { if clusterName != "upstream" && !strings.HasPrefix(clusterName, "downstream") { continue } + if err := loadConfigMapAndSecrets(r, tester.Kubeconfig, clusterName, clusterData); err != nil { return err } @@ -67,6 +68,7 @@ func Load(cli *cli.Context) error { if err := loadProjects(r, tester.Kubeconfig, "upstream", clusters["upstream"]); err != nil { return err } + return nil } @@ -89,19 +91,23 @@ func loadConfigMapAndSecrets(r *dart.Dart, kubeconfig string, clusterName string } log.Printf("Load resources on cluster %q (#ConfigMaps: %s, #Secrets: %s)\n", clusterName, configMapCount, secretCount) + if err := kubectl.K6run(kubeconfig, "generic/create_k8s_resources.js", envVars, tags, true, clusterData.KubernetesAddresses.Tunnel, false); err != nil { return fmt.Errorf("failed loading ConfigMaps and Secrets on cluster %q: %w", clusterName, err) } + return nil } func loadRolesAndUsers(r *dart.Dart, kubeconfig string, clusterName string, clusterData tofu.Cluster) error { roleCount := strconv.Itoa(r.TestVariables.TestRoles) userCount := strconv.Itoa(r.TestVariables.TestUsers) + clusterAdd, err := getAppAddressFor(clusterData) if err != nil { return fmt.Errorf("failed loading Roles and Users on cluster %q: %w", clusterName, err) } + envVars := map[string]string{ "BASE_URL": clusterAdd.Public.HTTPSURL, "USERNAME": "admin", @@ -122,15 +128,18 @@ func loadRolesAndUsers(r *dart.Dart, kubeconfig string, clusterName string, clus if err := kubectl.K6run(kubeconfig, "generic/create_roles_users.js", envVars, tags, true, clusterAdd.Local.HTTPSURL, false); err != nil { return fmt.Errorf("failed loading Roles and Users on cluster %q: %w", clusterName, err) } + return nil } func loadProjects(r *dart.Dart, kubeconfig string, clusterName string, clusterData tofu.Cluster) error { projectCount := strconv.Itoa(r.TestVariables.TestProjects) + clusterAdd, err := getAppAddressFor(clusterData) if err != nil { return fmt.Errorf("failed loading Projects on cluster %q: %w", clusterName, err) } + envVars := map[string]string{ "BASE_URL": clusterAdd.Public.HTTPSURL, "USERNAME": "admin", @@ -148,5 +157,6 @@ func loadProjects(r *dart.Dart, kubeconfig string, clusterName string, clusterDa if err := kubectl.K6run(kubeconfig, "generic/create_projects.js", envVars, tags, true, clusterAdd.Local.HTTPSURL, false); err != nil { return fmt.Errorf("failed loading Projects on cluster %q: %w", clusterName, err) } + return nil } diff --git a/cmd/dartboard/subcommands/utils.go b/cmd/dartboard/subcommands/utils.go index 65e5b11cf..aa9853f83 100644 --- a/cmd/dartboard/subcommands/utils.go +++ b/cmd/dartboard/subcommands/utils.go @@ -48,10 +48,12 @@ type clusterAddresses struct { // prepare prepares tofu for execution and parses a dart file from the command line context func prepare(cli *cli.Context) (*tofu.Tofu, *dart.Dart, error) { dartPath := cli.String(ArgDart) + d, err := dart.Parse(dartPath) if err != nil { return nil, nil, err } + fmt.Printf("Using dart: %s\n", dartPath) fmt.Printf("OpenTofu main directory: %s\n", d.TofuMainDirectory) fmt.Printf("Using Tofu workspace: %s\n", d.TofuWorkspace) @@ -65,21 +67,26 @@ func prepare(cli *cli.Context) (*tofu.Tofu, *dart.Dart, error) { if err != nil { return nil, nil, err } + return tf, d, nil } // printAccessDetails prints to console addresses and kubeconfig file paths of a cluster for user convenience func printAccessDetails(r *dart.Dart, name string, cluster tofu.Cluster, rancherURL string) { fmt.Printf("*** %s CLUSTER\n", name) + if rancherURL != "" { fmt.Printf(" Rancher UI: %s (admin/%s)\n", rancherURL, r.ChartVariables.AdminPassword) } + fmt.Println(" Kubernetes API:") fmt.Printf("export KUBECONFIG=%q\n", cluster.Kubeconfig) fmt.Printf("kubectl config use-context %q\n", cluster.Context) + for node, command := range cluster.NodeAccessCommands { fmt.Printf(" Node %s: %q\n", node, command) } + fmt.Println() } @@ -101,6 +108,7 @@ func getAppAddressFor(cluster tofu.Cluster) (clusterAddresses, error) { localNetworkName = loadBalancerName } } + localNetworkHTTPPort := add.Tunnel.HTTPPort if localNetworkHTTPPort == 0 { localNetworkHTTPPort = add.Public.HTTPPort @@ -108,6 +116,7 @@ func getAppAddressFor(cluster tofu.Cluster) (clusterAddresses, error) { localNetworkHTTPPort = 80 } } + localNetworkHTTPSPort := add.Tunnel.HTTPSPort if localNetworkHTTPSPort == 0 { localNetworkHTTPSPort = add.Public.HTTPSPort @@ -125,6 +134,7 @@ func getAppAddressFor(cluster tofu.Cluster) (clusterAddresses, error) { clusterNetworkName = loadBalancerName } } + clusterNetworkHTTPPort := add.Public.HTTPPort if clusterNetworkHTTPPort == 0 { clusterNetworkHTTPPort = add.Private.HTTPPort @@ -132,6 +142,7 @@ func getAppAddressFor(cluster tofu.Cluster) (clusterAddresses, error) { clusterNetworkHTTPPort = 80 } } + clusterNetworkHTTPSPort := add.Public.HTTPSPort if clusterNetworkHTTPSPort == 0 { clusterNetworkHTTPSPort = add.Private.HTTPSPort @@ -167,5 +178,6 @@ func importImageIntoK3d(tf *tofu.Tofu, image string, cluster tofu.Cluster) error } } } + return nil } diff --git a/internal/dart/recipe.go b/internal/dart/recipe.go index 5ff60eb06..987285266 100644 --- a/internal/dart/recipe.go +++ b/internal/dart/recipe.go @@ -13,30 +13,30 @@ import ( // Dart is a "recipe" that encodes all parameters for a test run type Dart struct { + TofuVariables map[string]any `yaml:"tofu_variables"` TofuMainDirectory string `yaml:"tofu_main_directory"` TofuWorkspace string `yaml:"tofu_workspace"` - TofuParallelism int `yaml:"tofu_parallelism"` - TofuVariables map[string]any `yaml:"tofu_variables"` ChartVariables ChartVariables `yaml:"chart_variables"` TestVariables TestVariables `yaml:"test_variables"` + TofuParallelism int `yaml:"tofu_parallelism"` } type ChartVariables struct { - RancherReplicas int `yaml:"rancher_replicas"` - DownstreamRancherMonitoring bool `yaml:"downstream_rancher_monitoring"` + RancherAppsRepoOverride string `yaml:"rancher_apps_repo_override"` + RancherMonitoringVersion string `yaml:"rancher_monitoring_version"` AdminPassword string `yaml:"admin_password"` UserPassword string `yaml:"user_password"` RancherVersion string `yaml:"rancher_version"` - ForcePrimeRegistry bool `yaml:"force_prime_registry"` - RancherAppsRepoOverride string `yaml:"rancher_apps_repo_override"` - RancherChartRepoOverride string `yaml:"rancher_chart_repo_override"` + RancherValues string `yaml:"rancher_values"` + TesterGrafanaVersion string `yaml:"tester_grafana_version"` RancherImageOverride string `yaml:"rancher_image_override"` - RancherImageTagOverride string `yaml:"rancher_image_tag_override"` - RancherMonitoringVersion string `yaml:"rancher_monitoring_version"` CertManagerVersion string `yaml:"cert_manager_version"` - TesterGrafanaVersion string `yaml:"tester_grafana_version"` - RancherValues string `yaml:"rancher_values"` + RancherImageTagOverride string `yaml:"rancher_image_tag_override"` + RancherChartRepoOverride string `yaml:"rancher_chart_repo_override"` ExtraEnvironmentVariables []map[string]any `yaml:"extra_environment_variables"` + RancherReplicas int `yaml:"rancher_replicas"` + DownstreamRancherMonitoring bool `yaml:"downstream_rancher_monitoring"` + ForcePrimeRegistry bool `yaml:"force_prime_registry"` } type TestVariables struct { @@ -73,15 +73,19 @@ func Parse(path string) (*Dart, error) { if err != nil { return nil, fmt.Errorf("failed to read dart file: %w", err) } + result := defaultDart + err = yaml.Unmarshal(bytes, &result) if err != nil { return nil, fmt.Errorf("failed to unmarshal dart file: %w", err) } + tofuVars, err := yaml.Marshal(result.TofuVariables) if err != nil { return nil, fmt.Errorf("failed to marshal recipe's tofu variables: %w", err) } + log.Printf("\nTofu variables: \n%v\n", string(tofuVars)) result.ChartVariables.RancherVersion = normalizeVersion(result.ChartVariables.RancherVersion) @@ -104,6 +108,7 @@ func needsPrime(version string) bool { major, _ := strconv.Atoi(versionSplits[0]) minor, _ := strconv.Atoi(versionSplits[1]) patch, _ := strconv.Atoi(versionSplits[2]) + return (major == 2 && minor == 7 && patch >= 11) || (major == 2 && minor == 8 && patch >= 6) } diff --git a/internal/docker/docker.go b/internal/docker/docker.go index 666be4128..5d34de264 100644 --- a/internal/docker/docker.go +++ b/internal/docker/docker.go @@ -36,25 +36,35 @@ func Images(image string) ([]string, error) { log.Printf("Exec: docker %s\n", strings.Join(args, " ")) cmd := exec.Command("docker", args...) - var outStream strings.Builder - var errStream strings.Builder + + var ( + outStream strings.Builder + errStream strings.Builder + ) + cmd.Stdout = &outStream + cmd.Stderr = &errStream if err := cmd.Run(); err != nil { return nil, fmt.Errorf("%v", errStream.String()) } lines := strings.Split(strings.TrimSpace(outStream.String()), "\n") + var images []string + for _, line := range lines { if line != "" { var img Image + err := json.Unmarshal([]byte(line), &img) if err != nil { return nil, fmt.Errorf("error unmarshalling JSON output from docker images: %w", err) } + images = append(images, img.Repository+":"+img.Tag) } } + return images, nil } diff --git a/internal/helm/helm.go b/internal/helm/helm.go index e79f2f9c3..399261764 100644 --- a/internal/helm/helm.go +++ b/internal/helm/helm.go @@ -35,22 +35,30 @@ func Install(kubecfg, chartLocation, releaseName, namespace string, vals map[str chartLocation, "--create-namespace", } + if vals != nil { valueString := "" + for k, v := range vals { jsonVal, err := json.Marshal(v) if err != nil { return err } + valueString += k + "=" + string(jsonVal) + "," } + args = append(args, "--set-json="+valueString) } + args = append(args, extraArgs...) cmd := vendored.Command("helm", args...) + var errStream strings.Builder + cmd.Stdout = os.Stdout + cmd.Stderr = &errStream if err := cmd.Run(); err != nil { return fmt.Errorf("%v", errStream.String()) diff --git a/internal/k3d/k3d.go b/internal/k3d/k3d.go index 0f754cb02..bbd87d480 100644 --- a/internal/k3d/k3d.go +++ b/internal/k3d/k3d.go @@ -28,8 +28,11 @@ func ImageImport(k3dClusterName string, image string) error { args := []string{"image", "import", "--cluster", k3dClusterName, image} cmd := vendored.Command("k3d", args...) + var errStream strings.Builder + cmd.Stdout = os.Stdout + cmd.Stderr = &errStream if err := cmd.Run(); err != nil { return fmt.Errorf("%v", errStream.String()) diff --git a/internal/kubectl/kubectl.go b/internal/kubectl/kubectl.go index 165cb3e0c..19aa305fa 100644 --- a/internal/kubectl/kubectl.go +++ b/internal/kubectl/kubectl.go @@ -75,10 +75,12 @@ var ( func collectFileEntries(root string, exts map[string]bool) ([]FileEntry, error) { // Get valid path to root and ensure it exists root = filepath.Clean(root) + info, err := os.Stat(root) if err != nil { return nil, fmt.Errorf("stat root %q: %w", root, err) } + if !info.IsDir() { return nil, fmt.Errorf("root %q is not a directory", root) } @@ -96,7 +98,9 @@ func collectFileEntries(root string, exts map[string]bool) ([]FileEntry, error) } visited := map[string]bool{} // tracks visited resolved real paths to avoid cycles + var out []FileEntry + stack := []stackEntry{{virtualRel: "", realPath: resolvedRoot}} for len(stack) > 0 { @@ -115,9 +119,11 @@ func collectFileEntries(root string, exts map[string]bool) ([]FileEntry, error) if rp, err := filepath.EvalSymlinks(absReal); err == nil { absRealResolved = rp } + if visited[absRealResolved] { continue } + visited[absRealResolved] = true entries, err := os.ReadDir(cur.realPath) @@ -139,11 +145,13 @@ func collectFileEntries(root string, exts map[string]bool) ([]FileEntry, error) log.Printf("warning: broken symlink or cannot resolve %q: %v", entryRealPath, err) continue } + stat, err := os.Stat(target) if err != nil { log.Printf("warning: cannot stat symlink target %q: %v", target, err) continue } + if stat.IsDir() { // push directory to stack to preserve virtualRel stack = append(stack, stackEntry{virtualRel: virtualRel, realPath: target}) @@ -204,6 +212,7 @@ func getCachedEntries(root string, exts map[string]bool) ([]FileEntry, error) { cacheOnce.Do(func() { cachedEntries, cacheErr = collectFileEntries(root, exts) }) + return cachedEntries, cacheErr } @@ -212,6 +221,7 @@ func Exec(kubepath string, output io.Writer, args ...string) error { cmd := vendored.Command("kubectl", fullArgs...) var errStream strings.Builder + cmd.Stderr = &errStream cmd.Stdin = os.Stdin @@ -222,6 +232,7 @@ func Exec(kubepath string, output io.Writer, args ...string) error { if err := cmd.Run(); err != nil { return fmt.Errorf("error while running kubectl with params %v: %v", fullArgs, errStream.String()) } + return nil } @@ -234,21 +245,26 @@ func WaitRancher(kubePath string) error { if err != nil { return err } + err = WaitForReadyCondition(kubePath, "deployment", "rancher-webhook", "cattle-system", "available", 3) if err != nil { return err } + err = WaitForReadyCondition(kubePath, "deployment", "fleet-controller", "cattle-fleet-system", "available", 5) + return err } func WaitForReadyCondition(kubePath, resource, name, namespace string, condition string, minutes int) error { var err error + args := []string{"wait", resource, name} if len(namespace) > 0 { args = append(args, "--namespace", namespace) } + args = append(args, "--for", fmt.Sprintf("condition=%s=true", condition), fmt.Sprintf("--timeout=%dm", minutes)) maxRetries := minutes * 30 @@ -271,13 +287,16 @@ func WaitForReadyCondition(kubePath, resource, name, namespace string, condition func GetRancherFQDNFromLoadBalancer(kubePath string) (string, error) { ingress := map[string]string{} + err := Get(kubePath, "services", "", "", ".items[0].status.loadBalancer.ingress[0]", &ingress) if err != nil { return "", err } + if ip, ok := ingress["ip"]; ok { return ip + ".sslip.io", nil } + if hostname, ok := ingress["hostname"]; ok { return hostname, nil } @@ -287,6 +306,7 @@ func GetRancherFQDNFromLoadBalancer(kubePath string) (string, error) { func Get(kubePath string, kind string, name string, namespace string, jsonpath string, out any) error { output := new(bytes.Buffer) + args := []string{ "get", kind, @@ -294,11 +314,13 @@ func Get(kubePath string, kind string, name string, namespace string, jsonpath s if name != "" { args = append(args, name) } + if namespace != "" { args = append(args, "--namespace", namespace) } else { args = append(args, "--all-namespaces") } + args = append(args, "-o", fmt.Sprintf("jsonpath={%s}", jsonpath)) if err := Exec(kubePath, output, args...); err != nil { @@ -314,6 +336,7 @@ func Get(kubePath string, kind string, name string, namespace string, jsonpath s func GetStatus(kubepath, kind, name, namespace string) (map[string]any, error) { out := map[string]any{} + err := Get(kubepath, kind, name, namespace, ".status", &out) if err != nil { return nil, err @@ -326,10 +349,12 @@ func K6run(kubeconfig, testPath string, envVars, tags map[string]string, printLo // gather file entries root := "./charts/k6-files/test-files" exts := map[string]bool{".js": true, ".mjs": true, ".sh": true, ".env": true} + entries, err := getCachedEntries(root, exts) if err != nil { log.Fatal(err) } + relTestPath := testPath // get rel path to test file for _, e := range entries { @@ -341,12 +366,15 @@ func K6run(kubeconfig, testPath string, envVars, tags map[string]string, printLo // print what we are about to do quotedArgs := []string{"run"} + for k, v := range envVars { if k == "BASE_URL" { v = localBaseURL } + quotedArgs = append(quotedArgs, "-e", shellescape.Quote(fmt.Sprintf("%s=%s", k, v))) } + quotedArgs = append(quotedArgs, shellescape.Quote(relTestPath)) log.Printf("Running equivalent of:\n./bin/k6 %s\n", strings.Join(quotedArgs, " ")) @@ -356,6 +384,7 @@ func K6run(kubeconfig, testPath string, envVars, tags map[string]string, printLo if err != nil { return err } + err = Exec(kubeconfig, nil, "--namespace="+K6Namespace, "create", "secret", "generic", K6KubeSecretName, "--from-file=config="+path) if err != nil { @@ -367,16 +396,20 @@ func K6run(kubeconfig, testPath string, envVars, tags map[string]string, printLo args := []string{"run"} // ensure we get the complete summary args = append(args, "--summary-mode=full") + for k, v := range envVars { // substitute kubeconfig file path with path to secret if k == "KUBECONFIG" { v = "/kube/config" } + args = append(args, "-e", fmt.Sprintf("%s=%s", k, v)) } + for k, v := range tags { args = append(args, "--tag", fmt.Sprintf("%s=%s", k, v)) } + args = append(args, relTestPath) if record { args = append(args, "-o", "experimental-prometheus-rw") @@ -395,6 +428,7 @@ func K6run(kubeconfig, testPath string, envVars, tags map[string]string, printLo "subPath": e.Key, }) } + if _, ok := envVars["KUBECONFIG"]; ok { volumes = append(volumes, map[string]any{"name": K6KubeSecretName, "secret": map[string]string{"secretName": "kube"}}) volumeMounts = append(volumeMounts, map[string]string{"mountPath": "/kube", "name": K6KubeSecretName}) @@ -423,6 +457,7 @@ func K6run(kubeconfig, testPath string, envVars, tags map[string]string, printLo "volumes": volumes, }, } + overrideJSON, err := json.Marshal(override) if err != nil { return err diff --git a/internal/qase/client.go b/internal/qase/client.go index bf6ef8c63..006707d09 100644 --- a/internal/qase/client.go +++ b/internal/qase/client.go @@ -56,28 +56,36 @@ func NewCustomUnifiedClient(cfg *config.Config) (*CustomUnifiedClient, error) { // SetupQaseClient creates a new Qase client using the qase-go package. func SetupQaseClient() *CustomUnifiedClient { token := os.Getenv(config.QaseTestOpsAPITokenEnvVar) + projectCode := os.Getenv(config.QaseTestOpsProjectEnvVar) if token == "" { logrus.Fatalf("%s environment variable not set", config.QaseTestOpsAPITokenEnvVar) } + if projectCode == "" { logrus.Fatalf("%s environment variable not set", config.QaseTestOpsProjectEnvVar) } - var err error - var cfg *config.Config + var ( + err error + cfg *config.Config + ) cfgBuilder := config.NewConfigBuilder().LoadFromEnvironment() + cfg, err = cfgBuilder.Build() if err != nil { logrus.Fatalf("Failed to build Qase config from environment variables: %v", err) } + if cfg.Mode == "" { cfg.Mode = config.MODE_TESTOPS } + if cfg.Fallback == "" { cfg.Fallback = config.MODE_REPORT } + if cfg.Debug { logrus.SetLevel(logrus.DebugLevel) } @@ -103,11 +111,14 @@ func (c *CustomUnifiedClient) CreateTestRun(ctx context.Context, testRunName, pr resp, res, err := c.V1Client.GetAPIClient().RunsAPI.CreateRun(authCtx, projectCode).RunCreate(*runCreate).Execute() logResponseBody(res, "CreateTestRun") + if err != nil { return 0, fmt.Errorf("failed to create test run: %w", err) } + runID := *resp.Result.Id c.Config.TestOps.Run.ID = &runID + return runID, nil } @@ -121,6 +132,7 @@ func (c *CustomUnifiedClient) GetTestRun(ctx context.Context, projectCode string resp, res, err := c.V1Client.GetAPIClient().RunsAPI.GetRun(authCtx, projectCode, int32(runID)).Execute() logResponseBody(res, "GetTestRun") + if err != nil { return nil, fmt.Errorf("failed to get test run: %w", err) } @@ -135,6 +147,7 @@ func (c *CustomUnifiedClient) GetTestCase(ctx context.Context, projectCode strin }) resp, res, err := c.V1Client.GetAPIClient().CasesAPI.GetCase(authCtx, projectCode, int32(caseID)).Execute() logResponseBody(res, "GetTestCase") + return resp.Result, err } @@ -151,10 +164,12 @@ func (c *CustomUnifiedClient) CompleteTestRun(ctx context.Context, projectCode s logrus.Debugf("Completing test run ID: %d", runID) _, res, err := c.V1Client.GetAPIClient().RunsAPI.CompleteRun(authCtx, c.Config.TestOps.Project, int32(runID)).Execute() logResponseBody(res, "CompleteTestRun") + if err != nil { return fmt.Errorf("failed to complete test run: %w", err) } } + return nil } @@ -167,13 +182,16 @@ func (c *CustomUnifiedClient) UploadAttachments(ctx context.Context, files []*os projectCode := c.Config.TestOps.Project var hashes []string + for _, file := range files { logrus.Debugf("Uploading attachment: %s", file.Name()) + hash, err := c.V1Client.UploadAttachment(ctx, projectCode, []*os.File{file}) if err != nil { logrus.Warnf("Failed to upload attachment %s: %v", file.Name(), err) continue } + if hash != "" { hashes = append(hashes, hash) } @@ -184,6 +202,7 @@ func (c *CustomUnifiedClient) UploadAttachments(ctx context.Context, files []*os } logrus.Infof("Successfully uploaded %d out of %d attachments.", len(hashes), len(files)) + return hashes, nil } @@ -197,6 +216,7 @@ func (c *CustomUnifiedClient) GetTestCaseByTitle(ctx context.Context, projectCod limit := int32(100) offset := int32(0) + var matchingCase *api_v1_client.TestCase for { @@ -206,6 +226,7 @@ func (c *CustomUnifiedClient) GetTestCaseByTitle(ctx context.Context, projectCod Offset(offset). Execute() logResponseBody(res, "GetTestCaseByTitle") + if err != nil { return nil, fmt.Errorf("failed to get test cases: %w", err) } @@ -239,9 +260,11 @@ func (c *CustomUnifiedClient) CreateTestResultV1(ctx context.Context, projectCod }) _, res, err := c.V1Client.GetAPIClient().ResultsAPI.CreateResult(authCtx, projectCode, int32(runID)).ResultCreate(result).Execute() logResponseBody(res, "CreateTestResultV1") + if err != nil || !strings.Contains(strings.ToLower(res.Status), "ok") { return fmt.Errorf("failed to create v1 test result or did not receive 'OK; response: %w", err) } + return nil } @@ -249,9 +272,11 @@ func (c *CustomUnifiedClient) CreateTestResultV1(ctx context.Context, projectCod func (c *CustomUnifiedClient) CreateTestResultV2(ctx context.Context, projectCode string, runID int64, result api_v2_client.ResultCreate) error { res, err := c.V2Client.GetAPIClient().ResultsAPI.CreateResultV2(ctx, projectCode, runID).ResultCreate(result).Execute() logResponseBody(res, "CreateTestResultV2") + if err != nil { return fmt.Errorf("failed to create v2 test result: %w", err) } + return nil } diff --git a/internal/tofu/format/format.go b/internal/tofu/format/format.go index 61fae6b92..8a3c204c3 100644 --- a/internal/tofu/format/format.go +++ b/internal/tofu/format/format.go @@ -65,6 +65,7 @@ func primitiveToHclString(value interface{}, isNested bool) (string, error) { if isNested { return fmt.Sprintf("\"%v\"", v), nil } + return fmt.Sprintf("%v", v), nil case bool: return strconv.FormatBool(v), nil @@ -79,6 +80,7 @@ func primitiveToHclString(value interface{}, isNested bool) (string, error) { vInt64 = int64(vInt32) } } + return fmt.Sprintf("%d", vInt64), nil case float32, float64: // explicitly convert to float64 if needed @@ -87,6 +89,7 @@ func primitiveToHclString(value interface{}, isNested bool) (string, error) { vFloat64 = float64(v.(float32)) return strconv.FormatFloat(vFloat64, 'f', -1, 32), nil } + return strconv.FormatFloat(vFloat64, 'f', -1, 64), nil default: return fmt.Sprintf("%v", v), fmt.Errorf("no defined case for type of value: %T", v) @@ -100,8 +103,11 @@ func primitiveToHclString(value interface{}, isNested bool) (string, error) { // good enough for the type of variables we deal with in Dartboard. func ConvertValueToHCL(value any, isNested bool) string { // We use type assertions to manually convert into []interface{} and map[string]interface{} if and when needed - var v string - var err error + var ( + v string + err error + ) + if slice, isSlice := value.([]any); isSlice { v = sliceToHclString(slice) } else if m, isMap := value.(map[string]any); isMap { @@ -109,8 +115,10 @@ func ConvertValueToHCL(value any, isNested bool) string { } else { v, err = primitiveToHclString(value, isNested) } + if err != nil { log.Panicf("%v", err) } + return v } diff --git a/internal/tofu/tofu.go b/internal/tofu/tofu.go index f7fe22c67..f3970daf9 100644 --- a/internal/tofu/tofu.go +++ b/internal/tofu/tofu.go @@ -33,9 +33,9 @@ import ( ) type ClusterAddress struct { + Name string `json:"name"` HTTPPort uint `json:"http_port"` HTTPSPort uint `json:"https_port"` - Name string `json:"name"` } type ClusterAppAddresses struct { @@ -51,13 +51,13 @@ type Addresses struct { } type Cluster struct { - AppAddresses ClusterAppAddresses `json:"app_addresses"` + NodeAccessCommands map[string]string `json:"node_access_commands"` + KubernetesAddresses Addresses `json:"kubernetes_addresses"` Name string `json:"name"` Context string `json:"context"` IngressClassName string `json:"ingress_class_name"` Kubeconfig string `json:"kubeconfig"` - NodeAccessCommands map[string]string `json:"node_access_commands"` - KubernetesAddresses Addresses `json:"kubernetes_addresses"` + AppAddresses ClusterAppAddresses `json:"app_addresses"` ReserveNodeForMonitoring bool `json:"reserve_node_for_monitoring"` } @@ -72,13 +72,14 @@ type Output struct { type Tofu struct { dir string workspace string + variables []string threads int verbose bool - variables []string } func New(ctx context.Context, variableMap map[string]interface{}, dir string, ws string, parallelism int, verbose bool) (*Tofu, error) { var variables []string + for k, v := range variableMap { variable := fmt.Sprintf("%s=%s", k, format.ConvertValueToHCL(v, false)) variables = append(variables, variable) @@ -96,6 +97,7 @@ func New(ctx context.Context, variableMap map[string]interface{}, dir string, ws for _, variable := range t.variables { args = append(args, "-var", variable) } + if err := t.exec(nil, args...); err != nil { return nil, err } @@ -109,6 +111,7 @@ func (t *Tofu) exec(output io.Writer, args ...string) error { cmd := vendored.Command("tofu", fullArgs...) var errStream strings.Builder + cmd.Stderr = &errStream cmd.Stdin = os.Stdin @@ -123,6 +126,7 @@ func (t *Tofu) exec(output io.Writer, args ...string) error { if err := cmd.Run(); err != nil { return fmt.Errorf("error while running tofu: %v", errStream.String()) } + return nil } @@ -142,6 +146,7 @@ func (t *Tofu) handleWorkspace(ctx context.Context) error { } log.Printf("Creating new tofu workspace: %s", t.workspace) + if err = t.newWorkspace(ctx); err != nil { return err } @@ -152,9 +157,10 @@ func (t *Tofu) handleWorkspace(ctx context.Context) error { func (t *Tofu) workspaceExists(ctx context.Context) (bool, error) { args := []string{"workspace", "list"} - var out bytes.Buffer - var err error - + var ( + out bytes.Buffer + err error + ) if err = t.exec(&out, args...); err != nil { return false, fmt.Errorf("failed to list workspaces: %v", err) } @@ -199,6 +205,7 @@ func (t *Tofu) commonArgs(command string) []string { for _, variable := range t.variables { args = append(args, "-var", variable) } + return args } diff --git a/internal/vendored/embed.go b/internal/vendored/embed.go index 98a035e41..2617c2725 100644 --- a/internal/vendored/embed.go +++ b/internal/vendored/embed.go @@ -22,8 +22,7 @@ var sourceFS embed.FS // ExtractBinaries extracts embedded binaries at runtime func ExtractBinaries() error { - - err := os.Mkdir(DestinationDir, 0755) + err := os.Mkdir(DestinationDir, 0o755) if err != nil && !os.IsExist(err) { return err } @@ -40,10 +39,12 @@ func ExtractBinaries() error { // skip existing destFile := filepath.Join(".bin", strings.TrimPrefix(path, SourceDir+"/")) + _, err = os.Stat(destFile) if err == nil { return nil } + if !os.IsNotExist(err) { return fmt.Errorf("failed to check if file %v exists: %v", destFile, err) } @@ -54,7 +55,7 @@ func ExtractBinaries() error { return fmt.Errorf("failed to read an embedded file %v: %v", path, err) } - err = os.WriteFile(destFile, content, 0755) + err = os.WriteFile(destFile, content, 0o755) if err != nil { return fmt.Errorf("failed to write %v: %v", destFile, err) } diff --git a/internal/vendored/run.go b/internal/vendored/run.go index fb45534eb..a67b831d2 100644 --- a/internal/vendored/run.go +++ b/internal/vendored/run.go @@ -18,6 +18,7 @@ func Command(name string, args ...string) *exec.Cmd { for i, arg := range args { quotedArgs[i] = shellescape.Quote(arg) } + log.Printf("Running command: \n%s %s\n", vendoredName, strings.Join(quotedArgs, " ")) return exec.Command(vendoredName, args...) diff --git a/k6/generic/k8s.js b/k6/generic/k8s.js index c90da682f..71b4e128d 100644 --- a/k6/generic/k8s.js +++ b/k6/generic/k8s.js @@ -1,6 +1,7 @@ import { check, sleep } from 'k6'; import encoding from 'k6/encoding'; import http from 'k6/http'; +import { WebSocket } from 'k6/experimental/websockets'; import * as YAML from '../lib/js-yaml-4.1.0.mjs' import { URL } from '../lib/url-1.0.0.js'; @@ -115,3 +116,74 @@ export function list(url, limit) { return responses } + +// executes a command in a pod and returns the output +export function exec(baseUrl, namespace, podName, container, command, timeoutSeconds = 30) { + const params = new URLSearchParams(); + params.append('container', container); + params.append('stdout', 'true'); + params.append('stderr', 'true'); + + if (Array.isArray(command)) { + command.forEach(cmd => params.append('command', cmd)); + } else { + params.append('command', 'sh'); + params.append('command', '-c'); + params.append('command', command); + } + + const wsUrl = `${baseUrl}/api/v1/namespaces/${namespace}/pods/${podName}/exec?${params.toString()}` + .replace('https://', 'wss://') + .replace('http://', 'ws://'); + + let output = ''; + let errorOutput = ''; + let completed = false; + + const ws = new WebSocket(wsUrl, 'v4.channel.k8s.io'); + + ws.addEventListener('open', () => { + console.debug(`Exec WebSocket connected to ${podName}`); + }); + + ws.addEventListener('message', (e) => { + if (typeof e.data === 'string') { + const data = e.data; + if (data.length > 1) { + const channel = data.charCodeAt(0); + const content = data.substring(1); + + if (channel === 1) { + output += content; + } else if (channel === 2) { + errorOutput += content; + } + } + } + }); + + ws.addEventListener('close', () => { + completed = true; + console.debug(`Exec WebSocket closed for ${podName}`); + }); + + ws.addEventListener('error', (e) => { + console.error(`Exec WebSocket error: ${e.error}`); + completed = true; + }); + + const startTime = Date.now(); + while (!completed && (Date.now() - startTime) < timeoutSeconds * 1000) { + sleep(0.1); + } + + if (!completed) { + ws.close(); + } + + return { + stdout: output.trim(), + stderr: errorOutput.trim(), + success: errorOutput === '' + }; +} diff --git a/k6/tests/STEVE_WATCH_STRESS_TEST.md b/k6/tests/STEVE_WATCH_STRESS_TEST.md new file mode 100644 index 000000000..d8dadb48f --- /dev/null +++ b/k6/tests/STEVE_WATCH_STRESS_TEST.md @@ -0,0 +1,179 @@ +# Steve Watch Stress Test with SQLite Caching + +This test stresses Steve's watch functionality with SQLite caching enabled, inspired by the stress test scripts from [this gist](https://gist.github.com/aruiz14/cf279761268a1458cb3838e6f41388ac). + +## Overview + +The test simulates a high-stress scenario for Steve by: + +1. **Creating 2000 concurrent WebSocket watchers** (configurable) that subscribe to Steve for resource changes +2. **Continuously creating and deleting resources** (ConfigMaps, Secrets, and Custom Resources) via the Kubernetes API +3. **Periodically updating CRD definitions** to trigger schema change events +4. **Running light read tests** (1 per second) to verify Steve remains responsive + +## Success Criteria + +The test is considered successful if: + +- **Steve responsiveness**: p95 response time for light reads stays below 100ms +- **SQLite WAL file size**: Remains below 10 MB after 10 minutes of stress testing (checked automatically via Kubernetes API) +- **Overall success rate**: At least 95% of operations succeed + +## Prerequisites + +- Access to a Kubernetes cluster with Rancher installed +- Rancher must be running with SQLite caching enabled (`-sql-cache` flag) +- Valid kubeconfig with appropriate permissions +- Rancher credentials (username/password or token) + +## Environment Variables + +| Variable | Description | Default | +|----------|-------------|---------| +| `STEVE_URL` | Steve server URL (Rancher URL) | `http://localhost:8080` | +| `KUBE_API_URL` | Kubernetes API URL | From `KUBECONFIG` | +| `KUBECONFIG` | Path to kubeconfig file | Required | +| `CONTEXT` | Kubeconfig context to use | Required | +| `USERNAME` | Rancher username | Required (or use TOKEN) | +| `PASSWORD` | Rancher password | Required (or use TOKEN) | +| `TOKEN` | Rancher session token | Alternative to USERNAME/PASSWORD | +| `NAMESPACE` | Test namespace | `test-configmaps` | +| `COUNT` | Number of concurrent watchers | `2000` | +| `WATCH_DURATION` | Test duration in seconds | `600` (10 minutes) | +| `RANCHER_NAMESPACE` | Rancher pod namespace | `cattle-system` | +| `RANCHER_POD_LABEL` | Label to find Rancher pod | `app=rancher` | + +## Running the Test + +```bash +k6 run \ + --env STEVE_URL=https://rancher.example.com \ + --env KUBECONFIG=/path/to/kubeconfig \ + --env CONTEXT=my-context \ + --env USERNAME=admin \ + --env PASSWORD=secret \ + --env COUNT=2000 \ + --env WATCH_DURATION=600 \ + k6/tests/steve_watch_stress_test.js +``` + +The test automatically monitors SQLite WAL file size every 10 seconds via the Kubernetes API and reports it as the `sqlite_wal_size_bytes` metric. + +## Test Scenarios + +The test runs the following scenarios in parallel: + +### 1. Watchers Scenario +- **Executor**: `per-vu-iterations` +- **VUs**: Equal to `COUNT` parameter (default: 2000) +- **Duration**: Full test duration +- **Behavior**: Each VU creates a WebSocket connection to Steve and subscribes to: + - ConfigMaps (with resource.changes mode and 4s debounce) + - Secrets (with resource.changes mode and 4s debounce) + - Custom CRD resources (example.com/foos) +- **Connection Lifetime**: Connections remain open for the full test duration plus a small random jitter (up to 10% of duration) to avoid all connections closing simultaneously + +### 2. Create/Delete Events Scenario +- **Executor**: `constant-arrival-rate` +- **Rate**: 10 iterations per second +- **Duration**: Full test duration +- **Behavior**: Each iteration: + 1. Creates a ConfigMap with 1MB of data + 2. Creates a Secret with 1MB of data + 3. Creates a Custom Resource instance + 4. Waits 100ms + 5. Deletes all three resources + +### 3. Update CRDs Scenario +- **Executor**: `constant-arrival-rate` +- **Rate**: ~0.33 iterations per second (once every 3 seconds) +- **Duration**: Full test duration +- **Behavior**: Alternates between two CRD schema versions: + - Version 1: Includes `additionalPrinterColumns` + - Version 2: Minimal schema without extra columns +- Simulates schema changes that Steve must process + +### 4. Light Read Test Scenario +- **Executor**: `constant-arrival-rate` +- **Rate**: 1 iteration per second +- **Duration**: Full test duration +- **Behavior**: Performs a light read of ConfigMaps in the test namespace via Steve API +- **Purpose**: Ensures Steve remains responsive under stress +- **Threshold**: p95 response time must be < 100ms + +### 5. WAL Size Check Scenario +- **Executor**: `constant-arrival-rate` +- **Rate**: 0.1 iterations per second (every 10 seconds) +- **Duration**: Full test duration +- **Behavior**: Executes command in Rancher pod via Kubernetes API to check WAL file size +- **Purpose**: Monitors SQLite WAL file growth during stress test +- **Threshold**: WAL size must remain below 10 MB + +## What the Test Validates + +### Automatically Validated +- ✅ Steve API responsiveness under load +- ✅ Success rate of resource operations +- ✅ HTTP request failure rates +- ✅ WebSocket connection stability +- ✅ SQLite WAL file size (via Kubernetes exec API) + +## Understanding the Results + +### Key Metrics + +- `steve_light_read_duration`: Response time for light reads (should stay < 100ms) +- `sqlite_wal_size_bytes`: SQLite WAL file size in bytes (should stay < 10 MB) +- `http_req_duration`: Overall HTTP request duration +- `http_req_failed`: Rate of failed HTTP requests (should be < 5%) +- `checks`: Overall success rate (should be > 95%) +- `watcher_errors`: Number of WebSocket watcher errors +- `event_create_errors`: Number of create/delete cycle errors +- `crd_update_errors`: Number of CRD update errors + +### Interpreting SQLite WAL Size + +The SQLite WAL (Write-Ahead Log) file grows as Steve processes changes. If it grows beyond 10 MB, it indicates: +- Steve may not be checkpointing the WAL efficiently +- The cache is under too much stress +- Potential performance degradation + +## Differences from Original Gist + +This k6 test is based on [the original gist](https://gist.github.com/aruiz14/cf279761268a1458cb3838e6f41388ac) but with these adaptations: + +1. **WebSocket watchers in JavaScript**: The Go-based watcher logic is reimplemented using k6's WebSocket support +2. **Kubernetes API for resource operations**: All CRUD operations use the Kubernetes API directly (not Steve) +3. **Integrated WAL size check**: Implements Kubernetes exec API via WebSocket to check WAL size from within k6 +4. **Parametrized and configurable**: All key parameters are exposed as environment variables +5. **Integrated metrics**: Uses k6's native metrics and thresholds for validation + +## Troubleshooting + +### WebSocket connection failures +- Verify `STEVE_URL` is accessible +- Check authentication credentials +- Ensure Steve is running with SQLite cache enabled + +### Resource creation failures +- Verify kubeconfig has sufficient permissions +- Check if namespace already exists from previous run +- Ensure CRD can be created in the cluster + +### High response times +- This is expected under stress - the test is designed to push limits +- Monitor SQLite WAL file size +- Check Rancher pod resource usage (CPU/memory) + +## Cleanup + +The test automatically cleans up resources in the teardown phase: +- Deletes the test namespace (which removes all ConfigMaps, Secrets, and Custom Resources) +- Deletes the Custom Resource Definition + +If the test is interrupted, manually clean up: + +```bash +kubectl delete namespace test-configmaps +kubectl delete crd foos.example.com +``` diff --git a/k6/tests/steve_watch_stress_test.js b/k6/tests/steve_watch_stress_test.js new file mode 100644 index 000000000..ff380511a --- /dev/null +++ b/k6/tests/steve_watch_stress_test.js @@ -0,0 +1,479 @@ +import { check, fail, sleep } from 'k6'; +import http from 'k6/http'; +import encoding from 'k6/encoding'; +import exec from 'k6/execution'; +import { Trend, Counter } from 'k6/metrics'; +import { WebSocket } from 'k6/experimental/websockets'; +import { customHandleSummary } from '../generic/k6_utils.js'; +import * as k8s from "../generic/k8s.js"; + +// Steve Watch Stress Test with SQLite Caching +// Stresses Steve's watch functionality with SQLite caching enabled +// Based on: https://gist.github.com/aruiz14/cf279761268a1458cb3838e6f41388ac + +const steveUrl = __ENV.STEVE_URL || 'http://localhost:8080'; +const kubeApiUrl = __ENV.KUBE_API_URL || k8s.kubeconfig.url; +const namespace = __ENV.NAMESPACE || 'test-configmaps'; +const count = parseInt(__ENV.COUNT || 2000); +const watchDuration = parseInt(__ENV.WATCH_DURATION || 600); +const username = __ENV.USERNAME; +const password = __ENV.PASSWORD; +const token = __ENV.TOKEN; +const rancherNamespace = __ENV.RANCHER_NAMESPACE || 'cattle-system'; +const rancherPodLabel = __ENV.RANCHER_POD_LABEL || 'app=rancher'; + +const dataBlob = encoding.b64encode('a'.repeat(750 * 1024)); + +const steveResponseTime = new Trend('steve_light_read_duration', true); +const walSize = new Trend('sqlite_wal_size_bytes', true); +const watcherErrors = new Counter('watcher_errors'); +const eventCreateErrors = new Counter('event_create_errors'); +const crdUpdateErrors = new Counter('crd_update_errors'); + +export const handleSummary = customHandleSummary; + +export const options = { + insecureSkipTLSVerify: true, + tlsAuth: [ + { + cert: k8s.kubeconfig.cert, + key: k8s.kubeconfig.key, + }, + ], + + setupTimeout: '300s', + teardownTimeout: '300s', + + scenarios: { + watchers: { + executor: 'per-vu-iterations', + exec: 'watcherScenario', + vus: count, + iterations: 1, + startTime: '10s', + maxDuration: `${watchDuration + 60}s`, + }, + createDeleteEvents: { + executor: 'constant-arrival-rate', + exec: 'createDeleteEventsScenario', + rate: 10, + timeUnit: '1s', + duration: `${watchDuration}s`, + preAllocatedVUs: 10, + maxVUs: 50, + startTime: '15s', + }, + updateCRDs: { + executor: 'constant-arrival-rate', + exec: 'updateCRDScenario', + rate: 0.33, + timeUnit: '1s', + duration: `${watchDuration}s`, + preAllocatedVUs: 1, + maxVUs: 5, + startTime: '15s', + }, + lightReadTest: { + executor: 'constant-arrival-rate', + exec: 'lightReadScenario', + rate: 1, + timeUnit: '1s', + duration: `${watchDuration}s`, + preAllocatedVUs: 1, + maxVUs: 5, + startTime: '15s', + }, + checkWALSize: { + executor: 'constant-arrival-rate', + exec: 'checkWALSizeScenario', + rate: 0.1, + timeUnit: '1s', + duration: `${watchDuration}s`, + preAllocatedVUs: 1, + maxVUs: 2, + startTime: '15s', + }, + }, + + thresholds: { + checks: ['rate>0.95'], + http_req_failed: ['rate<0.05'], + steve_light_read_duration: ['p(95)<100'], + sqlite_wal_size_bytes: ['max<10485760'], + }, +}; + +export function setup() { + console.log('Setting up Steve watch stress test'); + + let cookies = {}; + if (token) { + console.log('Using token for authentication'); + cookies = {R_SESS: token} + } else if (username && password) { + console.log(`Logging in as ${username}`); + const res = http.post(`${steveUrl}/v3-public/localProviders/local?action=login`, JSON.stringify({ + "description": "UI session", + "responseType": "cookie", + "username": username, + "password": password + }), { + headers: { "Content-Type": "application/json" } + }); + + check(res, { + 'logging in returns status 200': (r) => r.status === 200, + }); + + if (res.status !== 200) { + fail(`Failed to login: ${res.status} ${res.body}`); + } + + cookies = http.cookieJar().cookiesForURL(res.url); + } else { + fail("Please specify either USERNAME and PASSWORD or TOKEN"); + } + + // Clean up any leftovers from past runs + console.log('Cleaning up previous test resources'); + k8s.del(`${kubeApiUrl}/api/v1/namespaces/${namespace}`); + sleep(5); + + // Create namespace + console.log(`Creating namespace ${namespace}`); + const nsBody = { + "metadata": { + "name": namespace, + }, + }; + const nsRes = k8s.create(`${kubeApiUrl}/api/v1/namespaces`, nsBody); + check(nsRes, { + 'create namespace succeeds': (r) => r.status === 201 || r.status === 409, + }); + + // Create CRD + console.log('Creating Custom Resource Definition'); + const crdBody = { + "apiVersion": "apiextensions.k8s.io/v1", + "kind": "CustomResourceDefinition", + "metadata": { + "name": "foos.example.com" + }, + "spec": { + "conversion": { + "strategy": "None" + }, + "group": "example.com", + "names": { + "kind": "Foo", + "listKind": "FooList", + "plural": "foos", + "singular": "foo" + }, + "scope": "Cluster", + "versions": [{ + "additionalPrinterColumns": [{ + "jsonPath": ".metadata.name", + "name": "Name", + "type": "string" + }], + "name": "v1", + "schema": { + "openAPIV3Schema": { + "type": "object" + } + }, + "served": true, + "storage": true + }] + } + }; + const crdRes = k8s.create(`${kubeApiUrl}/apis/apiextensions.k8s.io/v1/customresourcedefinitions`, crdBody); + check(crdRes, { + 'create CRD succeeds': (r) => r.status === 201 || r.status === 409, + }); + + sleep(5); // Let CRD settle + + // Create initial test resources (configmap, secret, and custom resource) + console.log('Creating initial test resources'); + const cmBody = { + "metadata": { + "name": "foo", + "namespace": namespace + }, + "data": {} + }; + k8s.create(`${kubeApiUrl}/api/v1/namespaces/${namespace}/configmaps`, cmBody); + + const secretBody = { + "metadata": { + "name": "foo", + "namespace": namespace + }, + "type": "Opaque", + "data": {} + }; + k8s.create(`${kubeApiUrl}/api/v1/namespaces/${namespace}/secrets`, secretBody); + + const fooBody = { + "apiVersion": "example.com/v1", + "kind": "Foo", + "metadata": { + "name": "foo" + } + }; + k8s.create(`${kubeApiUrl}/apis/example.com/v1/foos`, fooBody); + + sleep(5); + + // Get initial resource versions for watchers + const configmapRV = getResourceVersion(`${steveUrl}/v1/configmaps?filter=metadata.namespace=${namespace}`, cookies); + const secretRV = getResourceVersion(`${steveUrl}/v1/secrets?filter=metadata.namespace=${namespace}`, cookies); + + console.log('Setup complete'); + console.log(`ConfigMap RV: ${configmapRV}, Secret RV: ${secretRV}`); + + return { + cookies: cookies, + configmapRV: configmapRV, + secretRV: secretRV + }; +} + +export function teardown(data) { + console.log('Tearing down test'); + k8s.del(`${kubeApiUrl}/api/v1/namespaces/${namespace}`); + k8s.del(`${kubeApiUrl}/apis/apiextensions.k8s.io/v1/customresourcedefinitions/foos.example.com`); + console.log('Teardown complete'); +} + +function getResourceVersion(url, cookies) { + const res = http.get(url, { cookies: cookies }); + if (res.status !== 200) { + console.warn(`Failed to get resource version from ${url}: ${res.status}`); + return ''; + } + const data = JSON.parse(res.body); + return data.revision || ''; +} + +export function watcherScenario(data) { + const vuId = exec.vu.idInTest; + const wsUrl = steveUrl.replace('http', 'ws') + '/v1/subscribe'; + + try { + const jar = http.cookieJar(); + jar.set(steveUrl, "R_SESS", data.cookies["R_SESS"]); + + const ws = new WebSocket(wsUrl, null, { jar: jar }); + let connected = false; + + ws.addEventListener('open', () => { + console.debug(`[Watcher ${vuId}] Connected`); + connected = true; + + ws.send(JSON.stringify({ + resourceType: 'configmaps', + mode: 'resource.changes', + debounceMs: 4000, + resourceVersion: data.configmapRV, + })); + + ws.send(JSON.stringify({ + resourceType: 'secrets', + mode: 'resource.changes', + debounceMs: 4000, + resourceVersion: data.secretRV, + })); + + ws.send(JSON.stringify({ + resourceType: 'example.com.foos', + mode: 'resource.changes', + debounceMs: 4000, + })); + + console.debug(`[Watcher ${vuId}] Subscribed`); + }); + + ws.addEventListener('error', (e) => { + if (e.error !== 'websocket: close sent') { + console.error(`[Watcher ${vuId}] Error: ${e.error}`); + watcherErrors.add(1); + } + }); + + ws.addEventListener('close', () => { + console.debug(`[Watcher ${vuId}] Disconnected`); + }); + + const jitterPercent = 0.05; + const jitter = (Math.random() - 0.5) * 2 * watchDuration * jitterPercent; + sleep(watchDuration + jitter); + + if (connected) { + ws.close(); + } + } catch (e) { + console.error(`[Watcher ${vuId}] Exception: ${e}`); + watcherErrors.add(1); + } +} + +export function createDeleteEventsScenario() { + try { + k8s.create(`${kubeApiUrl}/api/v1/namespaces/${namespace}/configmaps`, { + "metadata": { "name": "foo", "namespace": namespace }, + "data": { "1m": dataBlob } + }, false); + + k8s.create(`${kubeApiUrl}/api/v1/namespaces/${namespace}/secrets`, { + "metadata": { "name": "foo", "namespace": namespace }, + "type": "Opaque", + "data": { "1m": dataBlob } + }, false); + + k8s.create(`${kubeApiUrl}/apis/example.com/v1/foos`, { + "apiVersion": "example.com/v1", + "kind": "Foo", + "metadata": { "name": "foo" } + }, false); + + sleep(0.1); + + const delCm = k8s.del(`${kubeApiUrl}/api/v1/namespaces/${namespace}/configmaps/foo`); + const delSecret = k8s.del(`${kubeApiUrl}/api/v1/namespaces/${namespace}/secrets/foo`); + const delFoo = k8s.del(`${kubeApiUrl}/apis/example.com/v1/foos/foo`); + + const success = check(null, { + 'create/delete cycle completed': () => true, + 'configmap deleted': () => delCm.status === 200 || delCm.status === 404, + 'secret deleted': () => delSecret.status === 200 || delSecret.status === 404, + 'custom resource deleted': () => delFoo.status === 200 || delFoo.status === 404, + }); + + if (!success) { + eventCreateErrors.add(1); + } + } catch (e) { + console.error(`Create/Delete error: ${e}`); + eventCreateErrors.add(1); + } +} + +export function updateCRDScenario() { + try { + const useVersion1 = exec.scenario.iterationInTest % 2 === 0; + + const crdBody = { + "apiVersion": "apiextensions.k8s.io/v1", + "kind": "CustomResourceDefinition", + "metadata": { + "name": "foos.example.com" + }, + "spec": { + "conversion": { + "strategy": "None" + }, + "group": "example.com", + "names": { + "kind": "Foo", + "listKind": "FooList", + "plural": "foos", + "singular": "foo" + }, + "scope": "Cluster", + "versions": useVersion1 ? [{ + "additionalPrinterColumns": [{ + "jsonPath": ".metadata.name", + "name": "Name", + "type": "string" + }], + "name": "v1", + "schema": { + "openAPIV3Schema": { + "type": "object" + } + }, + "served": true, + "storage": true + }] : [{ + "name": "v1", + "schema": { + "openAPIV3Schema": { + "type": "object" + } + }, + "served": true, + "storage": true + }] + } + }; + + const url = `${kubeApiUrl}/apis/apiextensions.k8s.io/v1/customresourcedefinitions/foos.example.com`; + const res = http.put(url, JSON.stringify(crdBody), { + headers: { 'Content-Type': 'application/json' } + }); + + const success = check(res, { + 'CRD update succeeds': (r) => r.status === 200, + }); + + if (!success) { + console.error(`CRD update failed: ${res.status} ${res.body}`); + crdUpdateErrors.add(1); + } + + sleep(3); + } catch (e) { + console.error(`CRD update error: ${e}`); + crdUpdateErrors.add(1); + } +} + +export function lightReadScenario(data) { + const startTime = new Date().getTime(); + const url = `${steveUrl}/v1/configmaps?filter=metadata.namespace=${namespace}`; + const res = http.get(url, { cookies: data.cookies }); + const duration = new Date().getTime() - startTime; + steveResponseTime.add(duration); + check(res, { + 'light read returns 200': (r) => r.status === 200, + }); +} + +export function checkWALSizeScenario() { + try { + const podsUrl = `${kubeApiUrl}/api/v1/namespaces/${rancherNamespace}/pods?labelSelector=${encodeURIComponent(rancherPodLabel)}`; + const podsRes = http.get(podsUrl); + + if (podsRes.status !== 200) { + console.warn(`Failed to get Rancher pods: ${podsRes.status}`); + return; + } + + const pods = JSON.parse(podsRes.body); + if (!pods.items || pods.items.length === 0) { + console.warn('No Rancher pods found'); + return; + } + + const podName = pods.items[0].metadata.name; + const walPath = '/var/lib/rancher/informer_object_cache.db-wal'; + const cmd = `if [ -f ${walPath} ]; then stat -c %s ${walPath} 2>/dev/null || stat -f %z ${walPath} 2>/dev/null; else echo 0; fi`; + + const result = k8s.exec(kubeApiUrl, rancherNamespace, podName, 'rancher', cmd, 10); + + if (result.success && result.stdout) { + const size = parseInt(result.stdout); + if (!isNaN(size)) { + walSize.add(size); + if (size > 10485760) { + console.warn(`WAL size exceeds 10MB: ${size} bytes`); + } + } + } + } catch (e) { + console.error(`WAL size check error: ${e}`); + } +} diff --git a/qasereporter-k6/main.go b/qasereporter-k6/main.go index ccb904d67..8b2bcd244 100644 --- a/qasereporter-k6/main.go +++ b/qasereporter-k6/main.go @@ -202,6 +202,7 @@ func reportMetrics(params map[string]string) { resultBody := v1.NewResultCreate(status) resultBody.SetCaseId(testCaseID) resultBody.SetComment(comment) + if len(params) > 0 { resultBody.SetParam(params) } @@ -222,6 +223,7 @@ func getAndValidateTestCaseParameters(testCaseParameters []v1.TestCaseParameter) } logrus.Infof("Test case has %d parameter(s), validating against environment variables...", len(testCaseParameters)) + parametersMap := make(map[string]string) for _, parameter := range testCaseParameters { @@ -247,6 +249,7 @@ func getAndValidateTestCaseParameters(testCaseParameters []v1.TestCaseParameter) parametersMap[parameterTitle] = parameterValue } } + return parametersMap } diff --git a/qasereporter-k6/summary.go b/qasereporter-k6/summary.go index 1d3c78dec..acdbf1e25 100644 --- a/qasereporter-k6/summary.go +++ b/qasereporter-k6/summary.go @@ -37,7 +37,7 @@ type K6SummaryMetric struct { Thresholds map[string]K6SummaryThreshold `json:"thresholds,omitempty"` } -// K6SummaryThreshold represents a threshold with its pass/fail status. +// K6SummaryThreshold represents a threshold with its pass/fail status. type K6SummaryThreshold struct { OK bool `json:"ok"` } @@ -113,6 +113,7 @@ func reportSummary(params map[string]string) { resultBody.SetCaseId(testCaseID) resultBody.SetComment(comment) resultBody.SetAttachments(attachmentHashes) + if len(params) > 0 { resultBody.SetParam(params) }