feat: enhance run-remote robustness and reliability

• Fixed massive variance blocking regression detection - Added proper variance-based validation that prevents false positives when latency data is too unstable for reliable analysis

• Fixed missing test links in performance summaries - Corrected regression counting logic so clickable Grafana links appear in bullet lists for confirmed regressions

• Added conditional AWS credentials checking - Credentials are only required when actually needed (for infrastructure deployment or S3 upload, not when using existing infrastructure with --inventory)

• Added automatic tool installation - Missing dependencies like redis-server, zip, and memtier_benchmark are automatically installed on remote nodes using official Redis APT repository

• Enhanced S3 upload error handling - Upload failures don't crash the process but continue with warnings, ensuring benchmark results are preserved locally even when S3 is unavailable

These changes make run-remote much more robust, user-friendly, and reliable by eliminating false regression alerts, providing better diagnostic information, reducing setup requirements, and gracefully handling common failure scenarios.

Author: fcostaoliveira

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redisbench-admin"
-version = "0.11.38"
+version = "0.11.39"
 description = "Redis benchmark run helper. A wrapper around Redis and Redis Modules benchmark tools ( ftsb_redisearch, memtier_benchmark, redis-benchmark, aibench, etc... )."
 authors = ["filipecosta90 <[email protected]>","Redis Performance Group <[email protected]>"]
 readme = "README.md"

redisbench_admin/compare/compare.py

@@ -73,15 +73,7 @@ def deploy_command_logic(args, project_name, project_version):
     tf_triggering_env = "redisbench-admin-deploy"
     logging.info("Setting an infra timeout of {} secs".format(infra_timeout_secs))
     if args.destroy is False:
-        (
-            tf_return_code,
-            _,
-            _,
-            _,
-            _,
-            _,
-            _,
-        ) = setup_remote_environment(
+        (tf_return_code, _, _, _, _, _, _,) = setup_remote_environment(
             tf,
             tf_github_sha,
             tf_github_actor,

redisbench_admin/deploy/deploy.py

@@ -42,13 +42,7 @@ def export_command_logic(args, project_name, project_version):
     deployment_name = args.deployment_name
     deployment_type = args.deployment_type
     results_format = args.results_format
-    (
-        _,
-        github_branch,
-        github_org,
-        github_repo,
-        _,
-    ) = git_vars_crosscheck(
+    (_, github_branch, github_org, github_repo, _,) = git_vars_crosscheck(
         None, args.github_branch, args.github_org, args.github_repo, None
     )
     exporter_timemetric_path = None

redisbench_admin/export/export.py

@@ -400,9 +400,9 @@ def generate_outputs(self, use_case, **kwargs):
             "Main THREAD Flame Graph: " + use_case, details
         )
         if artifact_result is True:
-            outputs["Main THREAD Flame Graph {}".format(identifier)] = (
-                flame_graph_output
-            )
+            outputs[
+                "Main THREAD Flame Graph {}".format(identifier)
+            ] = flame_graph_output
         result &= artifact_result
 
         tid = self.pid
@@ -440,9 +440,9 @@ def generate_outputs(self, use_case, **kwargs):
         )
 
         if artifact_result is True:
-            outputs["perf report per dso,sym {}".format(identifier)] = (
-                perf_report_artifact
-            )
+            outputs[
+                "perf report per dso,sym {}".format(identifier)
+            ] = perf_report_artifact
         result &= artifact_result
 
         # generate perf report per dso,sym
@@ -460,9 +460,9 @@ def generate_outputs(self, use_case, **kwargs):
         )
 
         if artifact_result is True:
-            outputs["perf report per dso,sym with callgraph {}".format(identifier)] = (
-                perf_report_artifact
-            )
+            outputs[
+                "perf report per dso,sym with callgraph {}".format(identifier)
+            ] = perf_report_artifact
         result &= artifact_result
 
         # generate perf report per dso,sym,srcline
@@ -487,9 +487,9 @@ def generate_outputs(self, use_case, **kwargs):
         )
 
         if artifact_result is True:
-            outputs["perf report per dso,sym,srcline {}".format(identifier)] = (
-                perf_report_artifact
-            )
+            outputs[
+                "perf report per dso,sym,srcline {}".format(identifier)
+            ] = perf_report_artifact
         result &= artifact_result
 
         self.logger.info(
@@ -527,9 +527,9 @@ def generate_outputs(self, use_case, **kwargs):
         )
 
         if artifact_result is True:
-            outputs["perf report top self-cpu {}".format(identifier)] = (
-                perf_report_artifact
-            )
+            outputs[
+                "perf report top self-cpu {}".format(identifier)
+            ] = perf_report_artifact
         result &= artifact_result
 
         # generate perf report --stdio report
@@ -546,9 +546,9 @@ def generate_outputs(self, use_case, **kwargs):
             )
 
             if artifact_result is True:
-                outputs["perf report top self-cpu (dso={})".format(binary)] = (
-                    perf_report_artifact
-                )
+                outputs[
+                    "perf report top self-cpu (dso={})".format(binary)
+                ] = perf_report_artifact
         result &= artifact_result
 
         if self.callgraph_mode == "dwarf":
@@ -590,9 +590,9 @@ def generate_outputs(self, use_case, **kwargs):
                 )
                 result &= artifact_result
                 if artifact_result is True:
-                    outputs["Top entries in text form by LOC"] = (
-                        pprof_artifact_text_output
-                    )
+                    outputs[
+                        "Top entries in text form by LOC"
+                    ] = pprof_artifact_text_output
                     tabular_data_map["text-lines"] = tabular_data
                 self.logger.info("Generating pprof png output")
                 pprof_png_output = self.output + ".pprof.png"
@@ -604,9 +604,9 @@ def generate_outputs(self, use_case, **kwargs):
                     self.output,
                 )
                 if artifact_result is True:
-                    outputs["Output graph image in PNG format"] = (
-                        pprof_artifact_png_output
-                    )
+                    outputs[
+                        "Output graph image in PNG format"
+                    ] = pprof_artifact_png_output
                 result &= artifact_result
 
         # save stack collapsed

redisbench_admin/profilers/perf.py

@@ -115,6 +115,12 @@ def spin_up_redis_cluster_remote_redis(
     logname,
     redis_7=True,
 ):
+    # Import the function from standalone module
+    from redisbench_admin.run_remote.standalone import ensure_redis_server_available
+
+    # Ensure redis-server is available before trying to start cluster
+    ensure_redis_server_available(server_public_ip, username, private_key, ssh_port)
+
     logging.info("Generating the remote redis-server command arguments")
     redis_process_commands = []
     logfiles = []

redisbench_admin/run/cluster.py

@@ -206,10 +206,7 @@ def prepare_benchmark_parameters_specif_tooling(
         if isremote is True:
             benchmark_tool = "/tmp/{}".format(benchmark_tool)
             input_data_file = "/tmp/input.data"
-        (
-            command_arr,
-            command_str,
-        ) = prepare_tsbs_benchmark_command(
+        (command_arr, command_str,) = prepare_tsbs_benchmark_command(
             benchmark_tool,
             server_private_ip,
             server_plaintext_port,
@@ -221,10 +218,7 @@ def prepare_benchmark_parameters_specif_tooling(
             cluster_api_enabled,
         )
     if "memtier_benchmark" in benchmark_tool:
-        (
-            command_arr,
-            command_str,
-        ) = prepare_memtier_benchmark_command(
+        (command_arr, command_str,) = prepare_memtier_benchmark_command(
             benchmark_tool,
             server_private_ip,
             server_plaintext_port,
@@ -242,10 +236,7 @@ def prepare_benchmark_parameters_specif_tooling(
             ann_path = stdout[0].strip() + "/run/ann/pkg/multirun.py"
             logging.info("Remote ann-benchmark path: {}".format(ann_path))
 
-        (
-            command_arr,
-            command_str,
-        ) = prepare_ann_benchmark_command(
+        (command_arr, command_str,) = prepare_ann_benchmark_command(
             server_private_ip,
             server_plaintext_port,
             cluster_api_enabled,
@@ -259,10 +250,7 @@ def prepare_benchmark_parameters_specif_tooling(
         if isremote is True:
             benchmark_tool = "/tmp/{}".format(benchmark_tool)
             input_data_file = "/tmp/input.data"
-        (
-            command_arr,
-            command_str,
-        ) = prepare_ftsb_benchmark_command(
+        (command_arr, command_str,) = prepare_ftsb_benchmark_command(
             benchmark_tool,
             server_private_ip,
             server_plaintext_port,
@@ -279,10 +267,7 @@ def prepare_benchmark_parameters_specif_tooling(
         if isremote is True:
             benchmark_tool = "/tmp/{}".format(benchmark_tool)
             input_data_file = "/tmp/input.data"
-        (
-            command_arr,
-            command_str,
-        ) = prepare_aibench_benchmark_command(
+        (command_arr, command_str,) = prepare_aibench_benchmark_command(
             benchmark_tool,
             server_private_ip,
             server_plaintext_port,
@@ -787,10 +772,7 @@ def print_results_table_stdout(
     metric_names=[],
 ):
     # check which metrics to extract
-    (
-        _,
-        metrics,
-    ) = merge_default_and_config_metrics(
+    (_, metrics,) = merge_default_and_config_metrics(
         benchmark_config,
         default_metrics,
         None,

redisbench_admin/run/common.py

@@ -114,11 +114,7 @@ def common_properties_log(self, private_key):
     def async_runner_setup(
         self,
     ):
-        (
-            remote_setup,
-            deployment_type,
-            remote_id,
-        ) = fetch_remote_setup_from_config(
+        (remote_setup, deployment_type, remote_id,) = fetch_remote_setup_from_config(
             [{"type": "async", "setup": "runner"}],
             "https://github.com/RedisLabsModules/testing-infrastructure.git",
             "master",
@@ -233,11 +229,7 @@ def terraform_spin_or_reuse_env(
     tf_override_name=None,
     tf_folder_path=None,
 ):
-    (
-        remote_setup,
-        deployment_type,
-        remote_id,
-    ) = fetch_remote_setup_from_config(
+    (remote_setup, deployment_type, remote_id,) = fetch_remote_setup_from_config(
         benchmark_config["remote"],
         "https://github.com/RedisLabsModules/testing-infrastructure.git",
         "master",

redisbench_admin/run_async/async_terraform.py

@@ -28,9 +28,9 @@ def renderServiceFile(access_key, region, secret_key, gh_token, job_name, args,
         argv.append("--private_key")
         argv.append("/home/ubuntu/work_dir/tests/benchmarks/benchmarks.redislabs.pem")
     else:
-        argv[argv.index(args.private_key)] = (
-            "/home/ubuntu/work_dir/tests/benchmarks/benchmarks.redislabs.pem"
-        )
+        argv[
+            argv.index(args.private_key)
+        ] = "/home/ubuntu/work_dir/tests/benchmarks/benchmarks.redislabs.pem"
     if len(args.module_path) != 0:
         argv[argv.index(args.module_path[0])] = (
             "/home/ubuntu/work_dir/tests/benchmarks/"

redisbench_admin/run_async/render_files.py

@@ -687,17 +687,17 @@ def commandstats_latencystats_process_name(
             branch = variant_labels_dict["branch"]
 
         if version is not None:
-            variant_labels_dict["command_and_metric_and_version"] = (
-                "{} - {} - {}".format(command, metric, version)
-            )
-            variant_labels_dict["command_and_metric_and_setup_and_version"] = (
-                "{} - {} - {} - {}".format(command, metric, setup_name, version)
-            )
+            variant_labels_dict[
+                "command_and_metric_and_version"
+            ] = "{} - {} - {}".format(command, metric, version)
+            variant_labels_dict[
+                "command_and_metric_and_setup_and_version"
+            ] = "{} - {} - {} - {}".format(command, metric, setup_name, version)
 
         if branch is not None:
-            variant_labels_dict["command_and_metric_and_branch"] = (
-                "{} - {} - {}".format(command, metric, branch)
-            )
-            variant_labels_dict["command_and_metric_and_setup_and_branch"] = (
-                "{} - {} - {} - {}".format(command, metric, setup_name, branch)
-            )
+            variant_labels_dict[
+                "command_and_metric_and_branch"
+            ] = "{} - {} - {}".format(command, metric, branch)
+            variant_labels_dict[
+                "command_and_metric_and_setup_and_branch"
+            ] = "{} - {} - {} - {}".format(command, metric, setup_name, branch)