[add] Further improved the remote profiler daemon (#235)

Author: filipecosta90

poetry.lock

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redisbench-admin"
-version = "0.5.12"
+version = "0.5.13"
 description = "Redis benchmark run helper. A wrapper around Redis and Redis Modules benchmark tools ( ftsb_redisearch, memtier_benchmark, redis-benchmark, aibench, etc... )."
 authors = ["filipecosta90 <[email protected]>","Redis Performance Group <[email protected]>"]
 readme = "README.md"
@@ -27,7 +27,7 @@ python_terraform = "^0.10.1"
 GitPython = "^3.1.12"
 PyYAML = "^5.4.0"
 wget = "^3.2"
-pytablewriter = "^0.60.0"
+pytablewriter = {extras = ["html"], version = "^0.64.1"}
 sshtunnel = "^0.4.0"
 pyWorkFlow = "^0.0.2"
 Flask = "^2.0.1"

pyproject.toml

@@ -6,11 +6,9 @@
 
 # !/usr/bin/env python3
 import logging
-from threading import Thread
 
 from flask import Flask, request
 import daemonize
-from time import sleep
 import json
 import sys
 import os
@@ -41,10 +39,13 @@ def __init__(self):
 
     def main(self):
         app = Flask(__name__)
-        app.debug = False
-        app.use_reloader = True
+        app.use_reloader = False
         self.perf = Perf()
+        self.set_app_loggers(app)
+        self.create_app_endpoints(app)
+        app.run(debug=False, host="0.0.0.0")
 
+    def set_app_loggers(self, app):
         print("Writting log to {}".format(LOGNAME))
         handler = logging.handlers.RotatingFileHandler(LOGNAME, maxBytes=1024 * 1024)
         logging.getLogger("werkzeug").setLevel(logging.DEBUG)
@@ -53,10 +54,15 @@ def main(self):
         app.logger.addHandler(handler)
         self.perf.set_logger(app.logger)
 
+    def create_app_endpoints(self, app):
         @app.before_first_request
         def before_first_request():
             app.logger.setLevel(logging.INFO)
 
+        @app.get("/ping")
+        def ping():
+            return json.dumps({"result": True})
+
         @app.route("/profiler/<profiler_name>/start/<pid>", methods=["POST"])
         def profile_start(profiler_name, pid):
             setup_process_number = 1
@@ -146,6 +152,7 @@ def profile_status(profiler_name, pid):
         def profile_stop(profiler_name, pid):
             profile_res = self.perf.stop_profile()
             profilers_artifacts_matrix = []
+            bucket_location = "us-east-2"
             primary_id = 1
             total_primaries = 1
             if profile_res is True:
@@ -196,6 +203,7 @@ def profile_stop(profiler_name, pid):
                         [profile_artifact],
                         S3_BUCKET_NAME,
                         s3_bucket_path,
+                        bucket_location,
                     )
                     s3_link = list(url_map.values())[0]
                 profilers_artifacts_matrix.append(
@@ -215,31 +223,26 @@ def profile_stop(profiler_name, pid):
             self.perf = Perf()
             return json.dumps(status_dict)
 
-        Thread(target=app.run).start()
-
-        self.loop()
-
-    def loop(self):
-        while True:
-            sleep(1)
-
 
 d = PerfDaemon()
 
 
 def main():
-    global stop
     global d
 
-    def start():
+    def start(foreground=False):
         current_path = os.path.abspath(os.getcwd())
         print(
             "Starting perfdaemon. PID file {}. Daemon workdir: {}".format(
                 PID_FILE, current_path
             )
         )
         daemonize.Daemonize(
-            app="perfdaemon", pid=PID_FILE, action=d.main, chdir=current_path
+            app="perfdaemon",
+            pid=PID_FILE,
+            action=d.main,
+            chdir=current_path,
+            foreground=foreground,
         ).start()
 
     def stop():
@@ -250,7 +253,7 @@ def stop():
         os.system("kill -9 %s" % pid)
 
     def foreground():
-        d.main()
+        start(True)
 
     def usage():
         print("usage: start|stop|restart|foreground")

redisbench_admin/profilers/daemon.py

@@ -62,6 +62,11 @@ def start_profile(self, pid, output="", frequency=99):
             "dso": self.dso,
             "test_name": self.test_name,
             "setup_name": self.setup_name,
+            "github_actor": self.github_actor,
+            "github_branch": self.github_branch,
+            "github_repo_name": self.github_repo_name,
+            "github_org_name": self.github_org_name,
+            "github_sha": self.github_sha,
         }
         url = "http://{}/profiler/perf/start/{}?frequency={}".format(
             self.remote_endpoint, pid, frequency