perf_test: return 1 in case of ratio non 100% and in any fail reason (#118)

Author: lupin012

perf/run_perf_tests.py

@@ -71,7 +71,7 @@ def usage(argv):
     print("-c,--run-vegeta-on-core <...>         taskset format for vegeta (e.g. 0-1:2-3 or 0-2:3-4)                [default: " + DEFAULT_DAEMON_VEGETA_ON_CORE +"]")
     print("-T,--response-timeout <timeout>:      vegeta response timeout                                            [default: " + DEFAULT_VEGETA_RESPONSE_TIMEOUT + "]")
     print("-M,--max-body-rsp <size>:             max number of bytes to read from response bodies                   [default: " + DEFAULT_MAX_BODY_RSP + "]")
-    sys.exit(-1)
+    sys.exit(1)
 
 def get_process(process_name: str):
     """ Return the running process having specified name or None if not exists """
@@ -207,7 +207,7 @@ def __parse_args(self, argv):
             # print help information and exit:
             print(err)
             usage(argv)
-            sys.exit(-1)
+            sys.exit(1)
 
 
 class PerfTest:
@@ -240,7 +240,7 @@ def copy_and_extract_pattern_file(self):
         """ Copy the vegeta pattern file into /tmp/run_tests_xyz/ and untar the file """
         if os.path.exists(self.config.vegeta_pattern_tar_file) == 0:
             print ("ERROR: invalid pattern file: ", self.config.vegeta_pattern_tar_file)
-            sys.exit(-1)
+            sys.exit(1)
         cmd = "mkdir " +  RUN_TEST_DIRNAME
         status = os.system(cmd)
         cmd = "/bin/cp -f " + self.config.vegeta_pattern_tar_file + " " + VEGETA_TAR_FILE_NAME
@@ -249,15 +249,15 @@ def copy_and_extract_pattern_file(self):
         status = os.system(cmd)
         if int(status) != 0:
             print("Vegeta pattern copy failed. Test Aborted!")
-            sys.exit(-1)
+            sys.exit(1)
 
         cmd = "cd " + RUN_TEST_DIRNAME + "; tar xvf " + VEGETA_TAR_FILE_NAME + " > /dev/null"
         if self.config.tracing:
             print(f"Extracting Vegeta pattern: {cmd}")
         status = os.system(cmd)
         if int(status) != 0:
             print("Vegeta pattern untar failed. Test Aborted!")
-            sys.exit(-1)
+            sys.exit(1)
 
         # If address is provided substitute the address and port of daemon in the vegeta file
         if self.config.rpc_daemon_address != "localhost":
@@ -295,8 +295,8 @@ def execute(self, test_number, name, qps_value, duration):
         sys.stdout.flush()
         status = os.system(cmd)
         if int(status) != 0:
-            print("vegeta test fails: Test Aborted!")
-            return 0
+            print("vegeta attach fails: Test Aborted!")
+            return 1
 
         while 1:
             time.sleep(3)
@@ -309,13 +309,13 @@ def execute(self, test_number, name, qps_value, duration):
                 if pid == "" :
                     # the server is dead; kill vegeta and returns fails
                     os.system("kill -2 $(ps aux | grep 'vegeta' | grep -v 'grep' | grep -v 'python' | awk '{print $2}') 2> /dev/null")
-                    return 0
+                    print ("test failed: server is Dead")
+                    return 1
 
             pid = os.popen("ps aux | grep 'vegeta report' | grep -v 'grep' | awk '{print $2}'").read()
             if pid == "":
-                # Vegeta has completed its works, generate report and return OK
-                self.get_result(test_number, name, qps_value, duration)
-                return 1
+                # Vegeta has completed its works, generate report
+                return self.get_result(test_number, name, qps_value, duration)
 
     def execute_sequence(self, sequence, tag):
         """ Execute the sequence of tests """
@@ -327,13 +327,12 @@ def execute_sequence(self, sequence, tag):
                 test_name = "[{:d}.{:2d}] "
                 test_name_formatted = test_name.format(test_number, test_rep+1)
                 result = self.execute(test_name_formatted, tag, qps, duration)
-                if result == 0:
-                    print("Server dead test Aborted!")
-                    return 0
+                if result == 1:
+                    return 1
                 time.sleep(self.config.waiting_time)
             test_number = test_number + 1
             print("")
-        return 1
+        return 0
 
     def get_result(self, test_number, daemon_name, qps_value, duration):
         """ Processes the report file generated by vegeta and reads latency data """
@@ -348,7 +347,7 @@ def get_result(self, test_number, daemon_name, qps_value, duration):
             newline = file_raws[5].replace('\n', ' ')
             ratio = newline.split(' ')[34]
             if len(file_raws) > 8:
-                error = file_raws[8]
+                error = file_raws[8].rstrip()
                 print(" [ Ratio="+ratio+", MaxLatency="+max_latency+ " Error: " + error +"]")
             else:
                 error = ""
@@ -357,11 +356,17 @@ def get_result(self, test_number, daemon_name, qps_value, duration):
         finally:
             file.close()
 
+        if error != "" or ratio != "100.00%":
+            print ("test failed: ratio is not 100.00%")
+            return 1
+
         if self.config.create_test_report:
             self.test_report.write_test_report(daemon_name, test_number, threads, qps_value, duration, min_latency, latency_values[7], latency_values[8], \
                                                latency_values[9], latency_values[10], latency_values[11], max_latency, ratio, error)
         os.system("/bin/rm " + test_report_filename)
 
+        return 0
+
 
 class Hardware:
     """ Extract hardware information from the underlying platform. """
@@ -517,27 +522,25 @@ def main(argv):
 
     if config.test_mode in ("1", "3"):
         result = perf_test.execute_sequence(current_sequence, SILKRPC)
-        if result == 0:
-            print("Server dead test Aborted!")
+        if result == 1:
             if config.create_test_report:
                 test_report.close()
-            sys.exit(-1)
+            return 1
         if config.test_mode == "3":
             print("--------------------------------------------------------------------------------------------\n")
 
     if config.test_mode in ("2", "3"):
         result = perf_test.execute_sequence(current_sequence, RPCDAEMON)
-        if result == 0:
-            print("Server dead test Aborted!")
+        if result == 1:
             if config.create_test_report:
                 test_report.close()
-            sys.exit(-1)
+            return 1
 
     if config.create_test_report:
         test_report.close()
     perf_test.cleanup(0)
     print("Performance Test completed successfully.")
-
+    return 0
 
 #
 # module as main