Implement YAML+Jinja2 support, code re-org, remove out-dated functions

harness/__init__.py

@@ -2,8 +2,8 @@
             "bin",
             "libraries",
             "utilities",
-            "fundamental_types",
-            "machine_types"
+            "machine_types",
+            "schedulers"
           ]
 
 version = 3.1

harness/bin/check_executable_driver.py

@@ -7,6 +7,11 @@
 import getopt
 import string
 
+prefix = os.path.normpath(
+    os.path.join(os.path.abspath(os.path.dirname(__file__)), '..')
+)
+sys.path = [prefix] + sys.path
+
 # Harness imports
 from libraries.apptest import subtest
 from libraries.subtest_factory import SubtestFactory 

harness/bin/log_binary_execution_time.py

@@ -4,6 +4,11 @@
 import sys
 import os
 
+prefix = os.path.normpath(
+    os.path.join(os.path.abspath(os.path.dirname(__file__)), '..')
+)
+sys.path = [prefix] + sys.path
+
 from libraries.subtest_factory import SubtestFactory
 from libraries.status_file_factory import StatusFileFactory
 from libraries.status_file import StatusFile

harness/bin/runtests.py

@@ -6,6 +6,11 @@
 import os
 import sys
 
+prefix = os.path.normpath(
+    os.path.join(os.path.abspath(os.path.dirname(__file__)), '..')
+)
+sys.path = [prefix] + sys.path
+
 # My harness package imports
 from libraries import input_files
 from libraries import regression_test

harness/bin/test_harness_driver.py

@@ -24,6 +24,11 @@
 
 from shlex import split
 
+prefix = os.path.normpath(
+    os.path.join(os.path.abspath(os.path.dirname(__file__)), '..')
+)
+sys.path = [prefix] + sys.path
+
 # Harness imports
 from libraries.subtest_factory import SubtestFactory
 from libraries.layout_of_apps_directory import apptest_layout as layout
@@ -35,7 +40,6 @@
 from libraries import status_file
 from libraries.rgt_loggers import rgt_logger_factory
 from machine_types.machine_factory import MachineFactory
-from machine_types.base_machine import SetBuildRTEError
 
 MODULE_THRESHOLD_LOG_LEVEL = "DEBUG"
 """str : The logging level for this module. """
@@ -160,15 +164,22 @@ def auto_generated_scripts(harness_config,
     build_exit_value = 0
     if actions['build']:
         # Build the executable for this test on the specified machine
+        scripts_dir = os.getcwd()
         jstatus.log_event(status_file.StatusFile.EVENT_BUILD_START)
         try:
             build_exit_value = mymachine.build_executable()
-        except SetBuildRTEError as error:
-            message = f"Unable to set the build runtime environnment."
-            message += error.message
-            a_logger.doCriticalLogging(message)
-        finally:
-            jstatus.log_event(status_file.StatusFile.EVENT_BUILD_END, build_exit_value)
+        except KeyboardInterrupt:
+            a_logger.doCriticalLogging(f"Detected CTRL+C, aborting build.")
+            os.chdir(scripts_dir)
+            build_exit_value = 21
+            pass
+        except Exception:
+            a_logger.doCriticalLogging(f"Exception generated during build, aborting test launch.")
+            os.chdir(scripts_dir)
+            build_exit_value = 1
+            pass
+
+        jstatus.log_event(status_file.StatusFile.EVENT_BUILD_END, build_exit_value)
     #-----------------------------------------------------
     # In this section we run the the binary.             -
     #                                                    -

harness/libraries/__init__.py

@@ -9,7 +9,8 @@
                 'regression_test',
                 'status_file',
                 'repositories',
-                'rgt_loggers'
+                'rgt_loggers',
+                'rgt_state',
                 'command_line',
                 'get_machine_name',
                 'status_file_factory',

harness/libraries/apptest.py

@@ -533,9 +533,6 @@ def _start_test(self,
             message = ( "The command '{cmd}' has exited with a failure.\n"
                         "The exit return value is {value}.\n").format(cmd=starttestcomand,value=exit_status)
             self.logger.doCriticalLogging(message)
-
-
-            string1 = "Command failed: " + starttestcomand
             return 1
         else:
             message =  "'{cmd}' has executed sucessfully.\n".format(cmd=starttestcomand)

harness/libraries/layout_of_apps_directory.py

@@ -27,6 +27,7 @@ class apptest_layout:
     app_info_filename = 'application_info.txt'
     test_info_filename = 'test_info.txt'
     test_input_ini_filename = 'rgt_test_input.ini'
+    test_input_yaml_filename = 'rgt_test_input.yaml'
     test_kill_filename = '.kill_test'
     test_rc_filename = '.testrc'
     test_status_filename = 'rgt_status.txt'
@@ -73,6 +74,7 @@ class apptest_layout:
         'runarchive_dir'  : os.path.join("${pdir}", "${app}", "${test}", test_run_archive_dirname, "${id}"),
         'scripts_dir'     : os.path.join("${pdir}", "${app}", "${test}", test_scripts_dirname),
         'test_input_ini'  : os.path.join("${pdir}", "${app}", "${test}", test_scripts_dirname, test_input_ini_filename),
+        'test_input_yaml'  : os.path.join("${pdir}", "${app}", "${test}", test_scripts_dirname, test_input_yaml_filename),
         'kill_file'       : os.path.join("${pdir}", "${app}", "${test}", test_scripts_dirname, test_kill_filename),
         'status_dir'      : os.path.join("${pdir}", "${app}", "${test}", test_status_dirname, "${id}"),
         'job_id_file'     : os.path.join("${pdir}", "${app}", "${test}", test_status_dirname, "${id}", job_id_filename),
@@ -126,7 +128,8 @@ def check_paths(self):
             self.__logger.doErrorLogging(f"Could not find the Scripts directory for App={self.__appname}, Test={self.__testname}.")
             return False
         # Check that the an rgt_test_ini.ini file exists 
-        if not (os.path.exists(self.__apptest_layout['test_input_ini'])):
+        if not (os.path.exists(self.__apptest_layout['test_input_ini']) or \
+                os.path.exists(self.__apptest_layout['test_input_yaml'])):
             self.__logger.doErrorLogging(f"Could not find the test input file for App={self.__appname}, Test={self.__testname}.")
             return False
         return True
@@ -136,10 +139,14 @@ def get_harness_id(self):
         return self.__testid
 
     @property
-    def path_of_test_input_file(self):
-        """Returns the path to the subtest INI input file. """
+    def path_of_test_input_file_ini(self):
+        """Returns the path to the subtest input file. """
         return self.__apptest_layout['test_input_ini']
 
+    @property
+    def path_of_test_input_file_yaml(self):
+        """Returns the path to the subtest input file. """
+        return self.__apptest_layout['test_input_yaml']
 
     @property
     def path_to_logfile(self) :

harness/libraries/regression_test.py

@@ -13,7 +13,7 @@
 # Harness package imports.
 from libraries import apptest
 from libraries.subtest_factory import SubtestFactory
-from fundamental_types.rgt_state import RgtState
+from libraries.rgt_state import RgtState
 from libraries.rgt_loggers import rgt_logger_factory
 from machine_types.machine_factory import MachineFactory
 
@@ -262,37 +262,48 @@ def __run_subtests_asynchronously(self):
         # Submit futures by means of thread pool.
         with concurrent.futures.ThreadPoolExecutor(max_workers=self.__num_workers) as executor:
             for subtest in self.__app_subtests:
-                future = executor.submit(apptest.do_application_tasks,
+                # gracefully handle keyboard interrupts in main thread
+                try:
+                    future = executor.submit(apptest.do_application_tasks,
                                          self.__launch_id,
                                          subtest,
                                          self.__tasks,
                                          self.__stdout_stderr,
                                          self.__separate_build_stdio,
                                          self.__reuse_first_build,
                                          self.__reuse_build_from_id)
-                future_to_appname[future] = f'{subtest.getNameOfApplication()}.{subtest.getNameOfSubtest()}'
+                    future_to_appname[future] = f'{subtest.getNameOfApplication()}.{subtest.getNameOfSubtest()}'
+                except KeyboardInterrupt:
+                    pass
 
             # Log when all job tasks are initiated.
-            for my_future in concurrent.futures.as_completed(future_to_appname):
-                # appname is appname.testname, as set above
-                appname = future_to_appname[my_future]
-
-                # Check if an exception has been raised
-                my_future_exception = my_future.exception()
-                if my_future_exception:
-                    message = "Test {} exception encountered:\n{}".format(appname, my_future_exception)
-                    self.__myLogger.doCriticalLogging(message)
-
-                subtest_result = my_future.result()
-                if subtest_result:
-                    self.__launched_tests += 1
-                    message = "Test {} is launched.\n\n".format(appname)
-                    self.__myLogger.doErrorLogging(message)
-                else:
-                    self.__failed_tests += 1
-                    self.__failed_test_list.append(appname)
-                    message = "Test {} failed to launch.\n\n".format(appname)
-                    self.__myLogger.doErrorLogging(message)
+            all_finished = False
+            while not all_finished:
+                # gracefully handle keyboard interrupts in main thread
+                try:
+                    for my_future in concurrent.futures.as_completed(future_to_appname):
+                        # appname is appname.testname, as set above
+                        appname = future_to_appname[my_future]
+
+                        # Check if an exception has been raised
+                        my_future_exception = my_future.exception()
+                        if my_future_exception:
+                            message = "Test {} exception encountered:\n{}".format(appname, my_future_exception)
+                            self.__myLogger.doCriticalLogging(message)
+
+                        subtest_result = my_future.result()
+                        if subtest_result:
+                            self.__launched_tests += 1
+                            message = "Test {} is launched.\n\n".format(appname)
+                            self.__myLogger.doErrorLogging(message)
+                        else:
+                            self.__failed_tests += 1
+                            self.__failed_test_list.append(appname)
+                            message = "Test {} failed to launch.\n\n".format(appname)
+                            self.__myLogger.doErrorLogging(message)
+                    all_finished = True
+                except KeyboardInterrupt:
+                    pass
 
             message = "All tests are launched. Yahoo!!"
             self.__myLogger.doInfoLogging(message)