diff --git a/sumatra/core.py b/sumatra/core.py index 54de3548..282c26d2 100644 --- a/sumatra/core.py +++ b/sumatra/core.py @@ -51,48 +51,22 @@ def get_encoding(): return encoding -def run(args, cwd=None, shell=False, kill_tree=True, timeout=-1, env=None): +def run(args, cwd=None, shell=False, kill_tree=True, timeout=None, env=None): """ - Run a command with a timeout after which it will be forcibly - killed. - - Based on http://stackoverflow.com/a/3326559 + Run a command with a timeout. """ - class Alarm(Exception): - pass - def alarm_handler(signum, frame): - raise Alarm - p = subprocess.Popen(args, shell=shell, cwd=cwd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, env=env) - if timeout != -1: - signal.signal(signal.SIGALRM, alarm_handler) - signal.alarm(timeout) - try: - stdout, stderr = p.communicate() - stdout = stdout.decode(get_encoding()) - stderr = stderr.decode(get_encoding()) - if timeout != -1: - signal.alarm(0) - except Alarm: - pids = [p.pid] - if kill_tree: - pids.extend(_get_process_children(p.pid)) - for pid in pids: - # process might have died before getting to this line - # so wrap to avoid OSError: no such process - try: - os.kill(pid, signal.SIGKILL) - except OSError: - pass - return -9, '', '' - return p.returncode, str(stdout), str(stderr) + completed_command = subprocess.run(args, shell=shell, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, timeout=timeout) + stdout = completed_command.stdout + stdout = stdout.decode(get_encoding()) + stderr = completed_command.stderr + stderr = stderr.decode(get_encoding()) + return completed_command.returncode, str(stdout), str(stderr) def _get_process_children(pid): - p = subprocess.Popen('ps --no-headers -o pid --ppid %d' % pid, shell=True, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout, stderr = p.communicate() + completed_command = subprocess.run(['ps','--no-headers', '-o', 'pid', '--ppid', pid], shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout = completed_command.stdout return [int(child_pid) for child_pid in stdout.split()] diff --git a/sumatra/dependency_finder/matlab.py b/sumatra/dependency_finder/matlab.py index b05322bd..70015873 100644 --- a/sumatra/dependency_finder/matlab.py +++ b/sumatra/dependency_finder/matlab.py @@ -21,18 +21,6 @@ def __init__(self, module_name, path, version='unknown', diff='', source=None): super(Dependency, self).__init__(module_name, path, version, diff, source) -def save_dependencies(cmd, filename): - ''' save all dependencies to the file in the current folder ''' - file_dep = "depfun %s -toponly -quiet -print depfun.data;" %filename # save dependencies to a file - mat_args = cmd.split('-r ')[-1] - cmd = "%s; %s quit" %(mat_args, file_dep) - p = subprocess.Popen(['matlab','-nodesktop', '-nosplash', '-nojvm', ' -nodisplay', '-wait', '-r', cmd], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) - result = p.wait() - output = p.stdout.read() - # import pdb; pdb.set_trace() - return result, output - - def find_dependencies(filename, executable): #ifile = os.path.join(os.getcwd(), 'depfun.data') with open('depfun.data', 'r') as file_data: diff --git a/sumatra/launch.py b/sumatra/launch.py index b2da77ff..74d2450a 100644 --- a/sumatra/launch.py +++ b/sumatra/launch.py @@ -76,9 +76,8 @@ def pre_run(self, executable): """Run tasks before the simulation/analysis proper.""" # e.g. nrnivmodl # this implementation is a temporary hack. "pre_run" should probably be an Executable instance, not a string if hasattr(executable, "pre_run"): - p = subprocess.Popen(executable.pre_run, shell=True, stdout=None, - stderr=None, close_fds=True, cwd=self.working_directory) - result = p.wait() + completed_command = subprocess.run(executable.pre_run, shell=True, stdout=None, stderr=None, close_fds=True, cwd=self.working_directory) + result = completed_command.returncode def check_files(self, executable, main_file): """Check that all files exist and are accessible.""" @@ -95,16 +94,16 @@ def run(self, executable, main_file, arguments, append_label=None, capture_stder command line. Return resultcode. """ self.check_files(executable, main_file) - cmd = self.generate_command(executable, main_file, arguments) + cmd = executable.generate_command(main_file, arguments) if append_label: - cmd += " " + append_label - if 'matlab' in executable.name.lower(): + cmd = [*cmd, append_label] + if isinstance(executable, MatlabExecutable): ''' we will be executing Matlab and at the same time saving the dependencies in order to avoid opening of Matlab shell two times ''' - result, output = save_dependencies(cmd, main_file) - else: - result, output = tee.system2(cmd, cwd=self.working_directory, stdout=True, capture_stderr=capture_stderr) # cwd only relevant for local launch, not for MPI, for example - self.stdout_stderr = "".join(output) + cmd[-1] = cmd[-1] + ';deps = matlab.codetools.requiredFilesAndProducts(\'%s\', \'toponly\');writelines(deps, \'depfun.data\');' % main_file + + result, output = tee.system2(cmd, cwd=self.working_directory, stdout=True, capture_stderr=capture_stderr) # cwd only relevant for local launch, not for MPI, for example + self.stdout_stderr = output return result def __key(self): diff --git a/sumatra/programs.py b/sumatra/programs.py index f817f6ad..e3d13653 100644 --- a/sumatra/programs.py +++ b/sumatra/programs.py @@ -41,7 +41,7 @@ version_pattern = re.compile(r'\b(?P\d+(\.\d+){1,2}(\.?[a-z]+\d?)?)\b') -version_pattern_matlab = re.compile(r'(?<=SMT_DETECT_MATLAB_VERSION=)(?P\d.+)\b') +version_pattern_matlab = re.compile(r'(?<=Version: )(?P\d.+)\b') def version_in_command_line_output(command_line_output, pattern=version_pattern): @@ -100,9 +100,12 @@ def _find_executable(self, executable_name): print('Multiple versions found, using %s. If you wish to use a different version, please specify it explicitly' % executable) return executable + def generate_command(self, main_file, arguments): + return [self.path, main_file, arguments] + def _get_version(self): - returncode, output, err = run("%s --version" % self.path, - shell=True, timeout=5) + returncode, output, err = run([self.path, "--version"], + shell=False, timeout=5) return version_in_command_line_output(command_line_output=output + err) def __eq__(self, other): @@ -151,7 +154,8 @@ class PythonExecutable(Executable): name = "Python" executable_names = ('python', 'python2', 'python3', 'python2.5', 'python2.6', 'python2.7', 'python3.1', 'python3.2', - 'python3.3', 'python3.4', 'python3.5', 'python3.6') + 'python3.3', 'python3.4', 'python3.5', 'python3.6', + 'py') file_extensions = ('.py',) default_executable_name = "python" requires_script = True @@ -166,12 +170,14 @@ class MatlabExecutable(Executable): requires_script = True def _get_version(self): - returncode, output, err = run("matlab -nodesktop -nosplash -nojvm -r \"disp(['SMT_DETECT_MATLAB_VERSION=' version()]);quit\"", - shell=True) + returncode, output, err = run([self.path, "-h"], + shell=False) return version_in_command_line_output( command_line_output=output + err, pattern=version_pattern_matlab ) + def generate_command(self, main_file, arguments): + return [self.path, '-batch', main_file.replace('.m','')] @component @@ -208,7 +214,7 @@ def _get_version(self): closefile genesis_version.out quit """) - returncode, output, err = run("%s genesis_version.g" % self.path, shell=True) + returncode, output, err = run([self.path, "genesis_version.g"], shell=False) with open("genesis_version.out") as fd: version = fd.read() os.remove("genesis_version.g") @@ -225,10 +231,12 @@ def get_executable(path=None, script_file=None): """ if path: prog_name = os.path.basename(path) - program = Executable(path) + program = None for executable_type in get_registered_components(Executable).values(): if prog_name in executable_type.executable_names: program = executable_type(path) + if program is None: + program = Executable(path) elif script_file: script_path, ext = os.path.splitext(script_file) program = None diff --git a/sumatra/tee.py b/sumatra/tee.py index 476479a6..7891dbda 100644 --- a/sumatra/tee.py +++ b/sumatra/tee.py @@ -137,35 +137,11 @@ def nop(msg): # reason: if I have 'quote_command' Sumatra does not work in Windows (it encloses the command in quotes. I did not understand why should we quote) # I have never catched "The input line is too long" (yet?) # cmd = quote_command(cmd) - p = subprocess.Popen(cmd, cwd=cwd, shell=True, stdout=subprocess.PIPE, stderr=stderr, close_fds=(platform.system() == 'Linux')) if(log_command): mylogger("Running: %s" % cmd) - try: - while True: - try: - line = p.stdout.readline() - line = line.decode(encoding) - except Exception as e: - logging.error(e) - logging.error("The output of the command could not be decoded as %s\ncmd: %s\n line ignored: %s" %\ - (encoding, cmd, repr(line))) - pass - - output.append(line) - if not line: - break - #line = line.rstrip('\n\r') - mylogger(line.rstrip('\n\r')) # they are added by logging anyway - #import pdb; pdb.set_trace() - if(stdout): - print(line, end="") - sys.stdout.flush() - returncode = p.wait() - except KeyboardInterrupt: - # Popen.returncode: - # "A negative value -N indicates that the child was terminated by signal N (Unix only)." - # see https://docs.python.org/2/library/subprocess.html#subprocess.Popen.returncode - returncode = -signal.SIGINT + completed_command = subprocess.run(cmd, cwd=cwd, shell=False, stdout=subprocess.PIPE, stderr=stderr, close_fds=(platform.system() == 'Linux')); + returncode = completed_command.returncode + output = completed_command.stdout if(log_command): if(timing): def secondsToStr(t): @@ -176,7 +152,7 @@ def secondsToStr(t): mylogger("Returned: %d\n" % (returncode)) if not returncode == 0: # running a tool that returns non-zero? this deserves a warning - logging.warning("Returned: %d from: %s\nOutput %s" % (returncode, cmd, ''.join(output))) + logging.warning("Returned: %d from: %s\nOutput %s" % (returncode, cmd, output)) return(returncode, output)