WIP: More Windows-compatible

sumatra/core.py

@@ -51,48 +51,22 @@ def get_encoding():
     return encoding
 
 
-def run(args, cwd=None, shell=False, kill_tree=True, timeout=-1, env=None):
+def run(args, cwd=None, shell=False, kill_tree=True, timeout=None, env=None):
     """
-    Run a command with a timeout after which it will be forcibly
-    killed.
-
-    Based on http://stackoverflow.com/a/3326559
+    Run a command with a timeout.
     """
-    class Alarm(Exception):
-        pass
 
-    def alarm_handler(signum, frame):
-        raise Alarm
-    p = subprocess.Popen(args, shell=shell, cwd=cwd, stdout=subprocess.PIPE,
-                         stderr=subprocess.PIPE, env=env)
-    if timeout != -1:
-        signal.signal(signal.SIGALRM, alarm_handler)
-        signal.alarm(timeout)
-    try:
-        stdout, stderr = p.communicate()
-        stdout = stdout.decode(get_encoding())
-        stderr = stderr.decode(get_encoding())
-        if timeout != -1:
-            signal.alarm(0)
-    except Alarm:
-        pids = [p.pid]
-        if kill_tree:
-            pids.extend(_get_process_children(p.pid))
-        for pid in pids:
-            # process might have died before getting to this line
-            # so wrap to avoid OSError: no such process
-            try:
-                os.kill(pid, signal.SIGKILL)
-            except OSError:
-                pass
-        return -9, '', ''
-    return p.returncode, str(stdout), str(stderr)
+    completed_command = subprocess.run(args, shell=shell, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, timeout=timeout)
+    stdout = completed_command.stdout
+    stdout = stdout.decode(get_encoding())
+    stderr = completed_command.stderr
+    stderr = stderr.decode(get_encoding())
+    return completed_command.returncode, str(stdout), str(stderr)
 
 
 def _get_process_children(pid):
-    p = subprocess.Popen('ps --no-headers -o pid --ppid %d' % pid, shell=True,
-                         stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    stdout, stderr = p.communicate()
+    completed_command = subprocess.run(['ps','--no-headers', '-o', 'pid', '--ppid', pid], shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    stdout = completed_command.stdout
     return [int(child_pid) for child_pid in stdout.split()]
 
 

sumatra/dependency_finder/matlab.py

@@ -21,18 +21,6 @@ def __init__(self, module_name, path, version='unknown', diff='', source=None):
         super(Dependency, self).__init__(module_name, path, version, diff, source)
 
 
-def save_dependencies(cmd, filename):
-    ''' save all dependencies to the file in the current folder '''
-    file_dep = "depfun %s -toponly -quiet -print depfun.data;" %filename # save dependencies to a file
-    mat_args = cmd.split('-r ')[-1]
-    cmd = "%s; %s quit" %(mat_args, file_dep)
-    p = subprocess.Popen(['matlab','-nodesktop', '-nosplash', '-nojvm', ' -nodisplay', '-wait', '-r', cmd], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)    
-    result = p.wait()
-    output = p.stdout.read() 
-    # import pdb; pdb.set_trace() 
-    return result, output
-
-
 def find_dependencies(filename, executable):
     #ifile = os.path.join(os.getcwd(), 'depfun.data')
     with open('depfun.data', 'r') as file_data:

sumatra/launch.py

@@ -76,9 +76,8 @@ def pre_run(self, executable):
         """Run tasks before the simulation/analysis proper."""  # e.g. nrnivmodl
         # this implementation is a temporary hack. "pre_run" should probably be an Executable instance, not a string
         if hasattr(executable, "pre_run"):
-            p = subprocess.Popen(executable.pre_run, shell=True, stdout=None,
-                                 stderr=None, close_fds=True, cwd=self.working_directory)
-            result = p.wait()
+            completed_command = subprocess.run(executable.pre_run, shell=True, stdout=None, stderr=None, close_fds=True, cwd=self.working_directory)
+            result = completed_command.returncode
 
     def check_files(self, executable, main_file):
         """Check that all files exist and are accessible."""
@@ -95,16 +94,16 @@ def run(self, executable, main_file, arguments, append_label=None, capture_stder
         command line. Return resultcode.
         """
         self.check_files(executable, main_file)
-        cmd = self.generate_command(executable, main_file, arguments)
+        cmd = executable.generate_command(main_file, arguments)
         if append_label:
-            cmd += " " + append_label
-        if 'matlab' in executable.name.lower():
+            cmd = [*cmd, append_label]
+        if isinstance(executable, MatlabExecutable):
             ''' we will be executing Matlab and at the same time saving the
             dependencies in order to avoid opening of Matlab shell two times '''
-            result, output = save_dependencies(cmd, main_file)
-        else:
-            result, output = tee.system2(cmd, cwd=self.working_directory, stdout=True, capture_stderr=capture_stderr)  # cwd only relevant for local launch, not for MPI, for example
-        self.stdout_stderr = "".join(output)
+            cmd[-1] = cmd[-1] + ';deps = matlab.codetools.requiredFilesAndProducts(\'%s\', \'toponly\');writelines(deps, \'depfun.data\');' % main_file
+
+        result, output = tee.system2(cmd, cwd=self.working_directory, stdout=True, capture_stderr=capture_stderr)  # cwd only relevant for local launch, not for MPI, for example
+        self.stdout_stderr = output
         return result
 
     def __key(self):

sumatra/programs.py

@@ -41,7 +41,7 @@
 
 
 version_pattern = re.compile(r'\b(?P<version>\d+(\.\d+){1,2}(\.?[a-z]+\d?)?)\b')
-version_pattern_matlab = re.compile(r'(?<=SMT_DETECT_MATLAB_VERSION=)(?P<version>\d.+)\b')
+version_pattern_matlab = re.compile(r'(?<=Version: )(?P<version>\d.+)\b')
 
 
 def version_in_command_line_output(command_line_output, pattern=version_pattern):
@@ -100,9 +100,12 @@ def _find_executable(self, executable_name):
                 print('Multiple versions found, using %s. If you wish to use a different version, please specify it explicitly' % executable)
         return executable
 
+    def generate_command(self, main_file, arguments):
+        return [self.path, main_file, arguments]
+
     def _get_version(self):
-        returncode, output, err = run("%s --version" % self.path,
-                                      shell=True, timeout=5)
+        returncode, output, err = run([self.path, "--version"],
+                                      shell=False, timeout=5)
         return version_in_command_line_output(command_line_output=output + err)
 
     def __eq__(self, other):
@@ -151,7 +154,8 @@ class PythonExecutable(Executable):
     name = "Python"
     executable_names = ('python', 'python2', 'python3', 'python2.5',
                         'python2.6', 'python2.7', 'python3.1', 'python3.2',
-                        'python3.3', 'python3.4', 'python3.5', 'python3.6')
+                        'python3.3', 'python3.4', 'python3.5', 'python3.6',
+                        'py')
     file_extensions = ('.py',)
     default_executable_name = "python"
     requires_script = True
@@ -166,12 +170,14 @@ class MatlabExecutable(Executable):
     requires_script = True
 
     def _get_version(self):
-        returncode, output, err = run("matlab -nodesktop -nosplash -nojvm -r \"disp(['SMT_DETECT_MATLAB_VERSION=' version()]);quit\"",
-                                      shell=True)
+        returncode, output, err = run([self.path, "-h"],
+                                      shell=False)
         return version_in_command_line_output(
             command_line_output=output + err,
             pattern=version_pattern_matlab
         )
+    def generate_command(self, main_file, arguments):
+        return [self.path, '-batch', main_file.replace('.m','')]
 
 
 @component
@@ -208,7 +214,7 @@ def _get_version(self):
                         closefile genesis_version.out
                         quit
                     """)
-        returncode, output, err = run("%s genesis_version.g" % self.path, shell=True)
+        returncode, output, err = run([self.path, "genesis_version.g"], shell=False)
         with open("genesis_version.out") as fd:
             version = fd.read()
         os.remove("genesis_version.g")
@@ -225,10 +231,12 @@ def get_executable(path=None, script_file=None):
     """
     if path:
         prog_name = os.path.basename(path)
-        program = Executable(path)
+        program = None
         for executable_type in get_registered_components(Executable).values():
             if prog_name in executable_type.executable_names:
                 program = executable_type(path)
+        if program is None:
+             program = Executable(path)
     elif script_file:
         script_path, ext = os.path.splitext(script_file)
         program = None

sumatra/tee.py

@@ -137,35 +137,11 @@ def nop(msg):
         # reason: if I have 'quote_command' Sumatra does not work in Windows (it encloses the command in quotes. I did not understand why should we quote)
         # I have never catched "The input line is too long" (yet?)
         # cmd = quote_command(cmd)
-        p = subprocess.Popen(cmd, cwd=cwd, shell=True, stdout=subprocess.PIPE, stderr=stderr, close_fds=(platform.system() == 'Linux'))
         if(log_command):
                 mylogger("Running: %s" % cmd)
-        try:
-            while True:
-                    try:
-                            line = p.stdout.readline()
-                            line = line.decode(encoding)
-                    except Exception as e:
-                            logging.error(e)
-                            logging.error("The output of the command could not be decoded as %s\ncmd: %s\n line ignored: %s" %\
-                                    (encoding, cmd, repr(line)))
-                            pass
-
-                    output.append(line)
-                    if not line:
-                            break
-                    #line = line.rstrip('\n\r')
-                    mylogger(line.rstrip('\n\r')) # they are added by logging anyway
-                    #import pdb; pdb.set_trace()
-                    if(stdout):
-                            print(line, end="")
-                            sys.stdout.flush()
-            returncode = p.wait()
-        except KeyboardInterrupt:
-            # Popen.returncode:
-            #   "A negative value -N indicates that the child was terminated by signal N (Unix only)."
-            # see https://docs.python.org/2/library/subprocess.html#subprocess.Popen.returncode
-            returncode = -signal.SIGINT
+        completed_command = subprocess.run(cmd, cwd=cwd, shell=False, stdout=subprocess.PIPE, stderr=stderr, close_fds=(platform.system() == 'Linux'));
+        returncode = completed_command.returncode
+        output = completed_command.stdout
         if(log_command):
                 if(timing):
                         def secondsToStr(t):
@@ -176,7 +152,7 @@ def secondsToStr(t):
                         mylogger("Returned: %d\n" % (returncode))
 
         if not returncode == 0: # running a tool that returns non-zero? this deserves a warning
-                logging.warning("Returned: %d from: %s\nOutput %s" % (returncode, cmd, ''.join(output)))
+                logging.warning("Returned: %d from: %s\nOutput %s" % (returncode, cmd, output))
 
         return(returncode, output)