diff --git a/devtools/install_all.sh b/devtools/install_all.sh index e16ba005..20133e0d 100755 --- a/devtools/install_all.sh +++ b/devtools/install_all.sh @@ -83,8 +83,8 @@ fi #Compile RMG-Py make -#Update pyjulia to the latest version -$COMMAND_PKG update pyjulia -c conda_forge -y +#Update pyjulia to the latest version - Don't need this at moment - 0.6.1 is the latest version on RMG-Py +#$COMMAND_PKG update pyjulia -c conda_forge -y #Ensure that added paths etc. are set and then reactivate rmg_env . ~/.bashrc diff --git a/t3/runners/rmg_runner.py b/t3/runners/rmg_runner.py index 1b84d64e..daa783ee 100644 --- a/t3/runners/rmg_runner.py +++ b/t3/runners/rmg_runner.py @@ -96,6 +96,23 @@ def submit_job(project_directory: str, if not len(stdout): time.sleep(10) stdout, stderr = execute_command(cmd) + if stderr: + if cluster_soft.lower() == 'slurm' and any('AssocMaxSubmitJobLimit' in err_line for err_line in stderr): + logger.warning('Max number of submitted jobs was reached, sleeping...') + time.sleep(5 * 60) + submit_job(project_directory=project_directory, + logger=logger, + cluster_soft=cluster_soft, + memory=memory + ) + if cluster_soft.lower() == 'pbs' and any('qsub: would exceed' in err_line for err_line in stderr): + logger.warning('Max number of submitted jobs was reached, sleeping...') + time.sleep(5 * 60) + submit_job(project_directory=project_directory, + logger=logger, + cluster_soft=cluster_soft, + memory=memory + ) if not len(stdout): return None, None if len(stderr) > 0 or len(stdout) == 0: @@ -186,6 +203,14 @@ def run_rmg_incore(rmg_input_file_path: str, stdout, stderr = execute_command(commands, shell=True, no_fail=True, executable='/bin/bash') if 'RMG threw an exception and did not converge.\n' in stderr: return True + # Check for err file that can be generated by Julia + # Todo: Improve the error checking. This situation arose from Julia having a TypeError issue from a recent commit. + if os.path.isfile(os.path.join(project_directory, 'err.txt')): + with open(os.path.join(project_directory, 'err.txt'), 'r') as f: + lines = f.readlines() + if 'Traceback (most recent call last):\n' in lines: + # Raise an error that will inform the user that Julia has failed + raise RuntimeError('Julia has raised an error with RMS and therefore RMG cannot complete. Please check your Julia installation\n') return False diff --git a/tests/test_rmg_runner.py b/tests/test_rmg_runner.py index 79883a58..dc9f1a3e 100644 --- a/tests/test_rmg_runner.py +++ b/tests/test_rmg_runner.py @@ -148,7 +148,6 @@ def test_rmg_job_converged(self): assert converged assert error is None - def teardown_module(): """teardown any state that was previously setup with a setup_module method.""" file_paths = [os.path.join(EXAMPLES_BASE_PATH, 'minimal', 'submit.sh')]