Merge branch 'master'

Author: abartesaghi

docs/changelog.rst

@@ -4,6 +4,25 @@ Changelog
 
 **Legend**: :fa:`star,text-success` New feature, :fa:`plus-square,text-primary` Improvement, :fa:`bug,text-danger` Bug fix
 
+v0.6.5 (4/6/2024)
+******************
+
+   :fa:`plus-square,text-primary` Update format of logger messages to more clearly show the nextPYP version and resources assigned to each job.
+
+   :fa:`plus-square,text-primary` Use same tilt-axis angle convention for aligning tilt-series using IMOD and AreTomo2.
+
+   :fa:`bug,text-danger` Prevent error during tilt-series alignment with AreTomo2 when number of patches = 1.
+
+   :fa:`bug,text-danger` Fix bug in command line interface that ocurred when launching constrained refinement.
+
+   :fa:`bug,text-danger` Fix bug that was causing the server startup routines to be called during the CLI processing of the configuration file.
+
+   :fa:`bug,text-danger` Fix bug that ocurred when retrieving metadata from mdoc files.
+
+   :fa:`bug,text-danger` Fix bug when trying to retrieve tilt-series metadata from failed runs.
+
+   :fa:`bug,text-danger` Fix conflicts with library paths when running external executables.
+
 v0.6.4 (3/24/2024)
 ******************
 

nextpyp.toml

@@ -3,4 +3,4 @@
 # It is shown directly to the user in micromon's user interface.
 # To the extent that is feasible, it should follow Semantic Versioning conventions:
 # https://semver.org/
-version = "0.6.4"
+version = "0.6.5"

src/pyp/align/core.py

@@ -4176,7 +4176,7 @@ def align_movie_super(parameters, name, suffix, isfirst = False):
         # patch tracking
         patches_x = parameters["movie_motioncor_patch_x"] if "movie_motioncor_patch_x" in parameters else 1
         patches_y = parameters["movie_motioncor_patch_y"] if "movie_motioncor_patch_y" in parameters else 1
-        if patches_x > 1 or patches_y > 0:
+        if patches_x + patches_y > 2:
             patches = f" -Patch {parameters['movie_motioncor_patch_x']} {parameters['movie_motioncor_patch_y']}"
             if parameters.get("movie_motioncor_patch_overlap"):
                 patches += f" {parameters['movie_motioncor_patch_overlap']}"
@@ -5021,8 +5021,11 @@ def align_tilt_series(name, parameters, rotation=0):
             tilt_offset_option = "1" if parameters['tomo_ali_aretomo_measure_tiltoff'] else f"1 {parameters['tomo_ali_aretomo_tiltoff']}"
 
             # local motion by giving the number of patches
-            if parameters.get("tomo_ali_patches_x") and parameters.get("tomo_ali_patches_y"):
-                patches = f" -Patch {parameters['tomo_ali_patches_x']} {parameters['tomo_ali_patches_y']}"
+            # patch tracking
+            patches_x = parameters["tomo_ali_patches_x"] if "tomo_ali_patches_x" in parameters else 1
+            patches_y = parameters["tomo_ali_patches_y"] if "tomo_ali_patches_y" in parameters else 1
+            if patches_x + patches_y > 2:
+                patches = f" -Patch {patches_x} {patches_y}"
             else:
                 patches = ""
 
@@ -5193,7 +5196,7 @@ def align_tilt_series(name, parameters, rotation=0):
 -AngFile {name}.rawtlt \
 -VolZ {thickness} \
 -OutBin {binning_tomo} \
--TiltAxis {parameters['scope_tilt_axis']} \
+-TiltAxis {rotation} \
 -DarkTol {parameters['tomo_ali_aretomo_dark_tol']} \
 -AlignZ {specimen_thickness} \
 {reconstruct_option} \

src/pyp/preprocess/core.py

@@ -142,14 +142,22 @@ def read_tilt_series(
     aligned_tilts = []
 
     data_path = Path(resolve_path(parameters["data_path"])).parent
-    mdoc_path = Path(resolve_path(parameters["data_path_mdoc"])).parent if "data_path_mdoc" in parameters else Path()
+    mdoc_path = Path(resolve_path(parameters["data_path_mdoc"])).parent if "data_path_mdoc" in parameters and parameters["data_path_mdoc"] != None else None
     project_raw_path = Path(filename).parent
 
     name = os.path.basename(filename)
-    mdocs = list(mdoc_path.glob(f"{name}.mdoc")) + list(mdoc_path.glob(f"{name}.mrc.mdoc"))
+    mdoc_pattern = "*.mdoc"
+
+    mdocs = []
+    if mdoc_path is not None:
+        mdoc_pattern = Path(resolve_path(parameters["data_path_mdoc"])).name
+        mdocs = list(mdoc_path.glob(str(mdoc_pattern)))
+        mdocs = [str(file) for file in mdocs if str(file.name).replace(".mrc", "").replace(".mdoc", "") == name]
+
     if len(mdocs) == 0:
         # get the mdoc files from the path of raw data if it couldn't find them in mdoc path
-        mdocs = list(data_path.glob(f"{name}.mdoc")) + list(mdoc_path.glob(f"{name}.mrc.mdoc"))
+        mdocs = list(data_path.glob(mdoc_pattern))
+        mdocs = [str(file) for file in mdocs if str(file.name).replace(".mrc", "").replace(".mdoc", "") == name]
 
     # escape special character in case it contains [
     filename = glob.escape(filename)
@@ -610,7 +618,7 @@ def read_tilt_series(
         if metadata.get("drift"):
             for i in metadata["drift"]:
                 drift_metadata["drift"][i] = metadata["drift"][i].to_numpy()[:,-2:]
-        elif metadata.get("web").get("drift"):
+        elif metadata.get("web") and metadata.get("web").get("drift"):
             for i in metadata.get("web")["drift"]:
                 drift_metadata["drift"][i] = metadata.get("web")["drift"][i]
     else:

src/pyp/streampyp/jobs.py

@@ -129,7 +129,7 @@ def submit_commands(
             cmdlist.append(
                 "export csp_local_merge=csp_local_merge; {0} --stacks_files stacks.txt --par_files pars.txt --ordering_file ordering.txt --project_path_file project_dir.txt --output_basename $OUTPUT_BASENAME --path '{1}/$OUTPUT_BASENAME'\n".format(
                     run_pyp(command="pyp", script=True, cpus=threads),
-                    Path(get_pyp_configuration()["pyp"]["scratch"]),
+                    Path(os.environ["PYP_SCRATCH"]),
                 ),
             )
 

src/pyp/system/project_params.py

@@ -249,14 +249,16 @@ def create_micrographs_list(parameters):
 
         # look for mdoc files in mdoc folder
         mdocs = list()
-        if "data_path_mdoc" in parameters and parameters["data_path_mdoc"] != None and Path(resolve_path(parameters["data_path_mdoc"])).exists:
-            mdoc_folder = Path(resolve_path(parameters["data_path_mdoc"])).parent
-            mdocs = list(mdoc_folder.glob("*.mdoc"))
+        mdoc_pattern = "*.mdoc"
+        if "data_path_mdoc" in parameters and parameters["data_path_mdoc"] != None:
+            mdoc_folder = parameters["data_path_mdoc"].parent
+            mdoc_pattern = parameters["data_path_mdoc"].name
+            mdocs = list(mdoc_folder.glob(str(mdoc_pattern)))
         # if none found, look in raw data folder
         if len(mdocs) == 0:
             data_path = Path(resolve_path(parameters["data_path"]))
             data_folder = data_path.parent
-            mdocs = list(data_folder.glob("*.mdoc"))
+            mdocs = list(data_folder.glob(mdoc_pattern))
 
         if parameters["data_mode"] == "tomo": 
             if not parameters["movie_mdoc"] and len(parameters["movie_pattern"]) > 0 and len(glob.glob("raw/*" + movie_extension)) > 0:

src/pyp/system/utils.py

@@ -44,9 +44,9 @@ def phenix_load_command():
 def get_slurm_path():
     return "/opt/slurm/bin/"
 
-
 def get_imod_path():
-    return "/opt/IMOD".format(os.environ["PYP_DIR"])
+    return "/opt/IMOD"
+    # return "export LD_LIBRARY_PATH=/opt/IMOD/qtlib:/opt/IMOD/lib:$LD_LIBRARY_PATH /opt/IMOD"
 
 def cuda_path_prefix(command):
     config = get_pyp_configuration()

src/pyp_main.py

@@ -1335,7 +1335,7 @@ def tomo_swarm(project_path, filename, debug = False, keep = False, skip = False
     os.chdir(project_path)
 
     parameters = project_params.load_pyp_parameters()
-
+    
     # get file name
     name = os.path.basename(filename)
 
@@ -3230,9 +3230,7 @@ def clear_scratch(scratch,timeout=60):
             )
         else:
             logger.info(
-                "Job {} (v{}) launching on {} using {} task(s) {}".format(
-                jobid, version, socket.gethostname(), mpi_tasks, memory
-                )
+                f"nextPYP v{version} launching job {jobid} on {socket.gethostname()} using {mpi_tasks} task(s) {memory}"
             )
 
         config = get_pyp_configuration()
@@ -3269,12 +3267,6 @@ def clear_scratch(scratch,timeout=60):
         # TODO: switch to pyp.system.utils.get_imod_path()
         os.environ["IMAGICDIR"] = "/usr/bin"
         os.environ["IMOD_DIR"] = get_imod_path()
-        if "LD_LIBRARY_PATH" in os.environ:
-            os.environ["LD_LIBRARY_PATH"] = "{0}/qtlib:{0}/lib:{1}".format(
-                get_imod_path(), os.environ["LD_LIBRARY_PATH"]
-            )
-        else:
-            os.environ["LD_LIBRARY_PATH"] = "{0}/qtlib:{0}/lib".format(get_imod_path())
 
         os.environ["LD_LIBRARY_PATH"] = "{0}:{1}".format(
                 os.environ["LD_LIBRARY_PATH"], '/usr/local/pkgs/fftw-3.3.10-nompi_hf0379b8_106/lib/'