Additional changes in NN-picking routines to deal with names with special characters

Author: abartesaghi

src/pyp/detect/joint.py

@@ -20,7 +20,7 @@
 logger = initialize_pyp_logger(log_name=relative_path)
 
 def bin_image( input, output, binning, verbose ):
-    command = f"{get_imod_path()}/bin/newstack {input} {output} -bin {binning}; rm -f {output}~"
+    command = f"{get_imod_path()}/bin/newstack '{input}' '{output}' -bin {binning}; rm -f '{output}~'"
     local_run.run_shell_command(command, verbose=verbose)
 
 def sprtrain(args):
@@ -72,7 +72,7 @@ def sprtrain(args):
     os.chdir(scratch_train)
 
     logger.info(f"Training pyp model")
-    command = f"export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python; export PYTHONPATH=$PYTHONPATH:$PYP_DIR/external/spr_pick; python {os.environ['PYP_DIR']}/external/spr_pick/spr_pick/__main__.py train start --algorithm {args['detect_nn2d_algorithm']} --noise_value {args['detect_nn2d_noise_value']} --noise_style {args['detect_nn2d_noise_style']} --tau {args['detect_nn2d_tau']} --runs_dir {runs_dir} --train_dataset {train_images} --train_label {train_coords} --iterations {args['detect_nn2d_iterations']} --alpha {args['detect_nn2d_alpha']} --train_batch_size {args['detect_nn2d_batch_size']} --nms {args['detect_dist']} --num {args['detect_nn2d_num']} --bb {args['detect_nn2d_bb']} --patch_size {args['detect_nn2d_patch_size']} --validation_dataset {validation_images} --validation_label {validation_coords} 2>&1 | tee {os.path.join(os.getcwd(), 'log', time_stamp + '_spr_pick_train.log')}"
+    command = f"export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python; export PYTHONPATH=$PYTHONPATH:$PYP_DIR/external/spr_pick; python {os.environ['PYP_DIR']}/external/spr_pick/spr_pick/__main__.py train start --algorithm {args['detect_nn2d_algorithm']} --noise_value {args['detect_nn2d_noise_value']} --noise_style {args['detect_nn2d_noise_style']} --tau {args['detect_nn2d_tau']} --runs_dir '{runs_dir}' --train_dataset '{train_images}' --train_label '{train_coords}' --iterations {args['detect_nn2d_iterations']} --alpha {args['detect_nn2d_alpha']} --train_batch_size {args['detect_nn2d_batch_size']} --nms {args['detect_dist']} --num {args['detect_nn2d_num']} --bb {args['detect_nn2d_bb']} --patch_size {args['detect_nn2d_patch_size']} --validation_dataset '{validation_images}' --validation_label '{validation_coords}' 2>&1 | tee {os.path.join(os.getcwd(), 'log', time_stamp + '_spr_pick_train.log')}"
     local_run.run_shell_command(command, verbose=args['slurm_verbose'])
 
     # check for failure if not output was produced
@@ -103,7 +103,7 @@ def spreval(args,name):
 
     if 'detect_nn2d_ref' in args.keys() and os.path.exists( project_params.resolve_path(args['detect_nn2d_ref']) ):
         logger.info(f"Evaluating using model: {Path(project_params.resolve_path(args['detect_nn2d_ref'])).name}")
-        command = f"export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python; export PYTHONPATH=$PYTHONPATH:$PYP_DIR/external/spr_pick; python {os.environ['PYP_DIR']}/external/spr_pick/spr_pick/__main__.py eval --model {project_params.resolve_path(args['detect_nn2d_ref'])} --dataset {os.path.join( os.getcwd(), imgs_file)} --runs_dir {os.getcwd()} --num 1"
+        command = f"export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python; export PYTHONPATH=$PYTHONPATH:$PYP_DIR/external/spr_pick; python {os.environ['PYP_DIR']}/external/spr_pick/spr_pick/__main__.py eval --model '{project_params.resolve_path(args['detect_nn2d_ref'])}' --dataset '{os.path.join( os.getcwd(), imgs_file)}' --runs_dir '{os.getcwd()}' --num 1"
         local_run.run_shell_command(command, verbose=args['slurm_verbose'])
         results_folder = glob.glob("./*/")[0]
 
@@ -238,7 +238,7 @@ def tomotrain(args):
     else:
         compress = ""
 
-    command = f"export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python; export PYTHONPATH=$PYTHONPATH:$PYP_DIR/external/cet_pick; python {os.environ['PYP_DIR']}/external/cet_pick/cet_pick/main.py semi --down_ratio {args['detect_nn3d_down_ratio']} {compress} --num_epochs {args['detect_nn3d_num_epochs']} --bbox {args['detect_nn3d_bbox']} --contrastive --exp_id test_reprod --dataset semi --arch unet_4 {debug} --val_interval {args['detect_nn3d_val_interval']} --thresh {args['detect_nn3d_thresh']} --cr_weight {args['detect_nn3d_cr_weight']} --temp {args['detect_nn3d_temp']} --tau {args['detect_nn3d_tau']} --K {args['detect_nn3d_max_objects']} --lr {args['detect_nn3d_lr']} --train_img_txt {train_images} --train_coord_txt {train_coords} --val_img_txt {validation_images} --val_coord_txt {validation_coords} --test_img_txt {validation_images} --test_coord_txt {validation_coords} 2>&1 | tee {os.path.join( os.getcwd(), 'log', time_stamp + '_cet_pick_train.log')}"
+    command = f"export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python; export PYTHONPATH=$PYTHONPATH:$PYP_DIR/external/cet_pick; python {os.environ['PYP_DIR']}/external/cet_pick/cet_pick/main.py semi --down_ratio {args['detect_nn3d_down_ratio']} {compress} --num_epochs {args['detect_nn3d_num_epochs']} --bbox {args['detect_nn3d_bbox']} --contrastive --exp_id test_reprod --dataset semi --arch unet_4 {debug} --val_interval {args['detect_nn3d_val_interval']} --thresh {args['detect_nn3d_thresh']} --cr_weight {args['detect_nn3d_cr_weight']} --temp {args['detect_nn3d_temp']} --tau {args['detect_nn3d_tau']} --K {args['detect_nn3d_max_objects']} --lr {args['detect_nn3d_lr']} --train_img_txt '{train_images}' --train_coord_txt '{train_coords}' --val_img_txt '{validation_images}' --val_coord_txt '{validation_coords}' --test_img_txt '{validation_images}' --test_coord_txt '{validation_coords}' 2>&1 | tee {os.path.join( os.getcwd(), 'log', time_stamp + '_cet_pick_train.log')}"
     [ output, error ] = local_run.run_shell_command(command, verbose=args['slurm_verbose'])
 
     # display log if available
@@ -293,7 +293,7 @@ def tomoeval(args,name):
             compress = ""
 
 
-        command = f"export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python; export PYTHONPATH=$PYTHONPATH:$PYP_DIR/external/cet_pick; python {os.environ['PYP_DIR']}/external/cet_pick/cet_pick/test.py semi --gpus -1 --arch unet_4 --dataset semi_test --with_score --exp_id test_reprod --load_model {project_params.resolve_path(args['detect_nn3d_ref'])} {compress} --down_ratio 2 --contrastive --K {args['detect_nn3d_max_objects']} --out_thresh {args['detect_nn3d_thresh']} --test_img_txt {os.path.join( os.getcwd(), imgs_file)} --test_coord_txt {os.path.join( os.getcwd(), test_file)} 2>&1 | tee {os.path.join(project_folder, 'train', name + '_testing.log')}"
+        command = f"export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python; export PYTHONPATH=$PYTHONPATH:$PYP_DIR/external/cet_pick; python {os.environ['PYP_DIR']}/external/cet_pick/cet_pick/test.py semi --gpus -1 --arch unet_4 --dataset semi_test --with_score --exp_id test_reprod --load_model '{project_params.resolve_path(args['detect_nn3d_ref'])}' {compress} --down_ratio 2 --contrastive --K {args['detect_nn3d_max_objects']} --out_thresh {args['detect_nn3d_thresh']} --test_img_txt '{os.path.join( os.getcwd(), imgs_file)}' --test_coord_txt '{os.path.join( os.getcwd(), test_file)}' 2>&1 | tee '{os.path.join(project_folder, 'train', name + '_testing.log')}'"
         [ output, error ] = local_run.run_shell_command(command, verbose=args['slurm_verbose'])
         results_folder = os.getcwd()
 

src/pyp/detect/topaz.py

@@ -90,8 +90,8 @@ def sprtrain(args):
     command = f"{utils.get_topaz_path()}/topaz train \
 -n {args['detect_topaz2d_num_particles']} \
 --num-workers={args['slurm_tasks']} \
---train-images {scratch_train} \
---train-targets {train_coords} \
+--train-images '{scratch_train}' \
+--train-targets '{train_coords}' \
 {pretrained} \
 --method {args['detect_topaz2d_train_method']} \
 --num-epochs {args['detect_topaz2d_epochs']} \
@@ -104,7 +104,7 @@ def sprtrain(args):
 --minibatch-balance {args['detect_topaz2d_train_batchbalance']} \
 --epoch-size {args['detect_topaz2d_train_epochsize']} \
 --save-prefix=topaz_train \
--o model_training.txt 2>&1 | tee {os.path.join(train_folder, time_stamp + '_topaz_train.log')}"
+-o model_training.txt 2>&1 | tee '{os.path.join(train_folder, time_stamp + '_topaz_train.log')}'"
 # --model {args['detect_topaz2d_model']} \
 # --units {args['detect_topaz2d_units']} \
 # --dropout {args['detect_topaz2d_dropout']} \
@@ -135,7 +135,7 @@ def spreval(args,name):
 
     if 'detect_topaz2d_ref' in args.keys() and os.path.isfile(project_params.resolve_path(args['detect_topaz2d_ref'])):
         logger.info(f"Evaluating using model: {Path(project_params.resolve_path(args['detect_topaz2d_ref'])).name}")
-        model_arg = f"-m {project_params.resolve_path(args['detect_topaz2d_ref'])}"
+        model_arg = f"-m '{project_params.resolve_path(args['detect_topaz2d_ref'])}'"
     else:
         logger.info(f"Evaluating using pre-trained model")
         model_arg = f"--model {args['detect_topaz2d_pretrained_model']}"
@@ -173,9 +173,9 @@ def spreval(args,name):
 --max-radius {args['detect_topaz2d_extract_max_rad']} \
 --step-radius {args['detect_topaz2d_extract_step_rad']} \
 -x {binning} \
--o {coordinates_file} \
-{Path().cwd() / f'{name}_bin.mrc'} \
-2>&1 | tee {os.path.join(project_folder, 'log', time_stamp + '_topaz_extract.log')}"
+-o '{coordinates_file}' \
+'{Path().cwd() / f'{name}_bin.mrc'}' \
+2>&1 | tee '{os.path.join(project_folder, 'log', time_stamp + '_topaz_extract.log')}'"
 
     local_run.run_shell_command(command, verbose=args['slurm_verbose'])
 
@@ -216,7 +216,7 @@ def spreval(args,name):
         return np.array([])
 
 def bin_next_coordinates(coordinates,coordinates_bin, binning, verbose=False):
-    command = f"{utils.get_topaz_path()}/topaz convert -s {binning} -o {coordinates_bin} {coordinates}"
+    command = f"{utils.get_topaz_path()}/topaz convert -s {binning} -o '{coordinates_bin}' '{coordinates}'"
     local_run.run_shell_command(command, verbose=verbose)
 
 

src/pyp/refine/csp/particle_cspt.py

@@ -1721,7 +1721,7 @@ def csp_has_error(path_to_logs: Path, micrographs: dict) -> bool:
         micrograph_log = Path(path_to_logs, f"{micrograph}_csp.log")
         if micrograph_log.exists():
             # use "grep" to check if log files contain any error message
-            command = "grep -E %s %s" % ("'" + "|".join(ERROR_KEYWORDS) + "'", str(micrograph_log))
+            command = "grep -E %s '%s'" % ("'" + "|".join(ERROR_KEYWORDS) + "'", str(micrograph_log))
             [output, error] = local_run.run_shell_command(command, verbose=False)
 
             if len(output) > 0: