initial commit on using expand_list_file, not passing tests for file_list with missing files

Author: yucongalicechen

src/diffpy/labpdfproc/labpdfprocapp.py

@@ -1,9 +1,14 @@
 import sys
 from argparse import ArgumentParser
-from pathlib import Path
 
 from diffpy.labpdfproc.functions import apply_corr, compute_cve
-from diffpy.labpdfproc.tools import known_sources, load_user_metadata, set_output_directory, set_wavelength
+from diffpy.labpdfproc.tools import (
+    known_sources,
+    load_user_metadata,
+    set_input_lists,
+    set_output_directory,
+    set_wavelength,
+)
 from diffpy.utils.parsers.loaddata import loadData
 from diffpy.utils.scattering_objects.diffraction_objects import XQUANTITIES, Diffraction_object
 
@@ -89,45 +94,46 @@ def get_args(override_cli_inputs=None):
 
 def main():
     args = get_args()
+    args = set_input_lists(args)
     args.output_directory = set_output_directory(args)
     args.wavelength = set_wavelength(args)
     args = load_user_metadata(args)
 
-    filepath = Path(args.input_file)
-    outfilestem = filepath.stem + "_corrected"
-    corrfilestem = filepath.stem + "_cve"
-    outfile = args.output_directory / (outfilestem + ".chi")
-    corrfile = args.output_directory / (corrfilestem + ".chi")
+    for filepath in args.input_directory:
+        outfilestem = filepath.stem + "_corrected"
+        corrfilestem = filepath.stem + "_cve"
+        outfile = args.output_directory / (outfilestem + ".chi")
+        corrfile = args.output_directory / (corrfilestem + ".chi")
 
-    if outfile.exists() and not args.force_overwrite:
-        sys.exit(
-            f"Output file {str(outfile)} already exists. Please rerun "
-            f"specifying -f if you want to overwrite it."
-        )
-    if corrfile.exists() and args.output_correction and not args.force_overwrite:
-        sys.exit(
-            f"Corrections file {str(corrfile)} was requested and already "
-            f"exists. Please rerun specifying -f if you want to overwrite it."
-        )
+        if outfile.exists() and not args.force_overwrite:
+            sys.exit(
+                f"Output file {str(outfile)} already exists. Please rerun "
+                f"specifying -f if you want to overwrite it."
+            )
+        if corrfile.exists() and args.output_correction and not args.force_overwrite:
+            sys.exit(
+                f"Corrections file {str(corrfile)} was requested and already "
+                f"exists. Please rerun specifying -f if you want to overwrite it."
+            )
 
-    input_pattern = Diffraction_object(wavelength=args.wavelength)
-    xarray, yarray = loadData(args.input_file, unpack=True)
-    input_pattern.insert_scattering_quantity(
-        xarray,
-        yarray,
-        "tth",
-        scat_quantity="x-ray",
-        name=str(args.input_file),
-        metadata={"muD": args.mud, "anode_type": args.anode_type},
-    )
+        input_pattern = Diffraction_object(wavelength=args.wavelength)
+        xarray, yarray = loadData(args.input_file, unpack=True)
+        input_pattern.insert_scattering_quantity(
+            xarray,
+            yarray,
+            "tth",
+            scat_quantity="x-ray",
+            name=str(args.input_file),
+            metadata={"muD": args.mud, "anode_type": args.anode_type},
+        )
 
-    absorption_correction = compute_cve(input_pattern, args.mud, args.wavelength)
-    corrected_data = apply_corr(input_pattern, absorption_correction)
-    corrected_data.name = f"Absorption corrected input_data: {input_pattern.name}"
-    corrected_data.dump(f"{outfile}", xtype="tth")
+        absorption_correction = compute_cve(input_pattern, args.mud, args.wavelength)
+        corrected_data = apply_corr(input_pattern, absorption_correction)
+        corrected_data.name = f"Absorption corrected input_data: {input_pattern.name}"
+        corrected_data.dump(f"{outfile}", xtype="tth")
 
-    if args.output_correction:
-        absorption_correction.dump(f"{corrfile}", xtype="tth")
+        if args.output_correction:
+            absorption_correction.dump(f"{corrfile}", xtype="tth")
 
 
 if __name__ == "__main__":

src/diffpy/labpdfproc/tools.py

@@ -28,11 +28,20 @@ def set_output_directory(args):
     return output_dir
 
 
-def _parse_file_list_file(input_path):
-    with open(input_path, "r") as f:
-        lines = [line.strip() for line in f]
-        input_files = [Path(line).resolve() for line in lines if Path(line).is_file()]
-        return input_files
+def _parse_file_list_file(file_list_path):
+    with open(file_list_path, "r") as f:
+        # file_paths = [Path(file_path.strip()).resolve() for file_path in f.readlines()
+        # if Path(file_path.strip()).is_file()]
+        file_paths = [file_path.strip() for file_path in f.readlines()]
+        return file_paths
+
+
+def expand_list_file(input):
+    file_list_inputs = [input_name for input_name in input if "file_list" in str(input_name)]
+    for file_list_input in file_list_inputs:
+        input.remove(file_list_input)
+        input.extend(_parse_file_list_file(file_list_input))
+    return input
 
 
 def set_input_lists(args):
@@ -54,14 +63,12 @@ def set_input_lists(args):
     """
 
     input_paths = []
-    for input in args.input:
+    expanded_input = expand_list_file(args.input)
+    for input in expanded_input:
         input_path = Path(input).resolve()
         if input_path.exists():
             if input_path.is_file():
-                if "file_list" in input_path.name:
-                    input_paths.extend(_parse_file_list_file(input_path))
-                else:
-                    input_paths.append(input_path)
+                input_paths.append(input_path)
             elif input_path.is_dir():
                 input_files = input_path.glob("*")
                 input_files = [