added tests for file list and multiple files

Author: yucongalicechen

src/diffpy/labpdfproc/labpdfprocapp.py

@@ -19,10 +19,9 @@ def get_args(override_cli_inputs=None):
     p.add_argument("mud", help="Value of mu*D for your " "sample. Required.", type=float)
     p.add_argument(
         "input",
+        nargs="+",
         help="The filename or directory of the datafile to load. Required. "
-        "Supports either a single input file or directory, or a file containing a list of files. "
-        "If providing a file list, please ensure all files are in the same directory as the file list, "
-        "and each filename is written line by line in the file list. ",
+        "Supports either a single input file, a directory, a file containing a list of files, or multiple files. ",
     )
     p.add_argument(
         "-a",

src/diffpy/labpdfproc/tests/conftest.py

@@ -42,10 +42,10 @@ def user_filesystem(tmp_path):
     file_list_dir = Path(tmp_path).resolve() / "file_list_dir"
     file_list_dir.mkdir(parents=True, exist_ok=True)
     with open(os.path.join(file_list_dir, "file_list.txt"), "w") as f:
-        f.write("good_data.chi \n good_data.xy \n good_data.txt")
-    with open(os.path.join(file_list_dir, "invalid_file_list.txt"), "w") as f:
-        f.write("good_data.chi \n non_existing_file.xy \n non_existing_file.txt")
-    with open(os.path.join(file_list_dir, "invalid_format_file_list.txt"), "w") as f:
-        f.write("good_data.chi good_data.xy \n non_existing_file.txt")
+        f.write("good_data.chi \n good_data.xy \n good_data.txt \n missing_file.txt")
+    with open(os.path.join(file_list_dir, "file_list_example2.txt"), "w") as f:
+        f.write("input_dir/good_data.chi \n")
+        f.write("good_data.xy \n")
+        f.write(str(os.path.abspath(os.path.join(input_dir, "good_data.txt"))) + "\n")
 
     yield tmp_path

src/diffpy/labpdfproc/tests/test_tools.py

@@ -16,34 +16,47 @@
 
 # Use cases can be found here: https://github.com/diffpy/diffpy.labpdfproc/issues/48
 
-# This test covers existing single input file, directory, or a file list
-# We store absolute path into input_directory and file names into input_file
+# This test covers existing single input file, directory, a file list, and multiple files
+# We store absolute paths into input_directory and file names into input_file
 params_input = [
-    (["good_data.chi"], [".", "good_data.chi"]),
-    (["input_dir/good_data.chi"], ["input_dir", "good_data.chi"]),
-    (["./input_dir/good_data.chi"], ["input_dir", "good_data.chi"]),
-    (
+    (["good_data.chi"], [".", "good_data.chi"]),  # single good file, same directory
+    (["input_dir/good_data.chi"], ["input_dir", "good_data.chi"]),  # single good file, input directory
+    (  # glob current directory
         ["."],
         [
             ".",
             ["good_data.chi", "good_data.xy", "good_data.txt", "unreadable_file.txt", "binary.pkl"],
         ],
     ),
-    (
+    (  # glob input directory
         ["./input_dir"],
         [
             "input_dir",
             ["good_data.chi", "good_data.xy", "good_data.txt", "unreadable_file.txt", "binary.pkl"],
         ],
     ),
-    (
-        ["input_dir"],
+    (  # list of files provided (we skip if encountering an invalid files)
+        ["good_data.chi", "good_data.xy", "unreadable_file.txt", "missing_file.txt"],
         [
-            "input_dir",
-            ["good_data.chi", "good_data.xy", "good_data.txt", "unreadable_file.txt", "binary.pkl"],
+            ".",
+            ["good_data.chi", "good_data.xy", "unreadable_file.txt"],
         ],
     ),
-    (["file_list_dir/file_list.txt"], ["file_list_dir", ["good_data.chi", "good_data.xy", "good_data.txt"]]),
+    (  # list of files provided (with invalid files and files in different directory)
+        ["input_dir/good_data.chi", "good_data.xy", "missing_file.txt"],
+        [
+            ".",
+            ["input_dir/good_data.chi", "good_data.xy"],
+        ],
+    ),
+    (  # file_list.txt list of files provided
+        ["file_list_dir/file_list.txt"],
+        [".", ["good_data.chi", "good_data.xy", "good_data.txt"]],
+    ),
+    (  # file_list_example2.txt list of files provided with different paths
+        ["file_list_dir/file_list_example2.txt"],
+        [".", ["input_dir/good_data.chi", "good_data.xy", "input_dir/good_data.txt"]],
+    ),
 ]
 
 

src/diffpy/labpdfproc/tools.py

@@ -15,39 +15,55 @@ def set_input_files(args):
     args argparse.Namespace
         the arguments from the parser
 
-    It is implemented as this:
+    It is implemented as the following:
+    If user input multiple files, we store their common directory as input directory and all of their names.
     If input is a file, we first try to read it as a file list and store all listed file names.
-    If any filename is invalid, then proceed to treat it as a data file.
+    If the first filename is invalid, then we proceed to treat it as a data file.
     Otherwise if we have a directory, glob all files within it.
+    If there are any invalid filenames (for the cases of multiple files, file list, or directory), we skip them.
 
     Returns
     -------
     args argparse.Namespace
 
     """
 
-    if not Path(args.input).exists():
+    if len(args.input) > 1:
+        input_paths = []
+        input_paths_parent = []
+        for input in args.input:
+            if Path(input).is_file():
+                input_paths.append(Path(input).resolve())
+                input_paths_parent.append(Path(input).resolve().parent)
+        input_dir = Path(os.path.commonprefix([str(path) for path in input_paths_parent]))
+        input_file_name = [str(path.relative_to(input_dir)) for path in input_paths]
+        setattr(args, "input_directory", input_dir)
+        setattr(args, "input_file", input_file_name)
+        return args
+
+    if not Path(args.input[0]).exists():
         raise ValueError("Please specify valid input file or directory.")
 
-    if not Path(args.input).is_dir():
-        input_dir = Path.cwd() / Path(args.input).parent
-        file_names = []
-        with open(args.input, "r") as f:
-            for line in f:
-                if not os.path.isfile(line.strip()):
-                    file_names = []
-                    break
-                else:
-                    file_name = line.strip()
-                    file_names.append(file_name)
-
-        if len(file_names) > 0:
-            input_file_name = file_names
-        else:
-            input_file_name = Path(args.input).name
+    if not Path(args.input[0]).is_dir():
+        input_paths = []
+        input_paths_parent = []
+        with open(args.input[0], "r") as f:
+            lines = [line.strip() for line in f]
+            if not os.path.isfile(lines[0]):
+                input_dir = Path.cwd() / Path(args.input[0]).parent
+                input_file_name = Path(args.input[0]).name
+            else:
+                for line in lines:
+                    if not os.path.isfile(line):
+                        continue
+                    else:
+                        input_paths.append(Path(line).resolve())
+                        input_paths_parent.append(Path(line).resolve().parent)
+                input_dir = Path(os.path.commonprefix([str(path) for path in input_paths_parent]))
+                input_file_name = [str(path.relative_to(input_dir)) for path in input_paths]
 
     else:
-        input_dir = Path(args.input).resolve()
+        input_dir = Path(args.input[0]).resolve()
         input_files = [file for file in glob.glob(str(input_dir) + "/*", recursive=True) if os.path.isfile(file)]
         input_file_name = [os.path.basename(input_file_path) for input_file_path in input_files]