Merge pull request #54 from diffpy/input

input

Author: sbillinge

src/diffpy/labpdfproc/tests/conftest.py

@@ -1,42 +1,46 @@
-import os
 from pathlib import Path
 
 import pytest
 
 
 @pytest.fixture
 def user_filesystem(tmp_path):
-    directory = Path(tmp_path)
-    os.chdir(directory)
-
-    input_dir = Path(tmp_path).resolve() / "input_dir"
+    base_dir = Path(tmp_path)
+    input_dir = base_dir / "input_dir"
     input_dir.mkdir(parents=True, exist_ok=True)
 
     chi_data = "dataformat = twotheta\n mode = xray\n # chi_Q chi_I\n 1 2\n 3 4\n 5 6\n 7 8\n"
     xy_data = "1 2\n 3 4\n 5 6\n 7 8"
-    unreadable_data = "This is an unreadable file."
+    unreadable_data = "This is a file with no data that is non-readable by " "LoadData"
     binary_data = b"\x00\x01\x02\x03\x04"
 
-    with open("good_data.chi", "w") as f:
+    with open(base_dir / "good_data.chi", "w") as f:
         f.write(chi_data)
-    with open("good_data.xy", "w") as f:
+    with open(base_dir / "good_data.xy", "w") as f:
         f.write(xy_data)
-    with open("good_data.txt", "w") as f:
+    with open(base_dir / "good_data.txt", "w") as f:
         f.write(chi_data)
-    with open("unreadable_file.txt", "w") as f:
+    with open(base_dir / "unreadable_file.txt", "w") as f:
         f.write(unreadable_data)
-    with open("binary.pkl", "wb") as f:
+    with open(base_dir / "binary.pkl", "wb") as f:
         f.write(binary_data)
 
-    with open(os.path.join(input_dir, "good_data.chi"), "w") as f:
+    with open(input_dir / "good_data.chi", "w") as f:
         f.write(chi_data)
-    with open(os.path.join(input_dir, "good_data.xy"), "w") as f:
+    with open(input_dir / "good_data.xy", "w") as f:
         f.write(xy_data)
-    with open(os.path.join(input_dir, "good_data.txt"), "w") as f:
+    with open(input_dir / "good_data.txt", "w") as f:
         f.write(chi_data)
-    with open(os.path.join(input_dir, "unreadable_file.txt"), "w") as f:
+    with open(input_dir / "unreadable_file.txt", "w") as f:
         f.write(unreadable_data)
-    with open(os.path.join(input_dir, "binary.pkl"), "wb") as f:
+    with open(input_dir / "binary.pkl", "wb") as f:
         f.write(binary_data)
 
+    with open(input_dir / "file_list.txt", "w") as f:
+        f.write("good_data.chi \n good_data.xy \n good_data.txt \n missing_file.txt")
+    with open(input_dir / "file_list_example2.txt", "w") as f:
+        f.write("input_dir/good_data.chi \n")
+        f.write("good_data.xy \n")
+        f.write(f"{str(input_dir.resolve() / 'good_data.txt')}\n")
+
     yield tmp_path

src/diffpy/labpdfproc/tests/test_fixtures.py

@@ -0,0 +1,18 @@
+import os
+
+import pytest
+
+from diffpy.utils.parsers import loadData
+
+
+# Test that our readable and unreadable files are indeed readable and
+# unreadable by loadData (which is our definition of readable and unreadable)
+def test_loadData_with_input_files(user_filesystem):
+    os.chdir(user_filesystem)
+    xarray_chi, yarray_chi = loadData("good_data.chi", unpack=True)
+    xarray_xy, yarray_xy = loadData("good_data.xy", unpack=True)
+    xarray_txt, yarray_txt = loadData("good_data.txt", unpack=True)
+    with pytest.raises(ValueError):
+        xarray_txt, yarray_txt = loadData("unreadable_file.txt", unpack=True)
+    with pytest.raises(ValueError):
+        xarray_pkl, yarray_pkl = loadData("binary.pkl", unpack=True)

src/diffpy/labpdfproc/tests/test_tools.py

@@ -1,10 +1,104 @@
+import os
 import re
 from pathlib import Path
 
 import pytest
 
 from diffpy.labpdfproc.labpdfprocapp import get_args
-from diffpy.labpdfproc.tools import known_sources, load_user_metadata, set_output_directory, set_wavelength
+from diffpy.labpdfproc.tools import (
+    known_sources,
+    load_user_metadata,
+    set_input_lists,
+    set_output_directory,
+    set_wavelength,
+)
+
+# Use cases can be found here: https://github.com/diffpy/diffpy.labpdfproc/issues/48
+
+# This test covers existing single input file, directory, a file list, and multiple files
+# We store absolute path into input_directory and file names into input_file
+params_input = [
+    (["good_data.chi"], ["good_data.chi"]),  # single good file, same directory
+    (["input_dir/good_data.chi"], ["input_dir/good_data.chi"]),  # single good file, input directory
+    (  # glob current directory
+        ["."],
+        ["good_data.chi", "good_data.xy", "good_data.txt", "unreadable_file.txt", "binary.pkl"],
+    ),
+    (  # glob input directory
+        ["./input_dir"],
+        [
+            "input_dir/good_data.chi",
+            "input_dir/good_data.xy",
+            "input_dir/good_data.txt",
+            "input_dir/unreadable_file.txt",
+            "input_dir/binary.pkl",
+        ],
+    ),
+    (  # glob list of input directories
+        [".", "./input_dir"],
+        [
+            "./good_data.chi",
+            "./good_data.xy",
+            "./good_data.txt",
+            "./unreadable_file.txt",
+            "./binary.pkl",
+            "input_dir/good_data.chi",
+            "input_dir/good_data.xy",
+            "input_dir/good_data.txt",
+            "input_dir/unreadable_file.txt",
+            "input_dir/binary.pkl",
+        ],
+    ),
+    (  # file_list.txt list of files provided
+        ["input_dir/file_list.txt"],
+        ["good_data.chi", "good_data.xy", "good_data.txt"],
+    ),
+    (  # file_list_example2.txt list of files provided in different directories
+        ["input_dir/file_list_example2.txt"],
+        ["input_dir/good_data.chi", "good_data.xy", "input_dir/good_data.txt"],
+    ),
+]
+
+
+@pytest.mark.parametrize("inputs, expected", params_input)
+def test_set_input_lists(inputs, expected, user_filesystem):
+    base_dir = Path(user_filesystem)
+    os.chdir(base_dir)
+    expected_paths = [base_dir.resolve() / expected_path for expected_path in expected]
+
+    cli_inputs = ["2.5"] + inputs
+    actual_args = get_args(cli_inputs)
+    actual_args = set_input_lists(actual_args)
+    assert list(actual_args.input_paths).sort() == expected_paths.sort()
+
+
+# This test covers non-existing single input file or directory, in this case we raise an error with message
+params_input_bad = [
+    (
+        ["non_existing_file.xy"],
+        "Cannot find non_existing_file.xy. Please specify valid input file(s) or directories.",
+    ),
+    (
+        ["./input_dir/non_existing_file.xy"],
+        "Cannot find ./input_dir/non_existing_file.xy. Please specify valid input file(s) or directories.",
+    ),
+    (["./non_existing_dir"], "Cannot find ./non_existing_dir. Please specify valid input file(s) or directories."),
+    (  # list of files provided (with missing files)
+        ["good_data.chi", "good_data.xy", "unreadable_file.txt", "missing_file.txt"],
+        "Cannot find missing_file.txt. Please specify valid input file(s) or directories.",
+    ),
+]
+
+
+@pytest.mark.parametrize("inputs, msg", params_input_bad)
+def test_set_input_files_bad(inputs, msg, user_filesystem):
+    base_dir = Path(user_filesystem)
+    os.chdir(base_dir)
+    cli_inputs = ["2.5"] + inputs
+    actual_args = get_args(cli_inputs)
+    with pytest.raises(FileNotFoundError, match=msg[0]):
+        actual_args = set_input_lists(actual_args)
+
 
 params1 = [
     ([], ["."]),
@@ -16,6 +110,7 @@
 
 @pytest.mark.parametrize("inputs, expected", params1)
 def test_set_output_directory(inputs, expected, user_filesystem):
+    os.chdir(user_filesystem)
     expected_output_directory = Path(user_filesystem) / expected[0]
     cli_inputs = ["2.5", "data.xy"] + inputs
     actual_args = get_args(cli_inputs)
@@ -26,6 +121,7 @@ def test_set_output_directory(inputs, expected, user_filesystem):
 
 
 def test_set_output_directory_bad(user_filesystem):
+    os.chdir(user_filesystem)
     cli_inputs = ["2.5", "data.xy", "--output-directory", "good_data.chi"]
     actual_args = get_args(cli_inputs)
     with pytest.raises(FileExistsError):

src/diffpy/labpdfproc/tools.py

@@ -28,6 +28,42 @@ def set_output_directory(args):
     return output_dir
 
 
+def set_input_lists(args):
+    """
+    Set input directory and files.
+
+    It takes cli inputs, checks if they are files or directories and creates
+    a list of files to be processed which is stored in the args Namespace.
+
+    Parameters
+    ----------
+    args argparse.Namespace
+        the arguments from the parser
+
+    Returns
+    -------
+    args argparse.Namespace
+
+    """
+
+    input_paths = []
+    for input in args.input:
+        input_path = Path(input).resolve()
+        if input_path.exists():
+            if input_path.is_file():
+                input_paths.append(input_path)
+            elif input_path.is_dir():
+                input_files = input_path.glob("*")
+                input_files = [file.resolve() for file in input_files if file.is_file()]
+                input_paths.extend(input_files)
+            else:
+                raise FileNotFoundError(f"Cannot find {input}. Please specify valid input file(s) or directories.")
+        else:
+            raise FileNotFoundError(f"Cannot find {input}")
+    setattr(args, "input_paths", input_paths)
+    return args
+
+
 def set_wavelength(args):
     """
     Set the wavelength based on the given input arguments