[Enabler] [module_utils/copy.py] Replace calls to cp for dcp's Python API (#1831)

* Replace calls to cp for dcp

* Replace copy functions in copy.py

* Replace copy calls in modules

* Fix typo

* Fix return statement

* Add changelog fragment

* Fix sanity issues

Author: rexemin

changelogs/fragments/1831-replace-copy-commands-copy-util.yml

@@ -0,0 +1,4 @@
+trivial:
+  - module_utils/copy.py - Replaced calls to cp with calls to ZOAU's
+    datasets' Python API.
+    (https://github.com/ansible-collections/ibm_zos_core/pull/1831)

plugins/module_utils/copy.py

@@ -29,13 +29,12 @@
 from ansible_collections.ibm.ibm_zos_core.plugins.module_utils.import_handler import \
     ZOAUImportError
 
-from shlex import quote
-
 try:
-    from zoautil_py import datasets, gdgs
+    from zoautil_py import datasets, gdgs, exceptions as zoau_exceptions
 except Exception:
     datasets = ZOAUImportError(traceback.format_exc())
     gdgs = ZOAUImportError(traceback.format_exc())
+    zoau_exceptions = ZOAUImportError(traceback.format_exc())
 
 REPRO = """  REPRO INDATASET({}) -
     OUTDATASET({}) REPLACE """
@@ -79,146 +78,51 @@ def _validate_path(path):
     return parsed_args.get("path")
 
 
-def copy_uss2mvs(src, dest, ds_type, is_binary=False):
-    """Copy uss a file or path to an MVS data set.
+def copy_uss_mvs(src, dest, is_binary=False):
+    """Wrapper function for datasets.copy that handles possible
+    exceptions that may occur.
 
     Parameters
     ----------
     src : str
-        The uss file or path to be copied.
+        Source dataset or file.
     dest : str
-        The destination MVS data set, it must be a PS or PDS(E).
-    ds_type : str
-        The dsorg of the dest.
+        Destination dataset name or file.
 
     Keyword Parameters
     ------------------
     is_binary : bool
-        Whether the file to be copied contains binary data.
+        Whether to perform a binary copy.
 
     Returns
     -------
-    bool
-        The return code after the copy command executed successfully.
-    str
-        The stdout after the copy command executed successfully.
-    str
-        The stderr after the copy command executed successfully.
+    tuple
+        Tuple containing return code, standard output and standard
+        error from the datasets API.
 
     Raises
     ------
-    USSCmdExecError
-        When any exception is raised during the conversion.
+    ZOAUImportError
+        When there's an issue calling the datasets API.
     """
-    module = AnsibleModuleHelper(argument_spec={})
-    src = _validate_path(src)
-    dest = _validate_data_set_name(dest)
-    if ds_type == "PO":
-        cp_uss2mvs = "cp -CM -F rec {0} \"//'{1}'\"".format(quote(src), dest)
-    else:
-        cp_uss2mvs = "cp -F rec {0} \"//'{1}'\"".format(quote(src), dest)
-    if is_binary:
-        cp_uss2mvs = cp_uss2mvs.replace("rec", "bin", 1)
-    rc, out, err = module.run_command(cp_uss2mvs, errors='replace')
-    if rc:
-        raise USSCmdExecError(cp_uss2mvs, rc, out, err)
-    return rc, out, err
-
-
-def copy_ps2uss(src, dest, is_binary=False):
-    """Copy a PS data set to a uss file.
-
-    Parameters
-    ----------
-    src : str
-        The MVS data set to be copied, it must be a PS data set
-        or a PDS(E) member.
-    dest : str
-        The destination uss file.
-
-    Keyword Parameters
-    ------------------
-    is_binary : bool
-        Whether the file to be copied contains binary data.
-
-    Returns
-    -------
-    bool
-        The return code after the copy command executed successfully.
-    str
-        The stdout after the copy command executed successfully.
-    str
-        The stderr after the copy command executed successfully.
+    copy_args = {
+        "options": ""
+    }
 
-    Raises
-    ------
-    USSCmdExecError
-        When any exception is raised during the conversion.
-    """
-    module = AnsibleModuleHelper(argument_spec={})
-    src = _validate_data_set_name(src)
-    dest = _validate_path(dest)
-    cp_ps2uss = "cp -F rec \"//'{0}'\" {1}".format(src, quote(dest))
     if is_binary:
-        cp_ps2uss = cp_ps2uss.replace("rec", "bin", 1)
-    rc, out, err = module.run_command(cp_ps2uss, errors='replace')
-    if rc:
-        raise USSCmdExecError(cp_ps2uss, rc, out, err)
-    return rc, out, err
-
-
-def copy_pds2uss(src, dest, is_binary=False, asa_text=False):
-    """Copy the whole PDS(E) to a uss path.
-
-    Parameters
-    ----------
-    src : str
-        The MVS data set to be copied, it must be a PDS(E) data set.
-    dest : str
-        The destination uss path.
-
-    Keyword Parameters
-    ------------------
-    is_binary : bool
-        Whether the file to be copied contains binary data.
-    asa_text : bool
-        Whether the file to be copied contains ASA control
-        characters.
-
-    Returns
-    -------
-    bool
-        The return code after the USS command executed successfully.
-    str
-        The stdout after the USS command executed successfully.
-    str
-        The stderr after the USS command executed successfully.
-
-    Raises
-    ------
-    USSCmdExecError
-        When any exception is raised during the conversion.
-    """
-    module = AnsibleModuleHelper(argument_spec={})
-    src = _validate_data_set_name(src)
-    dest = _validate_path(dest)
-
-    cp_pds2uss = "cp -U -F rec \"//'{0}'\" {1}".format(src, quote(dest))
-
-    # When dealing with ASA control chars, each record follows a
-    # different format than what '-F rec' means, so we remove it
-    # to allow the system to leave the control chars in the
-    # destination.
-    if asa_text:
-        cp_pds2uss = cp_pds2uss.replace("-F rec", "", 1)
-    elif is_binary:
-        cp_pds2uss = cp_pds2uss.replace("rec", "bin", 1)
+        copy_args["options"] = "-B"
 
-    rc, out, err = module.run_command(cp_pds2uss, errors='replace')
-    if rc:
-        raise USSCmdExecError(cp_pds2uss, rc, out, err)
+    try:
+        datasets.copy(source=src, target=dest, **copy_args)
+    except zoau_exceptions.ZOAUException as copy_exception:
+        # Returning the exception content instead of raising it
+        # since a lot of code that uses this function expects it
+        # so they can decide what to do in case of an error.
+        return copy_exception.response.rc, \
+            copy_exception.response.stdout_response, \
+            copy_exception.response.stderr_response
 
-    return rc, out, err
+    return 0, "", ""
 
 
 def copy_gdg2uss(src, dest, is_binary=False, asa_text=False):
@@ -264,81 +168,6 @@ def copy_gdg2uss(src, dest, is_binary=False, asa_text=False):
     return True
 
 
-def copy_uss2uss_binary(src, dest):
-    """Copy a USS file to a USS location in binary mode.
-
-    Parameters
-    ----------
-    src : str
-        The source USS path.
-    dest : str
-        The destination USS path.
-
-    Returns
-    -------
-    bool
-        The return code after the USS command executed successfully.
-    str
-        The stdout after the USS command executed successfully.
-    str
-        The stderr after the USS command executed successfully.
-
-    Raises
-    ------
-    USSCmdExecError
-        When any exception is raised during the conversion.
-    """
-    module = AnsibleModuleHelper(argument_spec={})
-    src = _validate_path(src)
-    dest = _validate_path(dest)
-    cp_uss2uss = "cp -F bin {0} {1}".format(quote(src), quote(dest))
-    rc, out, err = module.run_command(cp_uss2uss, errors='replace')
-    if rc:
-        raise USSCmdExecError(cp_uss2uss, rc, out, err)
-    return rc, out, err
-
-
-def copy_mvs2mvs(src, dest, is_binary=False):
-    """Copy an MVS source to MVS target.
-
-    Parameters
-    ----------
-    src : str
-        Name of source data set.
-    dest : str
-        Name of destination data set.
-
-    Keyword Parameters
-    ------------------
-    is_binary : bool
-        Whether the data set to be copied contains binary data.
-
-    Returns
-    -------
-    bool
-        The return code after the USS command executed successfully.
-    str
-        The stdout after the USS command executed successfully.
-    str
-        The stderr after the USS command executed successfully.
-
-    Raises
-    ------
-    USSCmdExecError
-        When any exception is raised during the conversion.
-    """
-    module = AnsibleModuleHelper(argument_spec={})
-    src = _validate_data_set_name(src)
-    dest = _validate_data_set_name(dest)
-    cp_mvs2mvs = "cp -F rec \"//'{0}'\" \"//'{1}'\"".format(src, dest)
-    if is_binary:
-        cp_mvs2mvs = cp_mvs2mvs.replace("rec", "bin", 1)
-    rc, out, err = module.run_command(cp_mvs2mvs, errors='replace')
-    if rc:
-        raise USSCmdExecError(cp_mvs2mvs, rc, out, err)
-    return rc, out, err
-
-
 def copy_vsam_ps(src, dest, tmphlq=None):
     """Copy a VSAM(KSDS) data set to a PS data set vise versa.
 

plugins/module_utils/encode.py

@@ -510,10 +510,10 @@ def mvs_convert_encoding(
             if src_type == "PS":
                 temp_src_fo = NamedTemporaryFile()
                 temp_src = temp_src_fo.name
-                rc, out, err = copy.copy_ps2uss(src, temp_src)
+                rc, out, err = copy.copy_uss_mvs(src, temp_src)
             if src_type == "PO":
                 temp_src = mkdtemp()
-                rc, out, err = copy.copy_pds2uss(src, temp_src)
+                rc, out, err = copy.copy_uss_mvs(src, temp_src)
             if src_type == "KSDS":
                 reclen, space_u = self.listdsi_data_set(src.upper(), tmphlq=tmphlq)
                 # RDW takes the first 4 bytes in the VB format, hence we need to add an extra buffer to the vsam max recl.
@@ -522,7 +522,7 @@ def mvs_convert_encoding(
                 rc, out, err = copy.copy_vsam_ps(src.upper(), temp_ps, tmphlq=tmphlq)
                 temp_src_fo = NamedTemporaryFile()
                 temp_src = temp_src_fo.name
-                rc, out, err = copy.copy_ps2uss(temp_ps, temp_src)
+                rc, out, err = copy.copy_uss_mvs(temp_ps, temp_src)
             if dest_type == "PS" or dest_type == "KSDS":
                 temp_dest_fo = NamedTemporaryFile()
                 temp_dest = temp_dest_fo.name
@@ -538,17 +538,17 @@ def mvs_convert_encoding(
                         # RDW takes the first 4 bytes or records in the VB format, hence we need to add an extra buffer to the vsam max recl.
                         reclen += 4
                         temp_ps = self.temp_data_set(reclen, space_u)
-                        rc, out, err = copy.copy_uss2mvs(temp_dest, temp_ps, "PS")
+                        rc, out, err = copy.copy_uss_mvs(temp_dest, temp_ps)
                         rc, out, err = copy.copy_vsam_ps(temp_ps, dest.upper(), tmphlq=tmphlq)
                         convert_rc = True
                     elif dest_type == "PO":
                         for (dir, subdir, files) in walk(temp_dest):
                             for file in files:
                                 temp_file = path.join(validation.validate_safe_path(dir), validation.validate_safe_path(file))
-                                rc, out, err = copy.copy_uss2mvs(temp_file, dest, "PO")
+                                rc, out, err = copy.copy_uss_mvs(temp_file, dest)
                                 convert_rc = True
                     else:
-                        rc, out, err = copy.copy_uss2mvs(temp_dest, dest, dest_type)
+                        rc, out, err = copy.copy_uss_mvs(temp_dest, dest)
                         convert_rc = True
         except Exception:
             raise

plugins/modules/zos_copy.py

@@ -1657,7 +1657,7 @@ def _copy_to_file(self, src, dest, content_copy, conv_path):
         new_src = conv_path or src
         try:
             if self.is_binary:
-                copy.copy_uss2uss_binary(new_src, dest)
+                copy.copy_uss_mvs(new_src, dest, is_binary=True)
             else:
                 opts = dict()
                 opts["options"] = ""
@@ -1931,11 +1931,10 @@ def _mvs_copy_to_uss(
                             stderr=response.stderr_response
                         )
                 else:
-                    copy.copy_pds2uss(
+                    copy.copy_uss_mvs(
                         src,
                         dest,
-                        is_binary=self.is_binary,
-                        asa_text=self.asa_text
+                        is_binary=self.is_binary
                     )
         except CopyOperationError as err:
             raise err

plugins/modules/zos_mount.py

@@ -548,8 +548,7 @@
     backup as Backup,
 )
 from ansible_collections.ibm.ibm_zos_core.plugins.module_utils.copy import (
-    copy_ps2uss,
-    copy_uss2mvs,
+    copy_uss_mvs
 )
 
 
@@ -1045,7 +1044,7 @@ def run_module(module, arg_def):
         tmp_file_filename = tmp_file.name
         tmp_file.close()
 
-        copy_ps2uss(data_store, tmp_file_filename, False)
+        copy_uss_mvs(data_store, tmp_file_filename, is_binary=False)
 
         module.run_command(
             "chtag -tc ISO8859-1 " + tmp_file_filename, use_unsafe_shell=False, errors='replace'
@@ -1076,7 +1075,7 @@ def run_module(module, arg_def):
             module.run_command(
                 "mrm " + data_store, use_unsafe_shell=False, errors='replace'
             )
-            copy_uss2mvs(tmp_file_filename, data_store, "", True)
+            copy_uss_mvs(tmp_file_filename, data_store, is_binary=True)
 
         if os.path.isfile(tmp_file_filename):
             os.unlink(tmp_file_filename)