From 3c3e878cdb31ae882512a13b70f2988d25a4c164 Mon Sep 17 00:00:00 2001 From: Mingxun Wang Date: Thu, 7 Oct 2021 17:08:57 -0700 Subject: [PATCH] init --- proteosafe/msql-network-query-nf/Makefile | 7 + .../msql-network-query-nf/msql-nf/binding.xml | 40 ++ .../msql-network-query-nf/msql-nf/flow.xml | 28 ++ .../msql-network-query-nf/msql-nf/input.xml | 138 ++++++ .../msql-network-query-nf/msql-nf/result.xml | 425 ++++++++++++++++++ .../msql-network-query-nf/msql-nf/tool.xml | 73 +++ .../msql-network-query-nf/tools/msql-nf/bin | 1 + .../tools/msql-nf/cluster.config | 1 + .../tools/msql-nf/demangle_collection.py | 94 ++++ .../tools/msql-nf/ming_fileio_library.py | 1 + .../tools/msql-nf/ming_proteosafe_library.py | 1 + .../tools/msql-nf/nextflow_wrapper.py | 137 ++++++ .../tools/msql-nf/workflow.nf | 1 + 13 files changed, 947 insertions(+) create mode 100644 proteosafe/msql-network-query-nf/Makefile create mode 100644 proteosafe/msql-network-query-nf/msql-nf/binding.xml create mode 100644 proteosafe/msql-network-query-nf/msql-nf/flow.xml create mode 100644 proteosafe/msql-network-query-nf/msql-nf/input.xml create mode 100644 proteosafe/msql-network-query-nf/msql-nf/result.xml create mode 100644 proteosafe/msql-network-query-nf/msql-nf/tool.xml create mode 120000 proteosafe/msql-network-query-nf/tools/msql-nf/bin create mode 120000 proteosafe/msql-network-query-nf/tools/msql-nf/cluster.config create mode 100644 proteosafe/msql-network-query-nf/tools/msql-nf/demangle_collection.py create mode 120000 proteosafe/msql-network-query-nf/tools/msql-nf/ming_fileio_library.py create mode 120000 proteosafe/msql-network-query-nf/tools/msql-nf/ming_proteosafe_library.py create mode 100644 proteosafe/msql-network-query-nf/tools/msql-nf/nextflow_wrapper.py create mode 120000 proteosafe/msql-network-query-nf/tools/msql-nf/workflow.nf diff --git a/proteosafe/msql-network-query-nf/Makefile b/proteosafe/msql-network-query-nf/Makefile new file mode 100644 index 0000000..13aeeec --- /dev/null +++ b/proteosafe/msql-network-query-nf/Makefile @@ -0,0 +1,7 @@ +-include ../Makefile.credentials +include ../Makefile.deploytemplate + +WORKFLOW_NAME=msql-nf +TOOL_FOLDER_NAME=msql-nf +WORKFLOW_VERSION=release_30 +WORKFLOW_DESCRIPTION="MassQL Query Workflow - Documentation Link. Currently Supports v1.0 of MassQL" \ No newline at end of file diff --git a/proteosafe/msql-network-query-nf/msql-nf/binding.xml b/proteosafe/msql-network-query-nf/msql-nf/binding.xml new file mode 100644 index 0000000..373edf7 --- /dev/null +++ b/proteosafe/msql-network-query-nf/msql-nf/binding.xml @@ -0,0 +1,40 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/proteosafe/msql-network-query-nf/msql-nf/flow.xml b/proteosafe/msql-network-query-nf/msql-nf/flow.xml new file mode 100644 index 0000000..a53b303 --- /dev/null +++ b/proteosafe/msql-network-query-nf/msql-nf/flow.xml @@ -0,0 +1,28 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/proteosafe/msql-network-query-nf/msql-nf/input.xml b/proteosafe/msql-network-query-nf/msql-nf/input.xml new file mode 100644 index 0000000..74c4ec9 --- /dev/null +++ b/proteosafe/msql-network-query-nf/msql-nf/input.xml @@ -0,0 +1,138 @@ + + + MSQL-NF + MSQL-NF + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Spectrum Files + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/proteosafe/msql-network-query-nf/msql-nf/result.xml b/proteosafe/msql-network-query-nf/msql-nf/result.xml new file mode 100644 index 0000000..a44fb9c --- /dev/null +++ b/proteosafe/msql-network-query-nf/msql-nf/result.xml @@ -0,0 +1,425 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/proteosafe/msql-network-query-nf/msql-nf/tool.xml b/proteosafe/msql-network-query-nf/msql-nf/tool.xml new file mode 100644 index 0000000..1a94c3e --- /dev/null +++ b/proteosafe/msql-network-query-nf/msql-nf/tool.xml @@ -0,0 +1,73 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/proteosafe/msql-network-query-nf/tools/msql-nf/bin b/proteosafe/msql-network-query-nf/tools/msql-nf/bin new file mode 120000 index 0000000..bfd3c00 --- /dev/null +++ b/proteosafe/msql-network-query-nf/tools/msql-nf/bin @@ -0,0 +1 @@ +../../../../workflow/bin/ \ No newline at end of file diff --git a/proteosafe/msql-network-query-nf/tools/msql-nf/cluster.config b/proteosafe/msql-network-query-nf/tools/msql-nf/cluster.config new file mode 120000 index 0000000..1ac268f --- /dev/null +++ b/proteosafe/msql-network-query-nf/tools/msql-nf/cluster.config @@ -0,0 +1 @@ +../../../msql-repo-full-nf/tools/msql-repo-full-nf/cluster.config \ No newline at end of file diff --git a/proteosafe/msql-network-query-nf/tools/msql-nf/demangle_collection.py b/proteosafe/msql-network-query-nf/tools/msql-nf/demangle_collection.py new file mode 100644 index 0000000..b577c1d --- /dev/null +++ b/proteosafe/msql-network-query-nf/tools/msql-nf/demangle_collection.py @@ -0,0 +1,94 @@ +from pathlib import Path +import xmltodict +import argparse +from csv import DictReader +from collections import defaultdict +import sys +import shutil + +def arguments(): + parser = argparse.ArgumentParser(description='Demangle collection as aliases to folder') + parser.add_argument('-p','--params', type = Path, help='ProteoSAFe params.xml') + parser.add_argument('-i','--input_folder', type = Path, help='Input folder path') + parser.add_argument('-m','--input_mangled_prefix', type = str, help='Mangled prefix for input') + parser.add_argument('-o','--output_folder', type = Path, help='Output folder path') + parser.add_argument('-l','--output_list', type = Path, help='Output list of paths') + parser.add_argument('-r','--reverse', dest='reverse', action='store_true', help='Flag to demangle file collection') + parser.add_argument('-s','--preserve_suffix', dest='preserve_suffix', action='store_true', help='Flag to save suffix from demangled file collection') + parser.add_argument('-c','--copy', dest='copy', action='store_true', help='Flag to copy files into destination instead of symlink them') + return parser.parse_args() + +def read_params(input_file, mangled_prefix): + return get_mangled_file_mapping(parse_xml_file(input_file),mangled_prefix) + +def get_mangled_file_mapping(params, mangled_prefix): + all_mappings = params["upload_file_mapping"] + mangled_mapping = {} + demangled_mapping = {} + for mapping in all_mappings: + splits = mapping.split("|") + mangled_name = splits[0] + original_name = splits[1] + if mangled_prefix in mangled_name: + mangled_mapping[mangled_name] = Path(original_name) + demangled_mapping[original_name] = Path(mangled_name) + return mangled_mapping, demangled_mapping + +def parse_xml_file(input_file): + with open(input_file) as f: + key_value_pairs = defaultdict(list) + xml_obj = xmltodict.parse(f.read()) + + #print(json.dumps(xml_obj["parameters"])) + for parameter in xml_obj["parameters"]["parameter"]: + name = parameter["@name"] + value = parameter["#text"] + key_value_pairs[name].append(value) + + return key_value_pairs + +def main(): + + # don't fail on error, since it is likely to be run without inputs + args = arguments() + + if not (args.input_folder and args.output_folder and args.params and args.input_mangled_prefix): + print("Input folder, output folder, params, and collection prefix are required.") + sys.exit(0) + mangled_mapping, demangled_mapping = read_params(args.params, args.input_mangled_prefix) + output_list = None + + if args.output_list: + output_list = open(args.output_list, 'w') + + for input_file in args.input_folder.rglob('*'): + + if input_file.is_file(): + + input_path = args.input_folder.joinpath('/'.join(input_file.parts[1:])).absolute() + + if args.reverse: + if args.preserve_suffix: + suffix = input_file.suffix + input_file_str_no_suffix = '/'.join(input_file.with_suffix('').parts[1:]) + output_file = demangled_mapping.get(input_file_str_no_suffix).with_suffix(suffix) + else: + output_file = demangled_mapping.get('/'.join(input_file.parts[1:])) + else: + output_file = mangled_mapping.get(input_file.name) + + output_path = args.output_folder.joinpath(output_file) + output_path.parent.mkdir(parents=True, exist_ok=True) + if not args.copy: + output_path.symlink_to(input_path) + else: + shutil.copyfile(input_path, output_path) + + if output_list: + output_list.write('{}\n'.format(output_path)) + + if output_list: + output_list.close() + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/proteosafe/msql-network-query-nf/tools/msql-nf/ming_fileio_library.py b/proteosafe/msql-network-query-nf/tools/msql-nf/ming_fileio_library.py new file mode 120000 index 0000000..a983167 --- /dev/null +++ b/proteosafe/msql-network-query-nf/tools/msql-nf/ming_fileio_library.py @@ -0,0 +1 @@ +../../../msql/tools/msql/ming_fileio_library.py \ No newline at end of file diff --git a/proteosafe/msql-network-query-nf/tools/msql-nf/ming_proteosafe_library.py b/proteosafe/msql-network-query-nf/tools/msql-nf/ming_proteosafe_library.py new file mode 120000 index 0000000..43f41fb --- /dev/null +++ b/proteosafe/msql-network-query-nf/tools/msql-nf/ming_proteosafe_library.py @@ -0,0 +1 @@ +../../../msql/tools/msql/ming_proteosafe_library.py \ No newline at end of file diff --git a/proteosafe/msql-network-query-nf/tools/msql-nf/nextflow_wrapper.py b/proteosafe/msql-network-query-nf/tools/msql-nf/nextflow_wrapper.py new file mode 100644 index 0000000..3a8f98a --- /dev/null +++ b/proteosafe/msql-network-query-nf/tools/msql-nf/nextflow_wrapper.py @@ -0,0 +1,137 @@ +import json +import os +import argparse +import glob +import sys +import shutil +import pandas as pd +import ming_proteosafe_library +import pathlib + + +def main(): + parser = argparse.ArgumentParser(description="Proteosafe Wrapper for Nextflow") + parser.add_argument('workflow_params', help='workflow_params, from proteosafe') + parser.add_argument('nextflow_script', help='nextflow_script to actually run') + parser.add_argument('conda_activate', help='conda_activate, this is the path to the activate command in the main conda installation') + parser.add_argument('nextflow_conda_environment', help='nextflow_conda_environment, this likely should be wherever all your dependencies and nextflow are installed, e.g. nextflow or msql2') + parser.add_argument('--parametermapping', action='append', help='mapping of current workflow parameters to new parameters in the format: :') + parser.add_argument('--newparameters', action='append', help='parameter key: :') + + parser.add_argument('--metricoutput', default=None, help='output folder for metrics') + + # Settings if we want to monitor nextflow progress in ProteoSAFe + parser.add_argument('--updateproteosafefronteendstatus', default='NO', help='Enables outputting data to front end task directories to allow for current status, options YES and NO') + + # These settings are for the cluster run + parser.add_argument('--runcluster', default='NO', help='Tries to run this on the cluster, values are NO and YES') + parser.add_argument('--clusterconfig', default=None, help='Path to configuration file') + parser.add_argument('--user', default=None, help='username running the task') + parser.add_argument('--clusterpythonruntime', default=None, help='cluster python runtime') + parser.add_argument('--clusterworkprefix', default=None, help='clusterworkprefix') + parser.add_argument('--task', default=None, help='cluster python runtime') + + args = parser.parse_args() + + # Listing our system + os.system("hostname") + os.system("whoami") + os.system("pwd") + os.system("ls -l -h") + + output_stdout_file = os.path.abspath(os.path.join(args.metricoutput, "stdout.log")) + workflow_task_directory = "." + original_directory = os.getcwd() + + output_trace_filename = "trace.txt" + + if args.updateproteosafefronteendstatus == "YES": + output_trace_folder = os.path.join("/data/ccms-data/tasks/", args.user, args.task, "nextflow") + pathlib.Path(output_trace_folder).mkdir(parents=True, exist_ok=True) + output_trace_filename = os.path.join(output_trace_folder, "trace.txt") + + output_stdout_file = os.path.join(output_trace_folder, "stdout.log") + + if args.runcluster == "YES" and args.user in ["mwang87"]: + # Staging all files on gscratch because they might not be seen if we schedule outputs from local scratch disk + workflow_task_directory = os.path.join("/gscratch/nextflow_staging", args.task) + pathlib.Path(workflow_task_directory).mkdir(parents=True, exist_ok=True) + + pbs_cluster_work_dir = os.path.join(args.clusterworkprefix, args.task, "work") + + cmd = "source {} {} && \ + export NXF_OPTS='-Xms35G -Xmx35G' && \ + cd {} && nextflow run {} -c {} \ + -work-dir {} \ + -resume \ + --PYTHONRUNTIME={} \ + -with-trace {} \ + -with-dag dag.html \ + -with-report report.html \ + -with-timeline timeline.html > {} 2>&1".format(args.conda_activate, args.nextflow_conda_environment, + workflow_task_directory, + args.nextflow_script, args.clusterconfig, pbs_cluster_work_dir, args.clusterpythonruntime, + output_trace_filename, + output_stdout_file) + else: + cmd = "source {} {} && nextflow run {} \ + -with-trace {} \ + -with-dag dag.html \ + -with-report report.html \ + -with-timeline timeline.html > {} 2>&1".format(args.conda_activate, args.nextflow_conda_environment, + args.nextflow_script, + output_trace_filename, + output_stdout_file) + for parameter in args.newparameters: + print(parameter) + cmd += ' --{} "{}"'.format(parameter.split(":")[0], parameter.split(":")[1].replace("\n", "")) + + params_obj = ming_proteosafe_library.parse_xml_file(open(args.workflow_params)) + for parameter in args.parametermapping: + print(parameter) + new_param = parameter.split(":")[1] + old_param = parameter.split(":")[0] + + cmd += ' --{} "{}"'.format(new_param, params_obj[old_param][0].replace("\n", "")) + + # Saving the script + output_script = os.path.abspath(os.path.join(args.metricoutput, "run_nf.sh")) + with open(output_script, "w") as f: + f.write(cmd) + + print(cmd) + return_val = os.system(cmd) + if return_val != 0: + print("Error in Nextflow") + + # Copying back results + if workflow_task_directory != ".": + try: + cmd = "rsync -avp {}/ {}".format(workflow_task_directory, original_directory) + print(cmd) + os.system(cmd) + except: + pass + + # Copying the metric output to output folder + if args.metricoutput is not None: + try: + shutil.copyfile(output_trace_filename, os.path.join(args.metricoutput, "trace.txt")) + except: + pass + + try: + shutil.copyfile(output_stdout_file, os.path.join(args.metricoutput, "stdout.log")) + except: + pass + + try: + shutil.copyfile("report.html", os.path.join(args.metricoutput, "report.html")) + shutil.copyfile("timeline.html", os.path.join(args.metricoutput, "timeline.html")) + shutil.copyfile("dag.html", os.path.join(args.metricoutput, "dag.html")) + except: + pass + + +if __name__ == "__main__": + main() diff --git a/proteosafe/msql-network-query-nf/tools/msql-nf/workflow.nf b/proteosafe/msql-network-query-nf/tools/msql-nf/workflow.nf new file mode 120000 index 0000000..75ff0e3 --- /dev/null +++ b/proteosafe/msql-network-query-nf/tools/msql-nf/workflow.nf @@ -0,0 +1 @@ +../../../../workflow/workflow.nf \ No newline at end of file