From 3c9c0a47eee659057ec1c390e5a16547918c9e6e Mon Sep 17 00:00:00 2001 From: Jiayong Li Date: Tue, 11 May 2021 14:58:45 -0400 Subject: [PATCH 01/11] Add formatting for StepNameHint extension --- cwltool/extensions-v1.1.yml | 19 +++++++++++++++++++ cwltool/main.py | 2 ++ cwltool/process.py | 1 + 3 files changed, 22 insertions(+) diff --git a/cwltool/extensions-v1.1.yml b/cwltool/extensions-v1.1.yml index 2f5daf809..6df79311d 100644 --- a/cwltool/extensions-v1.1.yml +++ b/cwltool/extensions-v1.1.yml @@ -64,3 +64,22 @@ $graph: The number of MPI processes to start. If you give a string, this will be evaluated as a CWL Expression and it must evaluate to an integer. + +- name: StepNameHint + type: record + inVocab: false + extends: cwl:ProcessRequirement + doc: | + Provide a hint for naming the runtime workflow step in logs or user interface. + fields: + - name: class + type: string + doc: "Always 'StepNameHint'" + jsonldPredicate: + "_id": "@type" + "_type": "@vocab" + - name: stepname: + type: [string, Expression] + doc: | + A string or expression returning a string with the preferred name for the step. + If it is an expression, it is evaluated after the input object has been completely determined. diff --git a/cwltool/main.py b/cwltool/main.py index 1cf4a7ee0..76756cefe 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -662,12 +662,14 @@ def setup_schema( use_custom_schema("v1.2.0-dev1", "http://commonwl.org/cwltool", ext11) use_custom_schema("v1.2.0-dev2", "http://commonwl.org/cwltool", ext11) use_custom_schema("v1.2.0-dev3", "http://commonwl.org/cwltool", ext11) + use_custom_schema("v1.2", "http://commonwl.org/cwltool", ext11) else: use_standard_schema("v1.0") use_standard_schema("v1.1") use_standard_schema("v1.2.0-dev1") use_standard_schema("v1.2.0-dev2") use_standard_schema("v1.2.0-dev3") + use_standard_schema("v1.2") class ProvLogFormatter(logging.Formatter): diff --git a/cwltool/process.py b/cwltool/process.py index e444d04dd..5b2ea29d8 100644 --- a/cwltool/process.py +++ b/cwltool/process.py @@ -119,6 +119,7 @@ def filter(self, record: logging.LogRecord) -> bool: "http://commonwl.org/cwltool#NetworkAccess", "http://commonwl.org/cwltool#LoadListingRequirement", "http://commonwl.org/cwltool#InplaceUpdateRequirement", + "http://commonwl.org/cwltool#StepNameHint", ] cwl_files = ( From a3ad10b2036a70f6109a46d3f6e1d3a485c1f854 Mon Sep 17 00:00:00 2001 From: Jiayong Li Date: Mon, 21 Jun 2021 16:43:11 -0400 Subject: [PATCH 02/11] Change job name according to stepnamehint --- cwltool/workflow_job.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/cwltool/workflow_job.py b/cwltool/workflow_job.py index e67747cae..bb1e7eadc 100644 --- a/cwltool/workflow_job.py +++ b/cwltool/workflow_job.py @@ -69,7 +69,16 @@ def job( ) -> JobsGeneratorType: runtimeContext = runtimeContext.copy() runtimeContext.part_of = self.name - runtimeContext.name = shortname(self.id) + + has_custom_name = False + for hint in self.step.hints: + if hint["class"] == "http://commonwl.org/cwltool#StepNameHint": + vfinputs = {shortname(k): v for k, v in joborder.items()} + runtimeContext.name = expression.do_eval(hint["stepname"], vfinputs, self.step.requirements, None, None, {}) + has_custom_name = True + break + if not has_custom_name: + runtimeContext.name = shortname(self.id) _logger.info("[%s] start", self.name) From 55a91d806442294e5b8aee78bc72e051beec2393 Mon Sep 17 00:00:00 2001 From: Jiayong Li Date: Mon, 21 Jun 2021 16:44:29 -0400 Subject: [PATCH 03/11] Change stepnamehint record --- cwltool/extensions-v1.1.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cwltool/extensions-v1.1.yml b/cwltool/extensions-v1.1.yml index 6df79311d..728bb2df4 100644 --- a/cwltool/extensions-v1.1.yml +++ b/cwltool/extensions-v1.1.yml @@ -74,12 +74,12 @@ $graph: fields: - name: class type: string - doc: "Always 'StepNameHint'" + doc: "Always 'StepNameHint'" jsonldPredicate: - "_id": "@type" - "_type": "@vocab" - - name: stepname: - type: [string, Expression] + "_id": "@type" + "_type": "@vocab" + - name: stepname + type: [string, cwl:Expression] doc: | - A string or expression returning a string with the preferred name for the step. + A string or expression returning a string with the preferred name for the step. If it is an expression, it is evaluated after the input object has been completely determined. From 75c986cf83389e7f24fb15bc9d4e9fcf24c1d48d Mon Sep 17 00:00:00 2001 From: Jiayong Li Date: Wed, 4 Aug 2021 11:50:22 -0400 Subject: [PATCH 04/11] Change how to retrieve stepname from requirements/hints --- cwltool/workflow_job.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/cwltool/workflow_job.py b/cwltool/workflow_job.py index bb1e7eadc..c6245ec66 100644 --- a/cwltool/workflow_job.py +++ b/cwltool/workflow_job.py @@ -70,14 +70,11 @@ def job( runtimeContext = runtimeContext.copy() runtimeContext.part_of = self.name - has_custom_name = False - for hint in self.step.hints: - if hint["class"] == "http://commonwl.org/cwltool#StepNameHint": - vfinputs = {shortname(k): v for k, v in joborder.items()} - runtimeContext.name = expression.do_eval(hint["stepname"], vfinputs, self.step.requirements, None, None, {}) - has_custom_name = True - break - if not has_custom_name: + stepnameReq, is_required = self.step.get_requirement("http://commonwl.org/cwltool#StepNameHint") + if stepnameReq is not None and is_required is not None: + vfinputs = {shortname(k): v for k, v in joborder.items()} + runtimeContext.name = expression.do_eval(stepnameReq["stepname"], vfinputs, self.step.requirements, None, None, {}) + else: runtimeContext.name = shortname(self.id) _logger.info("[%s] start", self.name) From 9833f4147394481d4c4a7fa3b55ae7208d7a78c7 Mon Sep 17 00:00:00 2001 From: Jiayong Li Date: Wed, 4 Aug 2021 11:52:34 -0400 Subject: [PATCH 05/11] Add unit test for stepname --- tests/scatter-echo-wf.cwl | 26 ++++++++++++++++++++++++++ tests/scatter-echo-wf.yml | 1 + tests/test_ext.py | 15 +++++++++++++++ 3 files changed, 42 insertions(+) create mode 100644 tests/scatter-echo-wf.cwl create mode 100644 tests/scatter-echo-wf.yml diff --git a/tests/scatter-echo-wf.cwl b/tests/scatter-echo-wf.cwl new file mode 100644 index 000000000..0e3d08198 --- /dev/null +++ b/tests/scatter-echo-wf.cwl @@ -0,0 +1,26 @@ +#!/usr/bin/env cwl-runner +$namespaces: + cwltool: "http://commonwl.org/cwltool#" +cwlVersion: v1.1 +class: Workflow +requirements: + ScatterFeatureRequirement: {} + InlineJavascriptRequirement: {} + StepInputExpressionRequirement: {} + +inputs: + texts: + type: string[] + +outputs: [] + +steps: + echo: + run: echo_no_output.cwl + scatter: text + hints: + cwltool:StepNameHint: + stepname: $("test_" + inputs.text.split('.')[0]) + in: + text: texts + out: [] diff --git a/tests/scatter-echo-wf.yml b/tests/scatter-echo-wf.yml new file mode 100644 index 000000000..2a67a665e --- /dev/null +++ b/tests/scatter-echo-wf.yml @@ -0,0 +1 @@ +texts: ["a.vcf", "b.vcf", "c.vcf"] diff --git a/tests/test_ext.py b/tests/test_ext.py index 1eb4091e5..abc3206b4 100644 --- a/tests/test_ext.py +++ b/tests/test_ext.py @@ -295,3 +295,18 @@ def test_warn_large_inputs() -> None: ) finally: cwltool.process.FILE_COUNT_WARNING = was + + +def test_stepname() -> None: + stream = StringIO() + + main( + ["--enable-ext", + get_data("tests/scatter-echo-wf.cwl"), get_data("tests/scatter-echo-wf.yml")], + stderr=stream, + ) + + assert ( + "test_a" in stream.getvalue() and "test_b" in stream.getvalue() + and "test_c" in stream.getvalue() + ) From 8abb24c30744ce39264329508da31998c30b462e Mon Sep 17 00:00:00 2001 From: Jiayong Li Date: Wed, 4 Aug 2021 12:06:10 -0400 Subject: [PATCH 06/11] Change StepNameHint to StepName --- cwltool/extensions-v1.1.yml | 6 +++--- cwltool/process.py | 2 +- cwltool/workflow_job.py | 2 +- tests/echo_no_output.cwl | 9 +++++++++ tests/scatter-echo-wf.cwl | 2 +- 5 files changed, 15 insertions(+), 6 deletions(-) create mode 100644 tests/echo_no_output.cwl diff --git a/cwltool/extensions-v1.1.yml b/cwltool/extensions-v1.1.yml index 728bb2df4..8c0da4745 100644 --- a/cwltool/extensions-v1.1.yml +++ b/cwltool/extensions-v1.1.yml @@ -65,16 +65,16 @@ $graph: this will be evaluated as a CWL Expression and it must evaluate to an integer. -- name: StepNameHint +- name: StepName type: record inVocab: false extends: cwl:ProcessRequirement doc: | - Provide a hint for naming the runtime workflow step in logs or user interface. + Provide a string or expression for naming the runtime workflow step in logs or user interface. fields: - name: class type: string - doc: "Always 'StepNameHint'" + doc: "Always 'StepName'" jsonldPredicate: "_id": "@type" "_type": "@vocab" diff --git a/cwltool/process.py b/cwltool/process.py index fb738d3ce..1d64edf2c 100644 --- a/cwltool/process.py +++ b/cwltool/process.py @@ -119,7 +119,7 @@ def filter(self, record: logging.LogRecord) -> bool: "http://commonwl.org/cwltool#NetworkAccess", "http://commonwl.org/cwltool#LoadListingRequirement", "http://commonwl.org/cwltool#InplaceUpdateRequirement", - "http://commonwl.org/cwltool#StepNameHint", + "http://commonwl.org/cwltool#StepName", ] cwl_files = ( diff --git a/cwltool/workflow_job.py b/cwltool/workflow_job.py index c6245ec66..566fd5483 100644 --- a/cwltool/workflow_job.py +++ b/cwltool/workflow_job.py @@ -70,7 +70,7 @@ def job( runtimeContext = runtimeContext.copy() runtimeContext.part_of = self.name - stepnameReq, is_required = self.step.get_requirement("http://commonwl.org/cwltool#StepNameHint") + stepnameReq, is_required = self.step.get_requirement("http://commonwl.org/cwltool#StepName") if stepnameReq is not None and is_required is not None: vfinputs = {shortname(k): v for k, v in joborder.items()} runtimeContext.name = expression.do_eval(stepnameReq["stepname"], vfinputs, self.step.requirements, None, None, {}) diff --git a/tests/echo_no_output.cwl b/tests/echo_no_output.cwl new file mode 100644 index 000000000..2ed2614bf --- /dev/null +++ b/tests/echo_no_output.cwl @@ -0,0 +1,9 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.1 +class: CommandLineTool +inputs: + text: + type: string + inputBinding: {} +outputs: [] +baseCommand: echo diff --git a/tests/scatter-echo-wf.cwl b/tests/scatter-echo-wf.cwl index 0e3d08198..d24ce42ed 100644 --- a/tests/scatter-echo-wf.cwl +++ b/tests/scatter-echo-wf.cwl @@ -19,7 +19,7 @@ steps: run: echo_no_output.cwl scatter: text hints: - cwltool:StepNameHint: + cwltool:StepName: stepname: $("test_" + inputs.text.split('.')[0]) in: text: texts From 662584f34eee71c6e4ac643755db1e1927d7d5fe Mon Sep 17 00:00:00 2001 From: Jiayong Li Date: Wed, 4 Aug 2021 12:33:53 -0400 Subject: [PATCH 07/11] Simplify logic for detecting StepName requirement/hint --- cwltool/workflow_job.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cwltool/workflow_job.py b/cwltool/workflow_job.py index 566fd5483..7955c1176 100644 --- a/cwltool/workflow_job.py +++ b/cwltool/workflow_job.py @@ -71,7 +71,7 @@ def job( runtimeContext.part_of = self.name stepnameReq, is_required = self.step.get_requirement("http://commonwl.org/cwltool#StepName") - if stepnameReq is not None and is_required is not None: + if is_required is not None: vfinputs = {shortname(k): v for k, v in joborder.items()} runtimeContext.name = expression.do_eval(stepnameReq["stepname"], vfinputs, self.step.requirements, None, None, {}) else: From c7a0214d46a1693080b039cbbbeeaa500479855c Mon Sep 17 00:00:00 2001 From: Jiayong Li Date: Tue, 10 Aug 2021 11:19:34 -0400 Subject: [PATCH 08/11] Change StepName to JobName and add docstring to test --- cwltool/extensions-v1.1.yml | 8 ++++---- cwltool/workflow_job.py | 6 +++--- tests/scatter-echo-wf.cwl | 4 ++-- tests/test_ext.py | 3 ++- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/cwltool/extensions-v1.1.yml b/cwltool/extensions-v1.1.yml index 8c0da4745..5496cd7c0 100644 --- a/cwltool/extensions-v1.1.yml +++ b/cwltool/extensions-v1.1.yml @@ -65,7 +65,7 @@ $graph: this will be evaluated as a CWL Expression and it must evaluate to an integer. -- name: StepName +- name: JobName type: record inVocab: false extends: cwl:ProcessRequirement @@ -74,12 +74,12 @@ $graph: fields: - name: class type: string - doc: "Always 'StepName'" + doc: "Always 'JobName'" jsonldPredicate: "_id": "@type" "_type": "@vocab" - - name: stepname + - name: jobname type: [string, cwl:Expression] doc: | - A string or expression returning a string with the preferred name for the step. + A string or expression returning a string with the preferred name for the job. If it is an expression, it is evaluated after the input object has been completely determined. diff --git a/cwltool/workflow_job.py b/cwltool/workflow_job.py index 7955c1176..88801389d 100644 --- a/cwltool/workflow_job.py +++ b/cwltool/workflow_job.py @@ -70,10 +70,10 @@ def job( runtimeContext = runtimeContext.copy() runtimeContext.part_of = self.name - stepnameReq, is_required = self.step.get_requirement("http://commonwl.org/cwltool#StepName") - if is_required is not None: + jobnameReq, is_required = self.step.get_requirement("http://commonwl.org/cwltool#JobName") + if jobnameReq is not None: vfinputs = {shortname(k): v for k, v in joborder.items()} - runtimeContext.name = expression.do_eval(stepnameReq["stepname"], vfinputs, self.step.requirements, None, None, {}) + runtimeContext.name = expression.do_eval(jobnameReq["jobname"], vfinputs, self.step.requirements, None, None, {}) else: runtimeContext.name = shortname(self.id) diff --git a/tests/scatter-echo-wf.cwl b/tests/scatter-echo-wf.cwl index d24ce42ed..b108d0b94 100644 --- a/tests/scatter-echo-wf.cwl +++ b/tests/scatter-echo-wf.cwl @@ -19,8 +19,8 @@ steps: run: echo_no_output.cwl scatter: text hints: - cwltool:StepName: - stepname: $("test_" + inputs.text.split('.')[0]) + cwltool:JobName: + jobname: $("test_" + inputs.text.split('.')[0]) in: text: texts out: [] diff --git a/tests/test_ext.py b/tests/test_ext.py index abc3206b4..e8344c481 100644 --- a/tests/test_ext.py +++ b/tests/test_ext.py @@ -297,7 +297,8 @@ def test_warn_large_inputs() -> None: cwltool.process.FILE_COUNT_WARNING = was -def test_stepname() -> None: +def test_jobname() -> None: + """Test JobName requirement.""" stream = StringIO() main( From e4fed7f26d35b2922c2b811a27b89e52d72b61eb Mon Sep 17 00:00:00 2001 From: Jiayong Li Date: Tue, 10 Aug 2021 15:11:19 -0400 Subject: [PATCH 09/11] Change formatting --- cwltool/workflow_job.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cwltool/workflow_job.py b/cwltool/workflow_job.py index 88801389d..f7eea7b91 100644 --- a/cwltool/workflow_job.py +++ b/cwltool/workflow_job.py @@ -73,7 +73,8 @@ def job( jobnameReq, is_required = self.step.get_requirement("http://commonwl.org/cwltool#JobName") if jobnameReq is not None: vfinputs = {shortname(k): v for k, v in joborder.items()} - runtimeContext.name = expression.do_eval(jobnameReq["jobname"], vfinputs, self.step.requirements, None, None, {}) + runtimeContext.name = expression.do_eval(jobnameReq["jobname"], + vfinputs, self.step.requirements, None, None, {}) else: runtimeContext.name = shortname(self.id) From fd4e69948ebc93cf516c2576b3a9714ede165c4b Mon Sep 17 00:00:00 2001 From: Jiayong Li Date: Tue, 10 Aug 2021 15:55:30 -0400 Subject: [PATCH 10/11] Fix formatting according to py38-lint --- cwltool/workflow_job.py | 9 ++++++--- tests/test_ext.py | 10 +++++++--- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/cwltool/workflow_job.py b/cwltool/workflow_job.py index f7eea7b91..55b4d8504 100644 --- a/cwltool/workflow_job.py +++ b/cwltool/workflow_job.py @@ -70,11 +70,14 @@ def job( runtimeContext = runtimeContext.copy() runtimeContext.part_of = self.name - jobnameReq, is_required = self.step.get_requirement("http://commonwl.org/cwltool#JobName") + jobnameReq, is_required = self.step.get_requirement( + "http://commonwl.org/cwltool#JobName" + ) if jobnameReq is not None: vfinputs = {shortname(k): v for k, v in joborder.items()} - runtimeContext.name = expression.do_eval(jobnameReq["jobname"], - vfinputs, self.step.requirements, None, None, {}) + runtimeContext.name = expression.do_eval( + jobnameReq["jobname"], vfinputs, self.step.requirements, None, None, {} + ) else: runtimeContext.name = shortname(self.id) diff --git a/tests/test_ext.py b/tests/test_ext.py index e8344c481..e549f1697 100644 --- a/tests/test_ext.py +++ b/tests/test_ext.py @@ -302,12 +302,16 @@ def test_jobname() -> None: stream = StringIO() main( - ["--enable-ext", - get_data("tests/scatter-echo-wf.cwl"), get_data("tests/scatter-echo-wf.yml")], + [ + "--enable-ext", + get_data("tests/scatter-echo-wf.cwl"), + get_data("tests/scatter-echo-wf.yml") + ], stderr=stream, ) assert ( - "test_a" in stream.getvalue() and "test_b" in stream.getvalue() + "test_a" in stream.getvalue() + and "test_b" in stream.getvalue() and "test_c" in stream.getvalue() ) From d529d3fd0762eaa7e15da9a73ff4f7f6b21f2903 Mon Sep 17 00:00:00 2001 From: Jiayong Li Date: Tue, 10 Aug 2021 19:58:20 -0400 Subject: [PATCH 11/11] Change formatting according to lint --- tests/test_ext.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_ext.py b/tests/test_ext.py index e549f1697..d738a02de 100644 --- a/tests/test_ext.py +++ b/tests/test_ext.py @@ -305,7 +305,7 @@ def test_jobname() -> None: [ "--enable-ext", get_data("tests/scatter-echo-wf.cwl"), - get_data("tests/scatter-echo-wf.yml") + get_data("tests/scatter-echo-wf.yml"), ], stderr=stream, )