diff --git a/docs/process.md b/docs/process.md index 373ff47d48..be3053272a 100644 --- a/docs/process.md +++ b/docs/process.md @@ -488,6 +488,22 @@ workflow { } ``` +:::{versionadded} 23.06.0-edge +::: + +By default, the process will be aborted if an input `path` is null. The `nullable` option can be used to ensure that the process is executed even if the path is null: + +```groovy +process foo { + input: + path('data.txt', nullable: true) + + """ + [[ -f data.txt ]] your_command --in data.txt || other_command + """ +} +``` + ### Multiple input files A `path` input can also accept a collection of files instead of a single value. In this case, the input variable will be a Groovy list, and you can use it as such. @@ -927,6 +943,11 @@ Available options: `maxDepth` : Maximum number of directory levels to visit (default: no limit) +`nullable` +: :::{versionadded} 23.06.0-edge + ::: +: When `true` emit `null` (instead of aborting) if the output path doesn't exist (default: `false`) + `type` : Type of paths returned, either `file`, `dir` or `any` (default: `any`, or `file` if the specified file name pattern contains a double star (`**`)) @@ -1132,6 +1153,10 @@ output: In this example, the process is normally expected to produce an `output.txt` file, but in the cases where the file is legitimately missing, the process does not fail. The output channel will only contain values for those processes that produce `output.txt`. +:::{versionadded} 23.06.0-edge +Path outputs can also be defined as `nullable`. Whereas an `optional` path will emit nothing if the output file does not exist, a `nullable` path will emit a "null" path that can trigger downstream computations, specifically any process that has a `nullable` path input. +::: + ## When The `when` block allows you to define a condition that must be satisfied in order to execute the process. The condition can be any expression that returns a boolean value. diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy index 38cac391c9..1c0da1bbac 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy @@ -103,6 +103,7 @@ import nextflow.util.CacheHelper import nextflow.util.Escape import nextflow.util.LockManager import nextflow.util.LoggerHelper +import nextflow.util.NullPath import nextflow.util.TestOnly import org.codehaus.groovy.control.CompilerConfiguration import org.codehaus.groovy.control.customizers.ASTTransformationCustomizer @@ -1567,6 +1568,10 @@ class TaskProcessor { def path = param.glob ? splitter.strip(filePattern) : filePattern def file = workDir.resolve(path) def exists = param.followLinks ? file.exists() : file.exists(LinkOption.NOFOLLOW_LINKS) + if( !exists && param.nullable ) { + file = new NullPath(path) + exists = true + } if( exists ) result = [file] else @@ -1782,7 +1787,11 @@ class TaskProcessor { return new FileHolder(source, result) } - protected Path normalizeToPath( obj ) { + protected Path normalizeToPath( obj, boolean nullable=false ) { + + if( obj instanceof NullPath && !nullable ) + throw new ProcessUnrecoverableException("Path value cannot be null") + if( obj instanceof Path ) return obj @@ -1805,7 +1814,7 @@ class TaskProcessor { throw new ProcessUnrecoverableException("Not a valid path value: '$str'") } - protected List normalizeInputToFiles( Object obj, int count, boolean coerceToPath, FilePorter.Batch batch ) { + protected List normalizeInputToFiles( Object obj, int count, boolean coerceToPath, FilePorter.Batch batch, boolean nullable=false ) { Collection allItems = obj instanceof Collection ? obj : [obj] def len = allItems.size() @@ -1815,7 +1824,7 @@ class TaskProcessor { for( def item : allItems ) { if( item instanceof Path || coerceToPath ) { - def path = normalizeToPath(item) + def path = normalizeToPath(item, nullable) def target = executor.isForeignFile(path) ? batch.addToForeign(path) : path def holder = new FileHolder(target) files << holder @@ -2029,7 +2038,7 @@ class TaskProcessor { final param = entry.getKey() final val = entry.getValue() final fileParam = param as FileInParam - final normalized = normalizeInputToFiles(val, count, fileParam.isPathQualifier(), batch) + final normalized = normalizeInputToFiles(val, count, fileParam.isPathQualifier(), batch, fileParam.isNullable()) final resolved = expandWildcards( fileParam.getFilePattern(ctx), normalized ) ctx.put( param.name, singleItemOrList(resolved, task.type) ) count += resolved.size() diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/FileInParam.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/FileInParam.groovy index b52f8dad39..377c850b84 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/params/FileInParam.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/params/FileInParam.groovy @@ -32,6 +32,8 @@ class FileInParam extends BaseInParam implements PathQualifier { protected filePattern + private boolean nullable + private boolean pathQualifier @Override String getTypeName() { pathQualifier ? 'path' : 'file' } @@ -120,6 +122,15 @@ class FileInParam extends BaseInParam implements PathQualifier { return value } + FileInParam setNullable(boolean value) { + this.nullable = value + return this + } + + boolean isNullable() { + return nullable + } + @Override FileInParam setPathQualifier(boolean flag) { pathQualifier = flag diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/FileOutParam.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/FileOutParam.groovy index e305c46c3e..d95604efae 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/params/FileOutParam.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/params/FileOutParam.groovy @@ -80,8 +80,16 @@ class FileOutParam extends BaseOutParam implements OutParam, OptionalParam, Path */ boolean followLinks = true + /** + * When true the specified name is interpreted as a glob pattern + */ boolean glob = true + /** + * When true emit null (instead of aborting) if the output path doesn't exist + */ + boolean nullable = false + private GString gstring private Closure dynamicObj private String filePattern diff --git a/modules/nextflow/src/main/groovy/nextflow/util/NullPath.groovy b/modules/nextflow/src/main/groovy/nextflow/util/NullPath.groovy new file mode 100644 index 0000000000..fa9b201df7 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/util/NullPath.groovy @@ -0,0 +1,40 @@ +/* + * Copyright 2023, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.util + +import java.nio.file.Path + +import groovy.transform.EqualsAndHashCode +import groovy.transform.PackageScope +import groovy.util.logging.Slf4j + +/** + * @author : jorge + * + */ +@EqualsAndHashCode +@Slf4j +class NullPath implements Path { + + @PackageScope + @Delegate + Path delegate + + NullPath(String path) { + delegate = of(path) + } +} diff --git a/modules/nextflow/src/test/groovy/nextflow/script/params/ParamsOutTest.groovy b/modules/nextflow/src/test/groovy/nextflow/script/params/ParamsOutTest.groovy index 1c46c898ba..79852a5bdb 100644 --- a/modules/nextflow/src/test/groovy/nextflow/script/params/ParamsOutTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/script/params/ParamsOutTest.groovy @@ -941,7 +941,8 @@ class ParamsOutTest extends Dsl2Spec { separatorChar: ':', glob: true, optional: true, - includeInputs: true + includeInputs: true, + nullable: true return '' } @@ -963,6 +964,7 @@ class ParamsOutTest extends Dsl2Spec { !out0.getGlob() !out0.getOptional() !out0.getIncludeInputs() + !out0.isNullable() and: out1.getMaxDepth() == 5 @@ -973,6 +975,7 @@ class ParamsOutTest extends Dsl2Spec { out1.getGlob() out1.getOptional() out1.getIncludeInputs() + out1.isNullable() } def 'should set file options' () { diff --git a/tests/checks/nullable-path-input-fails.nf/.checks b/tests/checks/nullable-path-input-fails.nf/.checks new file mode 100644 index 0000000000..346e4a2ea0 --- /dev/null +++ b/tests/checks/nullable-path-input-fails.nf/.checks @@ -0,0 +1,18 @@ +set +e + +# +# run normal mode +# +echo '' +$NXF_RUN +[[ $? == 0 ]] && exit 1 + + +# +# RESUME mode +# +echo '' +$NXF_RUN -resume +[[ $? == 0 ]] && exit 1 + +exit 0 \ No newline at end of file diff --git a/tests/checks/nullable-path-output-fails.nf/.checks b/tests/checks/nullable-path-output-fails.nf/.checks new file mode 100644 index 0000000000..346e4a2ea0 --- /dev/null +++ b/tests/checks/nullable-path-output-fails.nf/.checks @@ -0,0 +1,18 @@ +set +e + +# +# run normal mode +# +echo '' +$NXF_RUN +[[ $? == 0 ]] && exit 1 + + +# +# RESUME mode +# +echo '' +$NXF_RUN -resume +[[ $? == 0 ]] && exit 1 + +exit 0 \ No newline at end of file diff --git a/tests/checks/nullable-path-succeeds.nf/.checks b/tests/checks/nullable-path-succeeds.nf/.checks new file mode 100644 index 0000000000..5c332305d9 --- /dev/null +++ b/tests/checks/nullable-path-succeeds.nf/.checks @@ -0,0 +1,16 @@ +set +e + +# +# run normal mode +# +echo '' +$NXF_RUN +[[ $? == 0 ]] || false + + +# +# RESUME mode +# +echo '' +$NXF_RUN -resume +[[ $? == 0 ]] || false diff --git a/tests/nullable-path-input-fails.nf b/tests/nullable-path-input-fails.nf new file mode 100644 index 0000000000..aad3dde4ff --- /dev/null +++ b/tests/nullable-path-input-fails.nf @@ -0,0 +1,24 @@ +#!/usr/bin/env nextflow + +process foo { + input: + val id + output: + tuple val(id), path('output.txt', nullable: true) + exec: + println id +} + +process bar { + input: + tuple val(id), path(file) + output: + val file + exec: + sleep 1000L + println file +} + +workflow { + channel.of('foo') | foo | bar | view() +} \ No newline at end of file diff --git a/tests/nullable-path-output-fails.nf b/tests/nullable-path-output-fails.nf new file mode 100644 index 0000000000..e3cd052084 --- /dev/null +++ b/tests/nullable-path-output-fails.nf @@ -0,0 +1,14 @@ +#!/usr/bin/env nextflow + +process foo { + input: + val id + output: + tuple val(id), path('output.txt') + exec: + println 'hi' +} + +workflow { + foo('foo') +} \ No newline at end of file diff --git a/tests/nullable-path-succeeds.nf b/tests/nullable-path-succeeds.nf new file mode 100644 index 0000000000..afadb4a1f5 --- /dev/null +++ b/tests/nullable-path-succeeds.nf @@ -0,0 +1,24 @@ +#!/usr/bin/env nextflow + +process foo { + input: + val id + output: + tuple val(id), path('output.txt', nullable: true) + exec: + println id +} + +process bar { + input: + tuple val(id), path(file, nullable: true) + output: + val file + exec: + sleep 1000L + println file +} + +workflow { + channel.of('foo') | foo | bar | view() +} \ No newline at end of file