Nullable input/output paths

docs/process.md

@@ -488,6 +488,22 @@ workflow {
 }
 ```
 
+:::{versionadded} 23.06.0-edge
+:::
+
+By default, the process will be aborted if an input `path` is null. The `nullable` option can be used to ensure that the process is executed even if the path is null:
+
+```groovy
+process foo {
+    input:
+    path('data.txt', nullable: true)
+
+    """
+    [[ -f data.txt ]] your_command --in data.txt || other_command
+    """
+}
+```
+
 ### Multiple input files
 
 A `path` input can also accept a collection of files instead of a single value. In this case, the input variable will be a Groovy list, and you can use it as such.
@@ -927,6 +943,11 @@ Available options:
 `maxDepth`
 : Maximum number of directory levels to visit (default: no limit)
 
+`nullable`
+: :::{versionadded} 23.06.0-edge
+  :::
+: When `true` emit `null` (instead of aborting) if the output path doesn't exist (default: `false`)
+
 `type`
 : Type of paths returned, either `file`, `dir` or `any` (default: `any`, or `file` if the specified file name pattern contains a double star (`**`))
 
@@ -1132,6 +1153,10 @@ output:
 
 In this example, the process is normally expected to produce an `output.txt` file, but in the cases where the file is legitimately missing, the process does not fail. The output channel will only contain values for those processes that produce `output.txt`.
 
+:::{versionadded} 23.06.0-edge
+Path outputs can also be defined as `nullable`. Whereas an `optional` path will emit nothing if the output file does not exist, a `nullable` path will emit a "null" path that can trigger downstream computations, specifically any process that has a `nullable` path input.
+:::
+
 ## When
 
 The `when` block allows you to define a condition that must be satisfied in order to execute the process. The condition can be any expression that returns a boolean value.

modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy

@@ -103,6 +103,7 @@ import nextflow.util.CacheHelper
 import nextflow.util.Escape
 import nextflow.util.LockManager
 import nextflow.util.LoggerHelper
+import nextflow.util.NullPath
 import nextflow.util.TestOnly
 import org.codehaus.groovy.control.CompilerConfiguration
 import org.codehaus.groovy.control.customizers.ASTTransformationCustomizer
@@ -1567,6 +1568,10 @@ class TaskProcessor {
                 def path = param.glob ? splitter.strip(filePattern) : filePattern
                 def file = workDir.resolve(path)
                 def exists = param.followLinks ? file.exists() : file.exists(LinkOption.NOFOLLOW_LINKS)
+                if( !exists && param.nullable ) {
+                    file = new NullPath(path)
+                    exists = true
+                }
                 if( exists )
                     result = [file]
                 else
@@ -1782,7 +1787,11 @@ class TaskProcessor {
         return new FileHolder(source, result)
     }
 
-    protected Path normalizeToPath( obj ) {
+    protected Path normalizeToPath( obj, boolean nullable=false ) {
+
+        if( obj instanceof NullPath && !nullable )
+            throw new ProcessUnrecoverableException("Path value cannot be null")
+
         if( obj instanceof Path )
             return obj
 
@@ -1805,7 +1814,7 @@ class TaskProcessor {
         throw new ProcessUnrecoverableException("Not a valid path value: '$str'")
     }
 
-    protected List<FileHolder> normalizeInputToFiles( Object obj, int count, boolean coerceToPath, FilePorter.Batch batch ) {
+    protected List<FileHolder> normalizeInputToFiles( Object obj, int count, boolean coerceToPath, FilePorter.Batch batch, boolean nullable=false ) {
 
         Collection allItems = obj instanceof Collection ? obj : [obj]
         def len = allItems.size()
@@ -1815,7 +1824,7 @@ class TaskProcessor {
         for( def item : allItems ) {
 
             if( item instanceof Path || coerceToPath ) {
-                def path = normalizeToPath(item)
+                def path = normalizeToPath(item, nullable)
                 def target = executor.isForeignFile(path) ? batch.addToForeign(path) : path
                 def holder = new FileHolder(target)
                 files << holder
@@ -2029,7 +2038,7 @@ class TaskProcessor {
             final param = entry.getKey()
             final val = entry.getValue()
             final fileParam = param as FileInParam
-            final normalized = normalizeInputToFiles(val, count, fileParam.isPathQualifier(), batch)
+            final normalized = normalizeInputToFiles(val, count, fileParam.isPathQualifier(), batch, fileParam.isNullable())
             final resolved = expandWildcards( fileParam.getFilePattern(ctx), normalized )
             ctx.put( param.name, singleItemOrList(resolved, task.type) )
             count += resolved.size()

modules/nextflow/src/main/groovy/nextflow/script/params/FileInParam.groovy

@@ -32,6 +32,8 @@ class FileInParam extends BaseInParam implements PathQualifier {
 
     protected filePattern
 
+    private boolean nullable
+
     private boolean pathQualifier
 
     @Override String getTypeName() { pathQualifier ? 'path' : 'file' }
@@ -120,6 +122,15 @@ class FileInParam extends BaseInParam implements PathQualifier {
             return value
     }
 
+    FileInParam setNullable(boolean value) {
+        this.nullable = value
+        return this
+    }
+
+    boolean isNullable() {
+        return nullable
+    }
+
     @Override
     FileInParam setPathQualifier(boolean flag) {
         pathQualifier = flag

modules/nextflow/src/main/groovy/nextflow/script/params/FileOutParam.groovy

@@ -80,8 +80,16 @@ class FileOutParam extends BaseOutParam implements OutParam, OptionalParam, Path
      */
     boolean followLinks = true
 
+    /**
+     * When true the specified name is interpreted as a glob pattern
+     */
     boolean glob = true
 
+    /**
+     * When true emit null (instead of aborting) if the output path doesn't exist
+     */
+    boolean nullable = false
+
     private GString gstring
     private Closure<String> dynamicObj
     private String filePattern

modules/nextflow/src/main/groovy/nextflow/util/NullPath.groovy

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2023, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package nextflow.util
+
+import java.nio.file.Path
+
+import groovy.transform.EqualsAndHashCode
+import groovy.transform.PackageScope
+import groovy.util.logging.Slf4j
+
+/**
+ * @author : jorge <[email protected]>
+ *
+ */
+@EqualsAndHashCode
+@Slf4j
+class NullPath implements Path {
+
+    @PackageScope
+    @Delegate
+    Path delegate
+
+    NullPath(String path) {
+        delegate = of(path)
+    }
+}

modules/nextflow/src/test/groovy/nextflow/script/params/ParamsOutTest.groovy

@@ -941,7 +941,8 @@ class ParamsOutTest extends Dsl2Spec {
                 separatorChar: ':',
                 glob: true,
                 optional: true,
-                includeInputs: true
+                includeInputs: true,
+                nullable: true
 
               return ''
             }
@@ -963,6 +964,7 @@ class ParamsOutTest extends Dsl2Spec {
         !out0.getGlob()
         !out0.getOptional()
         !out0.getIncludeInputs()
+        !out0.isNullable()
 
         and:
         out1.getMaxDepth() == 5
@@ -973,6 +975,7 @@ class ParamsOutTest extends Dsl2Spec {
         out1.getGlob()
         out1.getOptional()
         out1.getIncludeInputs()
+        out1.isNullable()
     }
 
     def 'should set file options' () {

tests/checks/nullable-path-input-fails.nf/.checks

@@ -0,0 +1,18 @@
+set +e
+
+#
+# run normal mode
+#
+echo ''
+$NXF_RUN 
+[[ $? == 0 ]] && exit 1
+
+
+#
+# RESUME mode
+#
+echo ''
+$NXF_RUN -resume 
+[[ $? == 0 ]] && exit 1
+
+exit 0

tests/checks/nullable-path-output-fails.nf/.checks

@@ -0,0 +1,18 @@
+set +e
+
+#
+# run normal mode
+#
+echo ''
+$NXF_RUN 
+[[ $? == 0 ]] && exit 1
+
+
+#
+# RESUME mode
+#
+echo ''
+$NXF_RUN -resume 
+[[ $? == 0 ]] && exit 1
+
+exit 0

tests/checks/nullable-path-succeeds.nf/.checks

@@ -0,0 +1,16 @@
+set +e
+
+#
+# run normal mode
+#
+echo ''
+$NXF_RUN 
+[[ $? == 0 ]] || false
+
+
+#
+# RESUME mode
+#
+echo ''
+$NXF_RUN -resume 
+[[ $? == 0 ]] || false

tests/nullable-path-input-fails.nf

@@ -0,0 +1,24 @@
+#!/usr/bin/env nextflow
+
+process foo {
+  input:
+    val id
+  output:
+    tuple val(id), path('output.txt', nullable: true)
+  exec:
+    println id
+}
+
+process bar {
+  input:
+    tuple val(id), path(file)
+  output:
+    val file
+  exec:
+    sleep 1000L
+    println file
+}
+
+workflow {
+    channel.of('foo') | foo | bar | view()
+}

tests/nullable-path-output-fails.nf

@@ -0,0 +1,14 @@
+#!/usr/bin/env nextflow
+
+process foo {
+  input:
+    val id
+  output:
+    tuple val(id), path('output.txt')
+  exec:
+    println 'hi'
+}
+
+workflow {
+    foo('foo')
+}

tests/nullable-path-succeeds.nf

@@ -0,0 +1,24 @@
+#!/usr/bin/env nextflow
+
+process foo {
+  input:
+    val id
+  output:
+    tuple val(id), path('output.txt', nullable: true)
+  exec:
+    println id
+}
+
+process bar {
+  input:
+    tuple val(id), path(file, nullable: true)
+  output:
+    val file
+  exec:
+    sleep 1000L
+    println file
+}
+
+workflow {
+    channel.of('foo') | foo | bar | view()
+}