Fix samplesheet nesting issues

CHANGELOG.md

@@ -6,6 +6,7 @@
 
 1. Move the unpinned version check to an observer. This makes sure the warning is always shown and not only when importing a function.
 2. Added a missing inherited method to the observer to fix issues with workflow output publishing
+3. Fixed unexpected failures with samplesheet schemas using `anyOf`, `allOf` and `oneOf`
 
 ## Improvements
 

plugins/nf-schema/src/main/nextflow/validation/samplesheet/SamplesheetConverter.groovy

@@ -11,6 +11,7 @@ import nextflow.Nextflow
 import static nextflow.validation.utils.Colors.getLogColors
 import static nextflow.validation.utils.Files.fileToJson
 import static nextflow.validation.utils.Files.fileToObject
+import static nextflow.validation.utils.Common.findDeep
 import nextflow.validation.config.ValidationConfig
 import nextflow.validation.exceptions.SchemaValidationException
 import nextflow.validation.validators.JsonSchemaValidator
@@ -85,7 +86,7 @@ class SamplesheetConverter {
             throw new SchemaValidationException(msg)
         }
 
-        def LinkedHashMap schemaMap = new JsonSlurper().parseText(schemaFile.text) as LinkedHashMap
+        def Map schemaMap = new JsonSlurper().parseText(schemaFile.text) as Map
         def List<String> schemaKeys = schemaMap.keySet() as List<String>
         if(schemaKeys.contains("properties") || !schemaKeys.contains("items")) {
             def msg = "${colors.red}The schema for '${samplesheetFile.toString()}' (${schemaFile.toString()}) is not valid. Please make sure that 'items' is the top level keyword and not 'properties'\n${colors.reset}\n"
@@ -109,12 +110,11 @@ class SamplesheetConverter {
 
         // Convert
         def List samplesheetList = fileToObject(samplesheetFile, schemaFile) as List
-
         this.rows = []
 
         def List channelFormat = samplesheetList.collect { entry ->
             resetMeta()
-            def Object result = formatEntry(entry, schemaMap["items"] as LinkedHashMap)
+            def Object result = formatEntry(entry, schemaMap["items"] as Map)
             if(isMeta()) {
                 if(result instanceof List) {
                     result.add(0,getMeta())
@@ -135,14 +135,14 @@ class SamplesheetConverter {
     This function processes an input value based on a schema. 
     The output will be created for addition to the output channel.
     */
-    private Object formatEntry(Object input, LinkedHashMap schema, String headerPrefix = "") {
+    private Object formatEntry(Object input, Map schema, String headerPrefix = "") {
 
         // Add default values for missing entries
-        input = input != null ? input : schema.containsKey("default") ? schema.default : []
+        input = input != null ? input : findDeep(schema, "default") != null ? findDeep(schema, "default") : []
 
         if (input instanceof Map) {
             def List result = []
-            def Map properties = schema["properties"] as Map
+            def Map properties = findDeep(schema, "properties") as Map
             def Set unusedKeys = input.keySet() - properties.keySet()
 
             // Check for properties in the samplesheet that have not been defined in the schema
@@ -157,12 +157,12 @@ class SamplesheetConverter {
                 // Add the value to the meta map if needed
                 if (metaIds) {
                     metaIds.each {
-                        meta["${it}"] = processMeta(value, schemaValues as LinkedHashMap, prefix)
+                        meta["${it}"] = processMeta(value, schemaValues as Map, prefix)
                     }
                 } 
                 // return the correctly casted value
                 else {
-                    result.add(formatEntry(value, schemaValues as LinkedHashMap, prefix))
+                    result.add(formatEntry(value, schemaValues as Map, prefix))
                 }
             }
             return result
@@ -172,7 +172,7 @@ class SamplesheetConverter {
             input.each {
                 // return the correctly casted value
                 def String prefix = headerPrefix ? "${headerPrefix}${count}." : "${count}."
-                result.add(formatEntry(it, schema["items"] as LinkedHashMap, prefix))
+                result.add(formatEntry(it, findDeep(schema, "items") as Map, prefix))
                 count++
             }
             return result
@@ -192,7 +192,7 @@ class SamplesheetConverter {
     to guess if it should be a file type
     */
     private Object processValue(Object value, Map schemaEntry) {
-        if(!(value instanceof String)) {
+        if(!(value instanceof String) || schemaEntry == null) {
             return value
         }
 
@@ -245,13 +245,13 @@ class SamplesheetConverter {
     This function processes an input value based on a schema. 
     The output will be created for addition to the meta map.
     */
-    private Object processMeta(Object input, LinkedHashMap schema, String headerPrefix) {
+    private Object processMeta(Object input, Map schema, String headerPrefix) {
         // Add default values for missing entries
-        input = input != null ? input : schema.containsKey("default") ? schema.default : []
+        input = input != null ? input : findDeep(schema, "default") != null ? findDeep(schema, "default") : []
 
         if (input instanceof Map) {
             def Map result = [:]
-            def Map properties = schema["properties"] as Map
+            def Map properties = findDeep(schema, "properties") as Map
             def Set unusedKeys = input.keySet() - properties.keySet()
 
             // Check for properties in the samplesheet that have not been defined in the schema
@@ -261,7 +261,7 @@ class SamplesheetConverter {
             properties.each { property, schemaValues ->
                 def value = input[property]
                 def String prefix = headerPrefix ? "${headerPrefix}${property}." : "${property}."
-                result[property] = processMeta(value, schemaValues as LinkedHashMap, prefix)
+                result[property] = processMeta(value, schemaValues as Map, prefix)
             }
             return result
         } else if (input instanceof List) {
@@ -270,7 +270,7 @@ class SamplesheetConverter {
             input.each {
                 // return the correctly casted value
                 def String prefix = headerPrefix ? "${headerPrefix}${count}." : "${count}."
-                result.add(processMeta(it, schema["items"] as LinkedHashMap, prefix))
+                result.add(processMeta(it, findDeep(schema, "items") as Map, prefix))
                 count++
             }
             return result

plugins/nf-schema/src/main/nextflow/validation/utils/Common.groovy

@@ -59,8 +59,17 @@ public class Common {
     //
     // Find a value in a nested map
     //
-    public static Object findDeep(Map m, String key) {
-        if (m.containsKey(key)) return m[key]
-        m.findResult { k, v -> v instanceof Map ? findDeep(v, key) : null }
+    public static Object findDeep(Object m, String key) {
+        if (m instanceof Map) {
+            if (m.containsKey(key)) {
+                return m[key]
+            } else {
+                return m.findResult { k, v -> findDeep(v, key) }
+            }
+        } 
+        else if (m instanceof List) {
+            return m.findResult { element -> findDeep(element, key) }
+        }
+        return null
     }
 }

plugins/nf-schema/src/test/nextflow/validation/SamplesheetConverterTest.groovy

@@ -608,4 +608,28 @@ class SamplesheetConverterTest extends Dsl2Spec{
         stdout.contains("second: [[array_meta:[]], [], [], [], [string1, string2], [25, 26], [25, 26.5], [], [1, 2, 3], [false, true, false], [${getRootString()}/src/testResources/testDir/testFile.txt], [[${getRootString()}/src/testResources/testDir/testFile.txt], [${getRootString()}/src/testResources/testDir/testFile.txt, ${getRootString()}/src/testResources/testDir2/testFile2.txt]]]" as String)
 
     }
+
+    def 'samplesheetToList - nested schema with oneOf/anyOf/allOf' () {
+        given:
+        def SCRIPT_TEXT = '''
+            include { samplesheetToList } from 'plugin/nf-schema'
+
+            workflow {
+                Channel.fromList(samplesheetToList("src/testResources/deeply_nested.yaml", "src/testResources/samplesheet_schema_deeply_nested_anyof.json")).view()       
+            }
+
+        '''
+
+        when:
+        dsl_eval(SCRIPT_TEXT)
+        def stdout = capture
+                .toString()
+                .readLines()
+                .findResults {it.startsWith('[') ? it : null }
+
+        then:
+        noExceptionThrown()
+        stdout.contains("[[mapMeta:this is in a map, arrayMeta:[metaString45, metaString478], otherArrayMeta:[metaString45, metaString478], meta:metaValue, metaMap:[entry1:entry1String, entry2:12.56]], [[string1, string2], string3, 1, 1, ${getRootString()}/file1.txt], [string4, string5, string6], [[string7, string8], [string9, string10]], test]" as String)
+
+    }
 }

plugins/nf-schema/src/testResources/samplesheet_schema_deeply_nested_anyof.json

@@ -0,0 +1,89 @@
+{
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "$id": "https://raw.githubusercontent.com/test/test/master/assets/schema_input.json",
+    "title": "Test schema for samplesheets",
+    "description": "Schema for the file provided with params.input",
+    "type": "array",
+    "items": {
+        "type": "object",
+        "properties": {
+            "map": {
+                "type": "object",
+                "properties": {
+                    "arrayTest": {
+                        "type": "array",
+                        "items": {
+                            "type": "string"
+                        }
+                    }, 
+                    "stringTest": {
+                        "type": "string"
+                    },
+                    "numberTest": {
+                        "type": "number"
+                    },
+                    "integerTest": {
+                        "type": "integer"
+                    },
+                    "fileTest": {
+                        "type": "string",
+                        "format": "file-path"
+                    },
+                    "mapMeta": {
+                        "type": "string",
+                        "meta": "mapMeta"
+                    }
+                }
+            },
+            "array": {
+                "anyOf": [
+                    {
+                        "type": "array",
+                        "items": {
+                            "type": "string"
+                        },
+                        "default": ["test"]
+                    },
+                    {
+                        "type": "string"
+                    }
+                ]
+            },
+            "arrayInArray": {
+                "type": "array",
+                "items": {
+                    "type": "array",
+                    "items": {
+                        "type": "string"
+                    }
+                }
+            },
+            "arrayMeta": {
+                "type": "array",
+                "meta": ["arrayMeta", "otherArrayMeta"],
+                "items": {
+                    "type": "string"
+                }
+            },
+            "value": {
+                "type": "string"
+            },
+            "meta": {
+                "type": "string",
+                "meta": "meta"
+            },
+            "metaMap": {
+                "type": "object",
+                "meta": "metaMap",
+                "properties": {
+                    "entry1": {
+                        "type": "string"
+                    },
+                    "entry2": {
+                        "type": "number"
+                    }
+                }
+            }
+        }
+    }
+}