diff --git a/CHANGELOG.md b/CHANGELOG.md index a10590d..8058b96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [#137](https://github.com/nf-core/bamtofastq/pull/137) - Recover help message - [#141](https://github.com/nf-core/bamtofastq/pull/141) - Update local sbwf and module to adhere to directory structure, update nf-test plugins - [#142](https://github.com/nf-core/bamtofastq/pull/142) - Add tests to local subworkflows, improve `.fai` generation +- [#143](https://github.com/nf-core/bamtofastq/pull/143) - Improve `fai` again and address review comments ### `Dependencies` diff --git a/assets/schema_input.json b/assets/schema_input.json index 55db47d..009dd32 100644 --- a/assets/schema_input.json +++ b/assets/schema_input.json @@ -31,7 +31,7 @@ "type": "string", "errorMessage": "Input filetypes must either be 'bam' or 'cram'", "meta": ["filetype"], - "enums": ["bam", "cram"] + "enum": ["bam", "cram"] } }, "required": ["sample_id", "mapped", "file_type"] diff --git a/modules.json b/modules.json index 7c07829..79f60ec 100644 --- a/modules.json +++ b/modules.json @@ -71,7 +71,7 @@ "nf-core": { "utils_nextflow_pipeline": { "branch": "master", - "git_sha": "05954dab2ff481bcb999f24455da29a5828af08d", + "git_sha": "1a545fcbd762911c21a64ced3dbef99b2b51ac75", "installed_by": ["subworkflows"] }, "utils_nfcore_pipeline": { @@ -81,7 +81,7 @@ }, "utils_nfschema_plugin": { "branch": "master", - "git_sha": "fdc08b8b1ae74f56686ce21f7ea11ad11990ce57", + "git_sha": "ce3424257de288c0ca25f097a68d3a289da23f12", "installed_by": ["subworkflows"] } } diff --git a/modules/local/checkpairedend/meta.yaml b/modules/local/checkpairedend/meta.yaml index 717346c..8145b8a 100644 --- a/modules/local/checkpairedend/meta.yaml +++ b/modules/local/checkpairedend/meta.yaml @@ -56,7 +56,7 @@ output: - samtools: type: string description: The tool name - - samtools --version 2>&1 | sed -n '1p' | awk '{print $2}': + - samtools version | sed '1!d;s/.* //': type: eval description: The expression to obtain the version of samtools @@ -68,7 +68,7 @@ topics: - samtools: type: string description: The tool name - - samtools --version 2>&1 | sed -n '1p' | awk '{print $2}': + - samtools version | sed '1!d;s/.* //': type: eval description: The expression to obtain the version of samtools diff --git a/subworkflows/local/alignment_to_fastq/meta.yaml b/subworkflows/local/alignment_to_fastq/meta.yml similarity index 100% rename from subworkflows/local/alignment_to_fastq/meta.yaml rename to subworkflows/local/alignment_to_fastq/meta.yml diff --git a/subworkflows/local/pre_conversion_qc/meta.yaml b/subworkflows/local/pre_conversion_qc/meta.yml similarity index 100% rename from subworkflows/local/pre_conversion_qc/meta.yaml rename to subworkflows/local/pre_conversion_qc/meta.yml diff --git a/subworkflows/local/prepare_indices/meta.yaml b/subworkflows/local/prepare_indices/meta.yml similarity index 71% rename from subworkflows/local/prepare_indices/meta.yaml rename to subworkflows/local/prepare_indices/meta.yml index 0357c1d..cdea09e 100644 --- a/subworkflows/local/prepare_indices/meta.yaml +++ b/subworkflows/local/prepare_indices/meta.yml @@ -36,10 +36,24 @@ input: Optional channel emitting reference FASTA files (required for CRAM indexing when reference is not embedded in header). structure: - - fasta: - type: file - description: Reference genome FASTA file - pattern: "*.{fa,fna,fasta}" + - fasta_fai: + type: channel + description: | + Channel emitting tuples: [meta, fasta, fai]. `meta.index` should be true if a + pre-built `.fai` is supplied, false to rebuild it via `samtools faidx`. Pass + `[ [id:'none', index:false], [], [] ]` if no reference is needed. + structure: + - meta: + type: map + description: Metadata map with reference id and `index` flag + - fasta: + type: file + description: Reference genome FASTA file + pattern: "*.{fa,fna,fasta}" + - fai: + type: file + description: Reference genome FASTA index file (optional, rebuilt if absent) + pattern: "*.fai" output: - ch_input_indexed: type: channel diff --git a/subworkflows/local/utils_nfcore_bamtofastq_pipeline/main.nf b/subworkflows/local/utils_nfcore_bamtofastq_pipeline/main.nf index cae580f..2fe28c6 100644 --- a/subworkflows/local/utils_nfcore_bamtofastq_pipeline/main.nf +++ b/subworkflows/local/utils_nfcore_bamtofastq_pipeline/main.nf @@ -8,14 +8,13 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -include { UTILS_NFSCHEMA_PLUGIN } from '../../nf-core/utils_nfschema_plugin' -include { paramsSummaryMap } from 'plugin/nf-schema' -include { samplesheetToList } from 'plugin/nf-schema' -include { paramsHelp } from 'plugin/nf-schema' -include { completionEmail } from '../../nf-core/utils_nfcore_pipeline' -include { completionSummary } from '../../nf-core/utils_nfcore_pipeline' -include { UTILS_NFCORE_PIPELINE } from '../../nf-core/utils_nfcore_pipeline' -include { UTILS_NEXTFLOW_PIPELINE } from '../../nf-core/utils_nextflow_pipeline' +include { UTILS_NFSCHEMA_PLUGIN } from '../../nf-core/utils_nfschema_plugin' +include { paramsSummaryMap } from 'plugin/nf-schema' +include { samplesheetToList } from 'plugin/nf-schema' +include { completionEmail } from '../../nf-core/utils_nfcore_pipeline' +include { completionSummary } from '../../nf-core/utils_nfcore_pipeline' +include { UTILS_NFCORE_PIPELINE } from '../../nf-core/utils_nfcore_pipeline' +include { UTILS_NEXTFLOW_PIPELINE } from '../../nf-core/utils_nextflow_pipeline' /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -88,6 +87,7 @@ workflow PIPELINE_INITIALISATION { before_text, after_text, command, + null, ) // @@ -136,7 +136,7 @@ workflow PIPELINE_COMPLETION { plaintext_email // boolean: Send plain-text email instead of HTML outdir // path: Path to output directory where results will be published monochrome_logs // boolean: Disable ANSI colour codes in log output - multiqc_report // string: Path to MultiQC report + multiqc_report // string: Path to MultiQC report main: summary_params = paramsSummaryMap(workflow, parameters_schema: "nextflow_schema.json") @@ -159,11 +159,10 @@ workflow PIPELINE_COMPLETION { } completionSummary(monochrome_logs) - } workflow.onError { - log.error "Pipeline failed. Please refer to troubleshooting docs for common issues: https://nf-co.re/docs/running/troubleshooting" + log.error("Pipeline failed. Please refer to troubleshooting docs for common issues: https://nf-co.re/docs/running/troubleshooting") } } @@ -207,26 +206,27 @@ def genomeExistsError() { // Generate methods description for MultiQC // def toolCitationText() { - // TODO nf-core: Optionally add in-text citation tools to this list. // Can use ternary operators to dynamically construct based conditions, e.g. params["run_xyz"] ? "Tool (Foo et al. 2023)" : "", // Uncomment function in methodsDescriptionText to render in MultiQC report def citation_text = [ "Tools used in the workflow included:", "FastQC (Andrews 2010),", - "MultiQC (Ewels et al. 2016)", - ".", + "fastq_utils (Fonseca & Manning 2023),", + "MultiQC (Ewels et al. 2016),", + "SAMtools (Danecek et al. 2021)", ].join(' ').trim() return citation_text } def toolBibliographyText() { - // TODO nf-core: Optionally add bibliographic entries to this list. // Can use ternary operators to dynamically construct based conditions, e.g. params["run_xyz"] ? "
  • Author (2023) Pub name, Journal, DOI
    • " : "", // Uncomment function in methodsDescriptionText to render in MultiQC report def reference_text = [ "
  • Andrews S, (2010) FastQC, URL: https://www.bioinformatics.babraham.ac.uk/projects/fastqc/).
    • ", - "
  • Ewels, P., Magnusson, M., Lundin, S., & Käller, M. (2016). MultiQC: summarize analysis results for multiple tools and samples in a single report. Bioinformatics , 32(19), 3047–3048. doi: /10.1093/bioinformatics/btw354
    • ", + "
  • Nuno Fonseca & Jonathan Manning (2023) fastq_utils v0.25.2, Zenodo. https://doi.org/10.5281/zenodo.7755574
    • ", + "
  • Ewels, P., Magnusson, M., Lundin, S., & Käller, M. (2016). MultiQC: summarize analysis results for multiple tools and samples in a single report. Bioinformatics , 32(19), 3047–3048. doi: 10.1093/bioinformatics/btw354
    • ", + "
  • Danecek, P., Bonfield, J. K., Liddle, J., Marshall, J., Ohan, V., Pollard, M. O., Whitwham, A., Keane, T., McCarthy, S. A., Davies, R. D., & Li, H. (2021). Twelve years of SAMtools and BCFtools. GigaScience, 10(2), giab008. doi: 10.1093/gigascience/giab008
    • ", ].join(' ').trim() return reference_text @@ -256,12 +256,8 @@ def methodsDescriptionText(mqc_methods_yaml) { meta["nodoi_text"] = meta.manifest_map.doi ? "" : "
  • If available, make sure to update the text to include the Zenodo DOI of version of the pipeline used.
    • " // Tool references - meta["tool_citations"] = "" - meta["tool_bibliography"] = "" - - // TODO nf-core: Only uncomment below if logic in toolCitationText/toolBibliographyText has been filled! - // meta["tool_citations"] = toolCitationText().replaceAll(", \\.", ".").replaceAll("\\. \\.", ".").replaceAll(", \\.", ".") - // meta["tool_bibliography"] = toolBibliographyText() + meta["tool_citations"] = toolCitationText().replaceAll(", \\.", ".").replaceAll("\\. \\.", ".").replaceAll(", \\.", ".") + meta["tool_bibliography"] = toolBibliographyText() def methods_text = mqc_methods_yaml.text diff --git a/subworkflows/nf-core/utils_nextflow_pipeline/main.nf b/subworkflows/nf-core/utils_nextflow_pipeline/main.nf index d6e593e..37939ac 100644 --- a/subworkflows/nf-core/utils_nextflow_pipeline/main.nf +++ b/subworkflows/nf-core/utils_nextflow_pipeline/main.nf @@ -73,11 +73,23 @@ def getWorkflowVersion() { def dumpParametersToJSON(outdir) { def timestamp = new java.util.Date().format('yyyy-MM-dd_HH-mm-ss') def filename = "params_${timestamp}.json" - def temp_pf = new File(workflow.launchDir.toString(), ".${filename}") - def jsonStr = groovy.json.JsonOutput.toJson(params) + def temp_pf = workflow.launchDir.resolve(".${filename}") + def jsonGenerator = new groovy.json.JsonGenerator.Options() + .excludeNulls() + .addConverter(Path) { Path path -> path.toUriString() } + .addConverter(Duration) { Duration duration -> duration.toMillis() } + .addConverter(MemoryUnit) { MemoryUnit memory -> memory.toBytes() } + .addConverter(nextflow.script.types.VersionNumber) { nextflow.script.types.VersionNumber version -> version.toString() } + .build() + def jsonStr = jsonGenerator.toJson(params) temp_pf.text = groovy.json.JsonOutput.prettyPrint(jsonStr) - - nextflow.extension.FilesEx.copyTo(temp_pf.toPath(), "${outdir}/pipeline_info/params_${timestamp}.json") + if (outdir instanceof Path) { + temp_pf.copyTo(outdir.resolve("pipeline_info/${filename}")) + } else if (outdir instanceof String) { + temp_pf.copyTo("${outdir}/pipeline_info/params_${timestamp}.json") + } else { + log.warn("Could not determine type of outdir, parameters JSON file will not be copied to output directory!") + } temp_pf.delete() } diff --git a/subworkflows/nf-core/utils_nfschema_plugin/main.nf b/subworkflows/nf-core/utils_nfschema_plugin/main.nf index 1df8b76..0d9d4e0 100644 --- a/subworkflows/nf-core/utils_nfschema_plugin/main.nf +++ b/subworkflows/nf-core/utils_nfschema_plugin/main.nf @@ -22,6 +22,7 @@ workflow UTILS_NFSCHEMA_PLUGIN { before_text // string: text to show before the help message and parameters summary after_text // string: text to show after the help message and parameters summary command // string: an example command of the pipeline + cli_typecast // boolean: whether to perform typecasting of CLI parameters. Set this to `null` to use the default behaviour main: @@ -38,7 +39,7 @@ workflow UTILS_NFSCHEMA_PLUGIN { } log.info paramsHelp( help_options, - (params.help instanceof String && params.help != "true") ? params.help : "", + (help instanceof String && help != "true") ? help : "", ) exit 0 } @@ -65,6 +66,9 @@ workflow UTILS_NFSCHEMA_PLUGIN { if(parameters_schema) { validateOptions << [parametersSchema: parameters_schema] } + if(cli_typecast != null) { + validateOptions << [cliTypecast: cli_typecast] + } validateParameters(validateOptions) } diff --git a/subworkflows/nf-core/utils_nfschema_plugin/meta.yml b/subworkflows/nf-core/utils_nfschema_plugin/meta.yml index f7d9f02..1d8c75a 100644 --- a/subworkflows/nf-core/utils_nfschema_plugin/meta.yml +++ b/subworkflows/nf-core/utils_nfschema_plugin/meta.yml @@ -25,6 +25,30 @@ input: option. When this input is empty it will automatically use the configured schema or "${projectDir}/nextflow_schema.json" as default. The schema should not be given in this way for meta pipelines. + - help: + type: boolean, string + description: | + Show the help message and exit. When a parameter name is given, show the help message for that parameter instead of the general help message. + - help_full: + type: boolean + description: Show the full help message and exit. + - show_hidden: + type: boolean + description: Show hidden parameters in the help message. + - before_text: + type: string + description: Text to show before the parameters summary and help message. + - after_text: + type: string + description: Text to show after the parameters summary and help message. + - command: + type: string + description: An example command to run the pipeline, to show in the help message and the summary. + - cli_typecast: + type: boolean + description: | + Whether to apply typecasting to the parameters given via the CLI before validation. + Set this to `null` to use the default behavior. output: - dummy_emit: type: boolean diff --git a/tests/fai.nf.test b/tests/fai.nf.test new file mode 100644 index 0000000..f4c65da --- /dev/null +++ b/tests/fai.nf.test @@ -0,0 +1,50 @@ +nextflow_pipeline { + + name "Test pipeline" + script "../main.nf" + tag "pipeline" + tag "pipeline_bamtofastq" + tag "cpu" + + test("-profile test | with fasta and fasta_fai") { + + when { + params { + outdir = "$outputDir" + fasta = 'https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/homo_sapiens/genome/genome.fasta' + fasta_fai = 'https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/homo_sapiens/genome/genome.fasta.fai' + } + } + + then { + // stable_name: All files + folders in ${params.outdir}/ with a stable name + def stable_name = getAllFilesFromDir(params.outdir, relative: true, includeDir: true, ignore: ['pipeline_info/*.{html,json,txt}']) + // stable_path: All files in ${params.outdir}/ with stable content + def stable_path = getAllFilesFromDir(params.outdir, ignoreFile: 'tests/.nftignore') + // bam_files: All bam files + def bam_files = getAllFilesFromDir(params.outdir, include: ['**/*.bam']) + // cram_files: All cram files + def cram_files = getAllFilesFromDir(params.outdir, include: ['**/*.cram']) + // Fasta file for cram verification with nft-bam + def fasta_base = 'https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/' + def fasta = fasta_base + 'genomics/homo_sapiens/genome/genome.fasta' + assert workflow.success + assertAll( + { assert snapshot( + // Number of successful tasks + workflow.trace.succeeded().size(), + // pipeline versions.yml file for multiqc from which Nextflow and pipeline versions are removed (all from the workflow key) + removeFromYamlMap("${outputDir}/pipeline_info/nf_core_bamtofastq_software_mqc_versions.yml", "Workflow"), + // All stable path name, with a relative path + stable_name.isEmpty() ? 'No stable paths' : stable_name, + // All files with stable contents + stable_path.isEmpty() ? 'No stable content' : stable_path, + // All cram files + bam_files.isEmpty() ? 'No BAM files' : bam_files.collect { file -> file.getName() + ":md5," + bam(file.toString()).readsMD5 }, + // All cram files + cram_files.isEmpty() ? 'No CRAM files' : cram_files.collect { file -> file.getName() + ":md5," + cram(file.toString(), fasta).readsMD5 }, + ).match() } + ) + } + } +} diff --git a/tests/fai.nf.test.snap b/tests/fai.nf.test.snap new file mode 100644 index 0000000..b303283 --- /dev/null +++ b/tests/fai.nf.test.snap @@ -0,0 +1,118 @@ +{ + "-profile test | with fasta and fasta_fai": { + "content": [ + 31, + { + "CAT_FASTQ": { + "cat": 9.5 + }, + "CHECKPAIREDEND": { + "samtools": "1.23.1" + }, + "COLLATE_FASTQ_MAP": { + "samtools": "1.23.1" + }, + "COLLATE_FASTQ_UNMAP": { + "samtools": "1.23.1" + }, + "FASTQC_POST_CONVERSION": { + "fastqc": "0.12.1" + }, + "FASTQC_PRE_CONVERSION": { + "fastqc": "0.12.1" + }, + "FASTQUTILS_INFO": { + "fastqutils": "0.25.3" + }, + "SAMTOOLS_FLAGSTAT": { + "samtools": "1.23.1" + }, + "SAMTOOLS_IDXSTATS": { + "samtools": "1.23.1" + }, + "SAMTOOLS_MERGE_UNMAP": { + "samtools": "1.23.1" + }, + "SAMTOOLS_STATS": { + "samtools": "1.23.1" + }, + "SAMTOOLS_VIEW_MAP_MAP": { + "samtools": "1.23.1" + }, + "SAMTOOLS_VIEW_MAP_UNMAP": { + "samtools": "1.23.1" + }, + "SAMTOOLS_VIEW_UNMAP_MAP": { + "samtools": "1.23.1" + }, + "SAMTOOLS_VIEW_UNMAP_UNMAP": { + "samtools": "1.23.1" + } + }, + [ + "fastqc", + "fastqc/test.post_conversion_1_fastqc.html", + "fastqc/test.post_conversion_1_fastqc.zip", + "fastqc/test.post_conversion_2_fastqc.html", + "fastqc/test.post_conversion_2_fastqc.zip", + "fastqc/test.pre_conversion_fastqc.html", + "fastqc/test.pre_conversion_fastqc.zip", + "fastqc/test2.post_conversion_1_fastqc.html", + "fastqc/test2.post_conversion_1_fastqc.zip", + "fastqc/test2.post_conversion_2_fastqc.html", + "fastqc/test2.post_conversion_2_fastqc.zip", + "fastqc/test2.pre_conversion_fastqc.html", + "fastqc/test2.pre_conversion_fastqc.zip", + "fastqutils", + "fastqutils/test.txt", + "fastqutils/test2.txt", + "multiqc", + "multiqc/multiqc_data", + "multiqc/multiqc_data/llms-full.txt", + "multiqc/multiqc_data/multiqc.log", + "multiqc/multiqc_data/multiqc.parquet", + "multiqc/multiqc_data/multiqc_citations.txt", + "multiqc/multiqc_data/multiqc_data.json", + "multiqc/multiqc_data/multiqc_software_versions.txt", + "multiqc/multiqc_data/multiqc_sources.txt", + "multiqc/multiqc_report.html", + "pipeline_info", + "pipeline_info/nf_core_bamtofastq_software_mqc_versions.yml", + "reads", + "reads/test2_1.merged.fastq.gz", + "reads/test2_2.merged.fastq.gz", + "reads/test_1.merged.fastq.gz", + "reads/test_2.merged.fastq.gz", + "samtools", + "samtools/test.flagstat", + "samtools/test.idxstats", + "samtools/test.stats", + "samtools/test2.flagstat", + "samtools/test2.idxstats", + "samtools/test2.stats" + ], + [ + "test.txt:md5,db31682c36cde02f1e8567d130537cac", + "test2.txt:md5,0222a414a69c92608af1580f3c6ce996", + "multiqc_citations.txt:md5,4c806e63a283ec1b7e78cdae3a923d4f", + "test2_1.merged.fastq.gz:md5,a4984fb5a60f80ccbfbfcf1cef9daaf3", + "test2_2.merged.fastq.gz:md5,bde23801e2d60604afb3bd0b0ab014ab", + "test_1.merged.fastq.gz:md5,9b0c08862a5e47e842a485ddad31b871", + "test_2.merged.fastq.gz:md5,40e17982d1231e561d6420234ae29357", + "test.flagstat:md5,a53f3d26e2e9851f7d528442bbfe9781", + "test.idxstats:md5,e179601fa7b8ebce81ac3765206f6c15", + "test.stats:md5,3c3f8fa56674fa987fd912b530f49c3c", + "test2.flagstat:md5,acbd95030a362a7e6491faa87d1b9c2c", + "test2.idxstats:md5,90f01313bc89c7fd096aad051cd71b66", + "test2.stats:md5,7e6537bde51ee4bb4a50562014ae2cd4" + ], + "No BAM files", + "No CRAM files" + ], + "timestamp": "2026-05-07T13:39:25.690773", + "meta": { + "nf-test": "0.9.5", + "nextflow": "26.04.0" + } + } +} \ No newline at end of file diff --git a/workflows/bamtofastq.nf b/workflows/bamtofastq.nf index 0f536cf..d80b48e 100644 --- a/workflows/bamtofastq.nf +++ b/workflows/bamtofastq.nf @@ -58,24 +58,13 @@ workflow BAMTOFASTQ { main: ch_multiqc_files = channel.empty() - ch_fasta = params.fasta - ? channel.fromPath(params.fasta).map { it -> [[id:it.baseName], it, []] }.collect() - : channel.value([ [id:'none'], [], [] ]) - - ch_fai = params.fasta_fai - ? channel.fromPath(params.fasta_fai).map { it -> [[id:it.simpleName], [], it] }.collect() - : channel.value([]) - - fasta_fai = ch_fasta.combine(ch_fai) - .map { meta, fasta, fai -> - if (meta.id == 'none') return [ [id:'none', index:false], [], [] ] - - def new_meta = meta.clone() - new_meta.index = (fai && !(fai instanceof List && fai.isEmpty())) ? true : false - - return [ new_meta, fasta, fai ?: [] ] - } - .first() + fasta_fai = params.fasta + ? channel.fromPath(params.fasta).map { fasta_file -> + def fai_file = params.fasta_fai ? file(params.fasta_fai, checkIfExists: true) : [] + def has_fai = !(fai_file instanceof List && fai_file.isEmpty()) + [ [id: fasta_file.baseName, index: has_fai], fasta_file, fai_file ] + }.collect() + : channel.value([ [id:'none', index:false], [], [] ]) // SUBWORKFLOW: Prepare indices bai/crai/fai if not provided PREPARE_INDICES( @@ -83,7 +72,7 @@ workflow BAMTOFASTQ { fasta_fai ) - ch_fasta_fai = PREPARE_INDICES.out.fasta_fai.collect() + ch_fasta_fai = PREPARE_INDICES.out.fasta_fai // SUBWORKFLOW: Pre conversion QC and stats ch_input = PREPARE_INDICES.out.ch_input_indexed