From d3498d5ea5a19adbecb6fe044163dc0a661aec3b Mon Sep 17 00:00:00 2001 From: sainsachiko Date: Tue, 10 Jun 2025 11:38:41 +0100 Subject: [PATCH 01/28] Add blobtk depth to calculate read coverage --- conf/base.config | 5 +++++ conf/modules.config | 6 ++++- docs/output.md | 13 +++++++++++ modules/local/blobtk_depth.nf | 34 +++++++++++++++++++++++++++++ subworkflows/local/convert_stats.nf | 10 +++++++-- 5 files changed, 65 insertions(+), 3 deletions(-) create mode 100644 modules/local/blobtk_depth.nf diff --git a/conf/base.config b/conf/base.config index 44123158..f86ea5b2 100644 --- a/conf/base.config +++ b/conf/base.config @@ -133,4 +133,9 @@ process { // 100k reads per hour for PacBio, 50m for HiC time = { 1.h * Math.ceil( meta.read_count / (meta.datatype == "pacbio" ? 100000 : 50000000)) * task.attempt } } + + withName: BLOBTK_DEPTH { + cpus = { log_increase_cpus(4, 2*task.attempt, 1, 2) } + memory = { 1.GB * Math.ceil( meta.read_count / 1000000 ) * task.attempt } + } } diff --git a/conf/modules.config b/conf/modules.config index 3c56abe3..44e70ee4 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -163,7 +163,7 @@ process { ext.args = { (meta.datatype == "pacbio" ? "-y pbccs " : "") + "-O bam" } } - withName: '.*:CONVERT_STATS:SAMTOOLS_.*' { + withName: '.*:CONVERT_STATS:SAMTOOLS_.*|BLOBTK_DEPTH' { beforeScript = { "export REF_PATH=spoof"} publishDir = [ path: { "${params.outdir}/read_mapping/${meta.datatype}" }, @@ -172,4 +172,8 @@ process { ] } + withName: BLOBTK_DEPTH { + ext.prefix = { "${cram.baseName}" } + } + } diff --git a/docs/output.md b/docs/output.md index c64b87b3..b8c92697 100644 --- a/docs/output.md +++ b/docs/output.md @@ -89,6 +89,19 @@ The filtered PacBio reads are aligned with `MINIMAP2_ALIGN`. The sorted and merg If provided using the `--header` option, all output alignments (`*.cram` or `*.bam`) will include any additional metadata supplied as a SAM header template, replacing the existing _@HD_ and _@SD_ entries (note that this behaviour can be altered by modifying the `ext.args` for `SAMTOOLS_REHEADER` in `modules.config`). +### Read coverage + +Read coverage of the output alignment file is calculated with [blotk_depth](https://github.com/genomehubs/blobtk/wiki/blobtk-depth). + +
+Output files + +- `read_mapping` + - `` + - `.unmasked...[cr|b]am.regions.bed.gz`: Read coverage in BED format + +
+ ### Statistics The output alignments, along with the index, are used to calculate mapping statistics. Output files are generated using `SAMTOOLS_STATS`, `SAMTOOLS_FLAGSTAT` and `SAMTOOLS_IDXSTATS`. diff --git a/modules/local/blobtk_depth.nf b/modules/local/blobtk_depth.nf new file mode 100644 index 00000000..53a5ec74 --- /dev/null +++ b/modules/local/blobtk_depth.nf @@ -0,0 +1,34 @@ +process BLOBTK_DEPTH { + tag "${meta.id}" + label 'process_single' + + if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { + exit 1, "BLOBTOOLKIT_DEPTH module does not support Conda. Please use Docker / Singularity / Podman instead." + } + container "docker.io/genomehubs/blobtk:0.6.5" + + input: + tuple val(meta), path(cram), path(bai) + + output: + tuple val(meta), path('*.regions.bed.gz') , emit: bed + path "versions.yml" , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + """ + blobtk depth \\ + -c ${cram} \\ + $args \\ + -O ${prefix}.regions.bed.gz \\ + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + blobtk: \$(blobtk --version | cut -d' ' -f2) + END_VERSIONS + """ +} diff --git a/subworkflows/local/convert_stats.nf b/subworkflows/local/convert_stats.nf index 76e3f6d1..3d2b6e59 100644 --- a/subworkflows/local/convert_stats.nf +++ b/subworkflows/local/convert_stats.nf @@ -2,6 +2,7 @@ // Convert BAM to CRAM, create index and calculate statistics // +include { BLOBTK_DEPTH } from '../../modules/local/blobtk_depth' include { CRUMBLE } from '../../modules/nf-core/crumble/main' include { SAMTOOLS_VIEW as SAMTOOLS_CRAM } from '../../modules/nf-core/samtools/view/main' include { SAMTOOLS_VIEW as SAMTOOLS_REINDEX } from '../../modules/nf-core/samtools/view/main' @@ -81,15 +82,20 @@ workflow CONVERT_STATS { // Set the BAM and BAI channels for emission ch_bam = SAMTOOLS_REINDEX.out.bam - ch_bai = SAMTOOLS_REINDEX.out.bai + ch_csi = SAMTOOLS_REINDEX.out.csi // If using BAM for stats, use the reindexed BAM if ( !("cram" in outfmt_options) ) { - ch_data_for_stats = ch_bam.join ( ch_bai ) + ch_data_for_stats = ch_bam.join ( ch_csi ) } } + // Calculate read depth + BLOBTK_DEPTH ( ch_data_for_stats ) + ch_versions = ch_versions.mix( BLOBTK_DEPTH.out.versions.first() ) + + // Calculate statistics SAMTOOLS_STATS ( ch_data_for_stats, fasta ) ch_versions = ch_versions.mix( SAMTOOLS_STATS.out.versions.first() ) From 937f0a5ea612157e72d410354d6b36a79119d5df Mon Sep 17 00:00:00 2001 From: Hanh Hoang <134130358+sainsachiko@users.noreply.github.com> Date: Fri, 20 Jun 2025 15:32:23 +0100 Subject: [PATCH 02/28] Apply suggestions from code review - Remove cpus limit as blobtk depth is single-threaded - Adjust coverage file name - Fix typos Co-authored-by: Matthieu Muffato --- conf/base.config | 1 - docs/output.md | 4 ++-- modules/local/blobtk_depth.nf | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/conf/base.config b/conf/base.config index f86ea5b2..a7954a04 100644 --- a/conf/base.config +++ b/conf/base.config @@ -135,7 +135,6 @@ process { } withName: BLOBTK_DEPTH { - cpus = { log_increase_cpus(4, 2*task.attempt, 1, 2) } memory = { 1.GB * Math.ceil( meta.read_count / 1000000 ) * task.attempt } } } diff --git a/docs/output.md b/docs/output.md index b8c92697..e86aa5a9 100644 --- a/docs/output.md +++ b/docs/output.md @@ -91,14 +91,14 @@ If provided using the `--header` option, all output alignments (`*.cram` or `*.b ### Read coverage -Read coverage of the output alignment file is calculated with [blotk_depth](https://github.com/genomehubs/blobtk/wiki/blobtk-depth). +Read coverage of the output alignment file is calculated with [blobtk depth](https://github.com/genomehubs/blobtk/wiki/blobtk-depth).
Output files - `read_mapping` - `` - - `.unmasked...[cr|b]am.regions.bed.gz`: Read coverage in BED format + - `.unmasked...[cr|b]am.coverage.bedGraph.gz`: Read coverage in bedGraph format
diff --git a/modules/local/blobtk_depth.nf b/modules/local/blobtk_depth.nf index 53a5ec74..653cf89a 100644 --- a/modules/local/blobtk_depth.nf +++ b/modules/local/blobtk_depth.nf @@ -24,7 +24,7 @@ process BLOBTK_DEPTH { blobtk depth \\ -c ${cram} \\ $args \\ - -O ${prefix}.regions.bed.gz \\ + -O ${prefix}.coverage.bedGraph.gz \\ cat <<-END_VERSIONS > versions.yml "${task.process}": From 8e2a7ccd771c09a40f23c2c16a59014eb509db00 Mon Sep 17 00:00:00 2001 From: sainsachiko Date: Mon, 23 Jun 2025 09:39:03 +0100 Subject: [PATCH 03/28] Add BGZIP to compress bedGraph coverage --- conf/base.config | 3 +- conf/modules.config | 7 +- modules.json | 5 + modules/local/blobtk_depth.nf | 7 +- modules/nf-core/tabix/bgzip/environment.yml | 9 + modules/nf-core/tabix/bgzip/main.nf | 56 +++++ modules/nf-core/tabix/bgzip/meta.yml | 61 +++++ .../tabix/bgzip/tests/bgzip_compress.config | 5 + .../nf-core/tabix/bgzip/tests/main.nf.test | 111 +++++++++ .../tabix/bgzip/tests/main.nf.test.snap | 218 ++++++++++++++++++ .../nf-core/tabix/bgzip/tests/vcf_none.config | 5 + subworkflows/local/convert_stats.nf | 5 + 12 files changed, 486 insertions(+), 6 deletions(-) create mode 100644 modules/nf-core/tabix/bgzip/environment.yml create mode 100644 modules/nf-core/tabix/bgzip/main.nf create mode 100644 modules/nf-core/tabix/bgzip/meta.yml create mode 100644 modules/nf-core/tabix/bgzip/tests/bgzip_compress.config create mode 100644 modules/nf-core/tabix/bgzip/tests/main.nf.test create mode 100644 modules/nf-core/tabix/bgzip/tests/main.nf.test.snap create mode 100644 modules/nf-core/tabix/bgzip/tests/vcf_none.config diff --git a/conf/base.config b/conf/base.config index a7954a04..df2c4f42 100644 --- a/conf/base.config +++ b/conf/base.config @@ -134,7 +134,8 @@ process { time = { 1.h * Math.ceil( meta.read_count / (meta.datatype == "pacbio" ? 100000 : 50000000)) * task.attempt } } - withName: BLOBTK_DEPTH { + withName: 'BLOBTK_DEPTH|BGZIP_BEDGRAPH' { memory = { 1.GB * Math.ceil( meta.read_count / 1000000 ) * task.attempt } } + } diff --git a/conf/modules.config b/conf/modules.config index 44e70ee4..a9314df6 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -163,7 +163,7 @@ process { ext.args = { (meta.datatype == "pacbio" ? "-y pbccs " : "") + "-O bam" } } - withName: '.*:CONVERT_STATS:SAMTOOLS_.*|BLOBTK_DEPTH' { + withName: '.*:CONVERT_STATS:SAMTOOLS_.*|BGZIP_BEDGRAPH' { beforeScript = { "export REF_PATH=spoof"} publishDir = [ path: { "${params.outdir}/read_mapping/${meta.datatype}" }, @@ -172,8 +172,11 @@ process { ] } - withName: BLOBTK_DEPTH { + withName: 'BLOBTK_DEPTH' { ext.prefix = { "${cram.baseName}" } } + withName: 'BGZIP_BEDGRAPH' { + ext.prefix = { "${input.baseName}" } + } } diff --git a/modules.json b/modules.json index a3700017..77dc2b6d 100644 --- a/modules.json +++ b/modules.json @@ -77,6 +77,11 @@ "installed_by": ["modules"], "patch": "modules/nf-core/samtools/view/samtools-view.diff" }, + "tabix/bgzip": { + "branch": "master", + "git_sha": "05954dab2ff481bcb999f24455da29a5828af08d", + "installed_by": ["modules"] + }, "untar": { "branch": "master", "git_sha": "666652151335353eef2fcd58880bcef5bc2928e1", diff --git a/modules/local/blobtk_depth.nf b/modules/local/blobtk_depth.nf index 653cf89a..1d96340f 100644 --- a/modules/local/blobtk_depth.nf +++ b/modules/local/blobtk_depth.nf @@ -11,8 +11,8 @@ process BLOBTK_DEPTH { tuple val(meta), path(cram), path(bai) output: - tuple val(meta), path('*.regions.bed.gz') , emit: bed - path "versions.yml" , emit: versions + tuple val(meta), path('*.coverage.bedGraph') , emit: bedgraph + path "versions.yml" , emit: versions when: task.ext.when == null || task.ext.when @@ -24,7 +24,8 @@ process BLOBTK_DEPTH { blobtk depth \\ -c ${cram} \\ $args \\ - -O ${prefix}.coverage.bedGraph.gz \\ + -O ${prefix}.coverage.bedGraph \\ + cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/tabix/bgzip/environment.yml b/modules/nf-core/tabix/bgzip/environment.yml new file mode 100644 index 00000000..6221bb53 --- /dev/null +++ b/modules/nf-core/tabix/bgzip/environment.yml @@ -0,0 +1,9 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/environment-schema.json +channels: + - conda-forge + - bioconda + +dependencies: + - bioconda::htslib=1.21 + - bioconda::tabix=1.11 diff --git a/modules/nf-core/tabix/bgzip/main.nf b/modules/nf-core/tabix/bgzip/main.nf new file mode 100644 index 00000000..c7e7462f --- /dev/null +++ b/modules/nf-core/tabix/bgzip/main.nf @@ -0,0 +1,56 @@ +process TABIX_BGZIP { + tag "$meta.id" + label 'process_single' + + conda "${moduleDir}/environment.yml" + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/92/92859404d861ae01afb87e2b789aebc71c0ab546397af890c7df74e4ee22c8dd/data' : + 'community.wave.seqera.io/library/htslib:1.21--ff8e28a189fbecaa' }" + + input: + tuple val(meta), path(input) + + output: + tuple val(meta), path("${output}"), emit: output + tuple val(meta), path("*.gzi") , emit: gzi, optional: true + path "versions.yml" , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + prefix = task.ext.prefix ?: "${meta.id}" + in_bgzip = ["gz", "bgz", "bgzf"].contains(input.getExtension()) + extension = in_bgzip ? input.getBaseName().tokenize(".")[-1] : input.getExtension() + output = in_bgzip ? "${prefix}.${extension}" : "${prefix}.${extension}.gz" + command = in_bgzip ? '-d' : '' + // Name the index according to $prefix, unless a name has been requested + split_args = args.split(' +|=') + if ((split_args.contains('-i') || split_args.contains('--index')) && !split_args.contains('-I') && !split_args.contains('--index-name')) { + args = args + " -I ${output}.gzi" + } + """ + bgzip $command -c $args -@${task.cpus} $input > ${output} + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + tabix: \$(echo \$(tabix -h 2>&1) | sed 's/^.*Version: //; s/ .*\$//') + END_VERSIONS + """ + + stub: + prefix = task.ext.prefix ?: "${meta.id}" + in_bgzip = ["gz", "bgz", "bgzf"].contains(input.getExtension()) + output = in_bgzip ? input.getBaseName() : "${prefix}.${input.getExtension()}.gz" + + """ + echo "" | gzip > ${output} + touch ${output}.gzi + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + tabix: \$(echo \$(tabix -h 2>&1) | sed 's/^.*Version: //; s/ .*\$//') + END_VERSIONS + """ +} diff --git a/modules/nf-core/tabix/bgzip/meta.yml b/modules/nf-core/tabix/bgzip/meta.yml new file mode 100644 index 00000000..7e44a077 --- /dev/null +++ b/modules/nf-core/tabix/bgzip/meta.yml @@ -0,0 +1,61 @@ +name: tabix_bgzip +description: Compresses/decompresses files +keywords: + - compress + - decompress + - bgzip + - tabix +tools: + - bgzip: + description: | + Bgzip compresses or decompresses files in a similar manner to, and compatible with, gzip. + homepage: https://www.htslib.org/doc/tabix.html + documentation: http://www.htslib.org/doc/bgzip.html + doi: 10.1093/bioinformatics/btp352 + licence: ["MIT"] + identifier: biotools:tabix +input: + - - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - input: + type: file + description: file to compress or to decompress +output: + - output: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - ${output}: + type: file + description: Output compressed/decompressed file + pattern: "*." + - gzi: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - '*.gzi': + type: file + description: Optional gzip index file for compressed inputs + pattern: "*.gzi" + - versions: + - versions.yml: + type: file + description: File containing software versions + pattern: "versions.yml" +authors: + - "@joseespinosa" + - "@drpatelh" + - "@maxulysse" + - "@nvnieuwk" +maintainers: + - "@joseespinosa" + - "@drpatelh" + - "@maxulysse" + - "@nvnieuwk" diff --git a/modules/nf-core/tabix/bgzip/tests/bgzip_compress.config b/modules/nf-core/tabix/bgzip/tests/bgzip_compress.config new file mode 100644 index 00000000..6b6ff55f --- /dev/null +++ b/modules/nf-core/tabix/bgzip/tests/bgzip_compress.config @@ -0,0 +1,5 @@ +process { + withName: TABIX_BGZIP { + ext.args = ' -i' + } +} diff --git a/modules/nf-core/tabix/bgzip/tests/main.nf.test b/modules/nf-core/tabix/bgzip/tests/main.nf.test new file mode 100644 index 00000000..00e7c098 --- /dev/null +++ b/modules/nf-core/tabix/bgzip/tests/main.nf.test @@ -0,0 +1,111 @@ +nextflow_process { + + name "Test Process TABIX_BGZIP" + script "../main.nf" + process "TABIX_BGZIP" + + tag "modules" + tag "modules_nfcore" + tag "tabix" + tag "tabix/bgzip" + + test("sarscov2_vcf_bgzip_compress") { + when { + process { + """ + input[0] = [ + [ id:'bgzip_test' ], + [ file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/vcf/test.vcf', checkIfExists: true) ] + ] + """ + } + } + + then { + assertAll ( + { assert process.success }, + { assert snapshot(process.out).match() }, + { assert snapshot( + file(process.out.output[0][1]).name + ).match("bgzip_test") + } + ) + } + } + + test("homo_genome_bedgz_compress") { + when { + process { + """ + input[0] = [ + [ id:'bedgz_test' ], + [ file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/genome.bed.gz', checkIfExists: true) ] + ] + """ + } + } + + then { + assertAll ( + { assert process.success }, + { assert snapshot(process.out).match() }, + { assert snapshot( + file(process.out.output[0][1]).name + ).match("bedgz_test") + } + ) + } + } + + test("sarscov2_vcf_bgzip_compress_stub") { + options '-stub' + config "./bgzip_compress.config" + + when { + process { + """ + input[0] = [ + [ id:"test_stub" ], + [ file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/vcf/test.vcf', checkIfExists: true) ] + ] + """ + } + } + + then { + assertAll ( + { assert process.success }, + { assert snapshot(process.out).match() }, + { assert snapshot( + file(process.out.output[0][1]).name + ).match("test_stub") + } + ) + } + } + + test("sarscov2_vcf_bgzip_compress_gzi") { + config "./bgzip_compress.config" + when { + process { + """ + input[0] = [ + [ id:"gzi_compress_test" ], + [ file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/vcf/test.vcf', checkIfExists: true) ] + ] + """ + } + } + + then { + assertAll ( + { assert process.success }, + { assert snapshot(process.out).match() }, + { assert snapshot( + file(process.out.gzi[0][1]).name + ).match("gzi_compress_test") + } + ) + } + } +} diff --git a/modules/nf-core/tabix/bgzip/tests/main.nf.test.snap b/modules/nf-core/tabix/bgzip/tests/main.nf.test.snap new file mode 100644 index 00000000..c605d54a --- /dev/null +++ b/modules/nf-core/tabix/bgzip/tests/main.nf.test.snap @@ -0,0 +1,218 @@ +{ + "gzi_compress_test": { + "content": [ + "gzi_compress_test.vcf.gz.gzi" + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.04.2" + }, + "timestamp": "2024-02-19T14:52:29.328146" + }, + "homo_genome_bedgz_compress": { + "content": [ + { + "0": [ + [ + { + "id": "bedgz_test" + }, + "bedgz_test.bed:md5,87a15eb9c2ff20ccd5cd8735a28708f7" + ] + ], + "1": [ + + ], + "2": [ + "versions.yml:md5,8721da9158d25c69b2215adf9cdc9fde" + ], + "gzi": [ + + ], + "output": [ + [ + { + "id": "bedgz_test" + }, + "bedgz_test.bed:md5,87a15eb9c2ff20ccd5cd8735a28708f7" + ] + ], + "versions": [ + "versions.yml:md5,8721da9158d25c69b2215adf9cdc9fde" + ] + } + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "24.10.5" + }, + "timestamp": "2025-03-26T13:52:19.285035543" + }, + "test_stub": { + "content": [ + "test_stub.vcf.gz" + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.04.2" + }, + "timestamp": "2024-02-19T14:52:20.811489" + }, + "sarscov2_vcf_bgzip_compress": { + "content": [ + { + "0": [ + [ + { + "id": "bgzip_test" + }, + "bgzip_test.vcf.gz:md5,8e722884ffb75155212a3fc053918766" + ] + ], + "1": [ + + ], + "2": [ + "versions.yml:md5,8721da9158d25c69b2215adf9cdc9fde" + ], + "gzi": [ + + ], + "output": [ + [ + { + "id": "bgzip_test" + }, + "bgzip_test.vcf.gz:md5,8e722884ffb75155212a3fc053918766" + ] + ], + "versions": [ + "versions.yml:md5,8721da9158d25c69b2215adf9cdc9fde" + ] + } + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "24.10.5" + }, + "timestamp": "2025-03-26T13:52:15.290470496" + }, + "sarscov2_vcf_bgzip_compress_gzi": { + "content": [ + { + "0": [ + [ + { + "id": "gzi_compress_test" + }, + "gzi_compress_test.vcf.gz:md5,8e722884ffb75155212a3fc053918766" + ] + ], + "1": [ + [ + { + "id": "gzi_compress_test" + }, + "gzi_compress_test.vcf.gz.gzi:md5,26fd00d4e26141cd11561f6e7d4a2ad0" + ] + ], + "2": [ + "versions.yml:md5,8721da9158d25c69b2215adf9cdc9fde" + ], + "gzi": [ + [ + { + "id": "gzi_compress_test" + }, + "gzi_compress_test.vcf.gz.gzi:md5,26fd00d4e26141cd11561f6e7d4a2ad0" + ] + ], + "output": [ + [ + { + "id": "gzi_compress_test" + }, + "gzi_compress_test.vcf.gz:md5,8e722884ffb75155212a3fc053918766" + ] + ], + "versions": [ + "versions.yml:md5,8721da9158d25c69b2215adf9cdc9fde" + ] + } + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "24.10.5" + }, + "timestamp": "2025-03-26T13:52:26.577148245" + }, + "bgzip_test": { + "content": [ + "bgzip_test.vcf.gz" + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.04.2" + }, + "timestamp": "2024-02-19T14:52:03.768295" + }, + "bedgz_test": { + "content": [ + "bedgz_test.bed" + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.04.2" + }, + "timestamp": "2024-02-19T14:52:12.453855" + }, + "sarscov2_vcf_bgzip_compress_stub": { + "content": [ + { + "0": [ + [ + { + "id": "test_stub" + }, + "test_stub.vcf.gz:md5,68b329da9893e34099c7d8ad5cb9c940" + ] + ], + "1": [ + [ + { + "id": "test_stub" + }, + "test_stub.vcf.gz.gzi:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "2": [ + "versions.yml:md5,8721da9158d25c69b2215adf9cdc9fde" + ], + "gzi": [ + [ + { + "id": "test_stub" + }, + "test_stub.vcf.gz.gzi:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "output": [ + [ + { + "id": "test_stub" + }, + "test_stub.vcf.gz:md5,68b329da9893e34099c7d8ad5cb9c940" + ] + ], + "versions": [ + "versions.yml:md5,8721da9158d25c69b2215adf9cdc9fde" + ] + } + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "24.10.5" + }, + "timestamp": "2025-03-26T13:52:22.850987971" + } +} \ No newline at end of file diff --git a/modules/nf-core/tabix/bgzip/tests/vcf_none.config b/modules/nf-core/tabix/bgzip/tests/vcf_none.config new file mode 100644 index 00000000..f3a3c467 --- /dev/null +++ b/modules/nf-core/tabix/bgzip/tests/vcf_none.config @@ -0,0 +1,5 @@ +process { + withName: TABIX_BGZIP { + ext.args = '' + } +} diff --git a/subworkflows/local/convert_stats.nf b/subworkflows/local/convert_stats.nf index 3d2b6e59..2480254a 100644 --- a/subworkflows/local/convert_stats.nf +++ b/subworkflows/local/convert_stats.nf @@ -3,6 +3,7 @@ // include { BLOBTK_DEPTH } from '../../modules/local/blobtk_depth' +include { TABIX_BGZIP as BGZIP_BEDGRAPH } from '../../modules/nf-core/tabix/bgzip/main' include { CRUMBLE } from '../../modules/nf-core/crumble/main' include { SAMTOOLS_VIEW as SAMTOOLS_CRAM } from '../../modules/nf-core/samtools/view/main' include { SAMTOOLS_VIEW as SAMTOOLS_REINDEX } from '../../modules/nf-core/samtools/view/main' @@ -95,6 +96,9 @@ workflow CONVERT_STATS { BLOBTK_DEPTH ( ch_data_for_stats ) ch_versions = ch_versions.mix( BLOBTK_DEPTH.out.versions.first() ) + BGZIP_BEDGRAPH ( BLOBTK_DEPTH.out.bedgraph ) + ch_versions = ch_versions.mix( BGZIP_BEDGRAPH.out.versions.first() ) + // Calculate statistics SAMTOOLS_STATS ( ch_data_for_stats, fasta ) @@ -110,6 +114,7 @@ workflow CONVERT_STATS { SAMTOOLS_IDXSTATS ( ch_data_for_stats ) ch_versions = ch_versions.mix( SAMTOOLS_IDXSTATS.out.versions.first() ) + emit: bam = ch_bam // channel: [ val(meta), /path/to/bam ] (optional) bai = ch_bai // channel: [ val(meta), /path/to/bai ] (optional) From 1f5e0c3a63003302c5ed2465b4f3ec608f39b64f Mon Sep 17 00:00:00 2001 From: sainsachiko Date: Mon, 23 Jun 2025 10:02:41 +0100 Subject: [PATCH 04/28] Update tabix/bgzip --- modules.json | 2 +- modules/nf-core/tabix/bgzip/meta.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules.json b/modules.json index 77dc2b6d..7d00cee4 100644 --- a/modules.json +++ b/modules.json @@ -79,7 +79,7 @@ }, "tabix/bgzip": { "branch": "master", - "git_sha": "05954dab2ff481bcb999f24455da29a5828af08d", + "git_sha": "52a49a8807214ba93b0fdc9fa401876d9845f62a", "installed_by": ["modules"] }, "untar": { diff --git a/modules/nf-core/tabix/bgzip/meta.yml b/modules/nf-core/tabix/bgzip/meta.yml index 7e44a077..72b9b99c 100644 --- a/modules/nf-core/tabix/bgzip/meta.yml +++ b/modules/nf-core/tabix/bgzip/meta.yml @@ -40,7 +40,7 @@ output: description: | Groovy Map containing sample information e.g. [ id:'test', single_end:false ] - - '*.gzi': + - "*.gzi": type: file description: Optional gzip index file for compressed inputs pattern: "*.gzi" From 07fc4460ff3ef160582d41f12fbf443a29e8f000 Mon Sep 17 00:00:00 2001 From: sainsachiko Date: Fri, 27 Jun 2025 14:13:48 +0100 Subject: [PATCH 05/28] Fix: BAM/CRAM reheader and BLOBTK DEPTH cache --- conf/base.config | 2 +- conf/modules.config | 19 ++++++--- modules/local/change_name.nf | 43 +++++++++++++++++++ modules/local/samtools_replaceheader.nf | 2 +- subworkflows/local/convert_stats.nf | 55 +++++++++++++++++-------- workflows/readmapping.nf | 18 +------- 6 files changed, 97 insertions(+), 42 deletions(-) create mode 100644 modules/local/change_name.nf diff --git a/conf/base.config b/conf/base.config index df2c4f42..1b17b6ae 100644 --- a/conf/base.config +++ b/conf/base.config @@ -135,7 +135,7 @@ process { } withName: 'BLOBTK_DEPTH|BGZIP_BEDGRAPH' { - memory = { 1.GB * Math.ceil( meta.read_count / 1000000 ) * task.attempt } + memory = { 4.GB * task.attempt } } } diff --git a/conf/modules.config b/conf/modules.config index a9314df6..cacfbbc5 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -26,7 +26,7 @@ process { // If custom header provided, this is inserted in place of existing // @HD and @SQ lines, while preserving any other header entries withName: SAMTOOLS_REHEADER { - ext.prefix = { "${meta.id}.reheader" } + ext.prefix = { "${file.baseName}" } } withName: SAMTOOLS_COLLATETOFASTA { @@ -163,20 +163,27 @@ process { ext.args = { (meta.datatype == "pacbio" ? "-y pbccs " : "") + "-O bam" } } - withName: '.*:CONVERT_STATS:SAMTOOLS_.*|BGZIP_BEDGRAPH' { - beforeScript = { "export REF_PATH=spoof"} + withName: '.*:CONVERT_STATS:SAMTOOLS_.*|BGZIP_BEDGRAPH|CHANGE_NAME' { + beforeScript = { "export REF_PATH=spoof" } publishDir = [ - path: { "${params.outdir}/read_mapping/${meta.datatype}" }, + path: { "${params.outdir}/read_mapping/${meta.datatype}${meta.merged ? '/merged' : ''}" }, mode: params.publish_dir_mode, - saveAs: { filename -> filename.equals('versions.yml') ? null : filename } + // Only publish BAM/CRAM from SAMTOOLS_REHEADER if params.header is true, else from: CHANGE_NAME and SAMTOOLS_CRAM + saveAs: { filename -> + filename == 'versions.yml' || ((task.process =~ /CHANGE_NAME|SAMTOOLS_CRAM/) && params.header) ? null : filename + } ] } withName: 'BLOBTK_DEPTH' { - ext.prefix = { "${cram.baseName}" } + ext.prefix = { "${meta.fasta}.${meta.datatype}.${meta.id}" } } withName: 'BGZIP_BEDGRAPH' { ext.prefix = { "${input.baseName}" } } + + withName: 'CHANGE_NAME' { + ext.prefix = { "${meta.fasta}.${meta.datatype}.${meta.id}" } + } } diff --git a/modules/local/change_name.nf b/modules/local/change_name.nf new file mode 100644 index 00000000..edca6926 --- /dev/null +++ b/modules/local/change_name.nf @@ -0,0 +1,43 @@ +process CHANGE_NAME { + tag "$meta.id" + label 'process_single' + + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'https://depot.galaxyproject.org/singularity/ubuntu:20.04' : + 'docker.io/ubuntu:20.04' }" + + input: + tuple val(meta), path(file) + + output: + tuple val(meta), path("*.${file.extension}"), emit: file + path "versions.yml", emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + def VERSION = "9.1" // WARN: Version information not provided by tool on CLI. Please update this string when bumping container versions. + def prefix = task.ext.prefix ?: "${meta.id}" + def new_fn = "${prefix}.${file.extension}" + """ + mv ${file} ${new_fn} + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + coreutils: $VERSION + END_VERSIONS + """ + + stub: + def VERSION = "9.1" // WARN: Version information not provided by tool on CLI. Please update this string when bumping container versions. + """ + touch ${new_fn} + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + coreutils: $VERSION + END_VERSIONS + """ +} + diff --git a/modules/local/samtools_replaceheader.nf b/modules/local/samtools_replaceheader.nf index 6143ac57..1c7d7f49 100644 --- a/modules/local/samtools_replaceheader.nf +++ b/modules/local/samtools_replaceheader.nf @@ -8,7 +8,7 @@ process SAMTOOLS_REHEADER { 'biocontainers/samtools:1.21--h50ea8bc_0' }" input: - tuple val(meta), path(file) + tuple val(meta), path(file, stageAs: "input/*") path(header) output: diff --git a/subworkflows/local/convert_stats.nf b/subworkflows/local/convert_stats.nf index 2480254a..f6994f83 100644 --- a/subworkflows/local/convert_stats.nf +++ b/subworkflows/local/convert_stats.nf @@ -2,20 +2,28 @@ // Convert BAM to CRAM, create index and calculate statistics // -include { BLOBTK_DEPTH } from '../../modules/local/blobtk_depth' -include { TABIX_BGZIP as BGZIP_BEDGRAPH } from '../../modules/nf-core/tabix/bgzip/main' + +// MODULE: local modules +include { SAMTOOLS_REHEADER as SAMTOOLS_REHEADER_BAM } from '../../modules/local/samtools_replaceheader' +include { SAMTOOLS_REHEADER as SAMTOOLS_REHEADER_CRAM } from '../../modules/local/samtools_replaceheader' +include { CHANGE_NAME } from '../../modules/local/change_name' + +// MODULE: nf-core modules include { CRUMBLE } from '../../modules/nf-core/crumble/main' include { SAMTOOLS_VIEW as SAMTOOLS_CRAM } from '../../modules/nf-core/samtools/view/main' -include { SAMTOOLS_VIEW as SAMTOOLS_REINDEX } from '../../modules/nf-core/samtools/view/main' +include { SAMTOOLS_INDEX } from '../../modules/nf-core/samtools/index/main' include { SAMTOOLS_STATS } from '../../modules/nf-core/samtools/stats/main' include { SAMTOOLS_FLAGSTAT } from '../../modules/nf-core/samtools/flagstat/main' include { SAMTOOLS_IDXSTATS } from '../../modules/nf-core/samtools/idxstats/main' +include { BLOBTK_DEPTH } from '../../modules/local/blobtk_depth' +include { TABIX_BGZIP as BGZIP_BEDGRAPH } from '../../modules/nf-core/tabix/bgzip/main' workflow CONVERT_STATS { take: bam // channel: [ val(meta), /path/to/bam, /path/to/bai ] fasta // channel: [ val(meta), /path/to/fasta ] + ch_header main: ch_versions = Channel.empty() @@ -23,7 +31,12 @@ workflow CONVERT_STATS { // Split outfmt parameter into a list def outfmt_options = params.outfmt.split(',').collect { it.trim() } - + + // Reserve fasta base name for prefix of CHANGE_NAME and BLOBTK_DEPTH + bam + .combine( fasta ) + .map{ meta, bam, meta1, fasta -> [ meta +[fasta:fasta.baseName], bam]} + .set { bam } // (Optionally) Compress the quality scores of Illumina and PacBio CCS alignments if ( params.compression == "crumble" ) { @@ -62,38 +75,46 @@ workflow CONVERT_STATS { // Combine CRAM and CRAI into one channel ch_cram = SAMTOOLS_CRAM.out.cram ch_crai = SAMTOOLS_CRAM.out.crai - } - - - // Re-generate BAM index if BAM is in outfmt - def ch_data_for_stats - if ("cram" in outfmt_options) { ch_data_for_stats = ch_cram.join( ch_crai ) } else { ch_data_for_stats = ch_bams_for_conversion } + + // Re-generate BAM index if BAM is in outfmt ch_bam = Channel.empty() ch_bai = Channel.empty() if ("bam" in outfmt_options) { - // Re-generate BAM index - SAMTOOLS_REINDEX ( ch_bams_for_conversion, fasta, [] ) - ch_versions = ch_versions.mix( SAMTOOLS_REINDEX.out.versions.first() ) + ch_bams_to_index = ch_bams_for_conversion.map{ it -> [it[0], it[1]]} + // Change name of BAM files to final name for publishing + ch_bam = CHANGE_NAME ( ch_bams_to_index ).file + ch_versions = ch_versions.mix( CHANGE_NAME.out.versions.first() ) + + // Reindex BAM + SAMTOOLS_INDEX ( ch_bam ) + ch_versions = ch_versions.mix( SAMTOOLS_INDEX.out.versions.first() ) // Set the BAM and BAI channels for emission - ch_bam = SAMTOOLS_REINDEX.out.bam - ch_csi = SAMTOOLS_REINDEX.out.csi + ch_bai = SAMTOOLS_INDEX.out.bai.mix(SAMTOOLS_INDEX.out.csi) // If using BAM for stats, use the reindexed BAM if ( !("cram" in outfmt_options) ) { - ch_data_for_stats = ch_bam.join ( ch_csi ) + ch_data_for_stats = ch_bam.join ( SAMTOOLS_INDEX.out.bai ) } } + // Optionally insert params.header information to bams + if ( params.header ) { + ch_bam = SAMTOOLS_REHEADER_BAM ( ch_bam, ch_header.first() ).bam + ch_cram = SAMTOOLS_REHEADER_CRAM ( ch_cram, ch_header.first() ).bam + ch_versions = ch_versions.mix ( SAMTOOLS_REHEADER_BAM.out.versions ) + .mix ( SAMTOOLS_REHEADER_CRAM.out.versions ) + } + // Calculate read depth - BLOBTK_DEPTH ( ch_data_for_stats ) + BLOBTK_DEPTH ( ch_bams_for_conversion ) ch_versions = ch_versions.mix( BLOBTK_DEPTH.out.versions.first() ) BGZIP_BEDGRAPH ( BLOBTK_DEPTH.out.bedgraph ) diff --git a/workflows/readmapping.nf b/workflows/readmapping.nf index 1087b7c9..9ee8f060 100644 --- a/workflows/readmapping.nf +++ b/workflows/readmapping.nf @@ -5,13 +5,6 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -// -// MODULE: Local modules -// - -include { SAMTOOLS_REHEADER } from '../modules/local/samtools_replaceheader' - - // // SUBWORKFLOW: Consisting of a mix of local and nf-core/modules // @@ -127,18 +120,9 @@ workflow READMAPPING { | mix( ALIGN_CLR.out.bam ) | mix( ALIGN_ONT.out.bam ) - // Optionally insert params.header information to bams - ch_reheadered_bams = Channel.empty() - if ( params.header ) { - SAMTOOLS_REHEADER( ch_aligned_bams, ch_header.first() ) - ch_reheadered_bams = SAMTOOLS_REHEADER.out.bam - ch_versions = ch_versions.mix ( SAMTOOLS_REHEADER.out.versions ) - } else { - ch_reheadered_bams = ch_aligned_bams - } // convert to cram and gather stats - CONVERT_STATS ( ch_reheadered_bams, PREPARE_GENOME.out.fasta ) + CONVERT_STATS ( ch_aligned_bams, PREPARE_GENOME.out.fasta, ch_header ) ch_versions = ch_versions.mix ( CONVERT_STATS.out.versions ) // From 8bf07e5756f48a8d7d4b3a6ff8c06e7a0d559618 Mon Sep 17 00:00:00 2001 From: sainsachiko Date: Fri, 27 Jun 2025 14:22:34 +0100 Subject: [PATCH 06/28] Use ln -s to change file names --- modules/local/change_name.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/local/change_name.nf b/modules/local/change_name.nf index edca6926..2136e2da 100644 --- a/modules/local/change_name.nf +++ b/modules/local/change_name.nf @@ -21,7 +21,7 @@ process CHANGE_NAME { def prefix = task.ext.prefix ?: "${meta.id}" def new_fn = "${prefix}.${file.extension}" """ - mv ${file} ${new_fn} + ln -s ${file} ${new_fn} cat <<-END_VERSIONS > versions.yml "${task.process}": From 677c7277a9dbfd88ee650cdde102a80f72f8a9b9 Mon Sep 17 00:00:00 2001 From: sainsachiko Date: Mon, 30 Jun 2025 15:07:16 +0100 Subject: [PATCH 07/28] Fix linting --- conf/modules.config | 3 ++- subworkflows/local/convert_stats.nf | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index b2f51172..e22fcbf8 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -168,7 +168,7 @@ process { publishDir = [ path: { "${params.outdir}/read_mapping/${meta.datatype}${meta.merged ? '/merged' : ''}" }, mode: params.publish_dir_mode, - // Only publish BAM/CRAM from SAMTOOLS_REHEADER if params.header is true, else from: CHANGE_NAME and SAMTOOLS_CRAM + // Only publish BAM/CRAM from SAMTOOLS_REHEADER if params.header is true, else from: CHANGE_NAME and SAMTOOLS_CRAM saveAs: { filename -> filename == 'versions.yml' || ((task.process =~ /CHANGE_NAME|SAMTOOLS_CRAM/) && params.header) ? null : filename } @@ -185,6 +185,7 @@ process { withName: 'CHANGE_NAME' { ext.prefix = { "${meta.fasta}.${meta.datatype}.${meta.id}" } + } withName: FASTQC { ext.prefix = { "${meta.id}.${meta.datatype}" } diff --git a/subworkflows/local/convert_stats.nf b/subworkflows/local/convert_stats.nf index f6994f83..8e7267e7 100644 --- a/subworkflows/local/convert_stats.nf +++ b/subworkflows/local/convert_stats.nf @@ -31,7 +31,7 @@ workflow CONVERT_STATS { // Split outfmt parameter into a list def outfmt_options = params.outfmt.split(',').collect { it.trim() } - + // Reserve fasta base name for prefix of CHANGE_NAME and BLOBTK_DEPTH bam .combine( fasta ) @@ -109,9 +109,9 @@ workflow CONVERT_STATS { ch_bam = SAMTOOLS_REHEADER_BAM ( ch_bam, ch_header.first() ).bam ch_cram = SAMTOOLS_REHEADER_CRAM ( ch_cram, ch_header.first() ).bam ch_versions = ch_versions.mix ( SAMTOOLS_REHEADER_BAM.out.versions ) - .mix ( SAMTOOLS_REHEADER_CRAM.out.versions ) + .mix ( SAMTOOLS_REHEADER_CRAM.out.versions ) } - + // Calculate read depth BLOBTK_DEPTH ( ch_bams_for_conversion ) From 9e02bf051ba399bfcd1736c4e1a4878fddeee77c Mon Sep 17 00:00:00 2001 From: sainsachiko Date: Mon, 30 Jun 2025 15:14:44 +0100 Subject: [PATCH 08/28] Remove module config of SAMTOOLS_REINDEX --- conf/modules.config | 5 ----- 1 file changed, 5 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index e22fcbf8..0db62c81 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -139,11 +139,6 @@ process { ext.prefix = { "${bam.baseName}" } } - withName: SAMTOOLS_REINDEX { - ext.args = '--write-index' - ext.prefix = { "${fasta.baseName}.${meta.datatype}.${meta.id}" } - } - withName: SAMTOOLS_IDXSTATS { ext.prefix = { "${bam.baseName}" } } From dc045374b4835d975adebeb157ddfe5065434acc Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Wed, 2 Jul 2025 13:27:43 +0100 Subject: [PATCH 09/28] Not used --- modules.json | 5 - .../nf-core/samtools/faidx/environment.yml | 8 - modules/nf-core/samtools/faidx/main.nf | 50 ---- modules/nf-core/samtools/faidx/meta.yml | 80 ------ .../nf-core/samtools/faidx/tests/main.nf.test | 122 --------- .../samtools/faidx/tests/main.nf.test.snap | 249 ------------------ .../samtools/faidx/tests/nextflow.config | 7 - .../samtools/faidx/tests/nextflow2.config | 6 - modules/nf-core/samtools/faidx/tests/tags.yml | 2 - 9 files changed, 529 deletions(-) delete mode 100644 modules/nf-core/samtools/faidx/environment.yml delete mode 100644 modules/nf-core/samtools/faidx/main.nf delete mode 100644 modules/nf-core/samtools/faidx/meta.yml delete mode 100644 modules/nf-core/samtools/faidx/tests/main.nf.test delete mode 100644 modules/nf-core/samtools/faidx/tests/main.nf.test.snap delete mode 100644 modules/nf-core/samtools/faidx/tests/nextflow.config delete mode 100644 modules/nf-core/samtools/faidx/tests/nextflow2.config delete mode 100644 modules/nf-core/samtools/faidx/tests/tags.yml diff --git a/modules.json b/modules.json index 6830ce56..32d0ca5c 100644 --- a/modules.json +++ b/modules.json @@ -45,11 +45,6 @@ "git_sha": "72e277acfd9e61a9f1368eafb4a9e83f5bcaa9f5", "installed_by": ["modules"] }, - "samtools/faidx": { - "branch": "master", - "git_sha": "b13f07be4c508d6ff6312d354d09f2493243e208", - "installed_by": ["modules"] - }, "samtools/flagstat": { "branch": "master", "git_sha": "2d20463181b1c38981a02e90d3084b5f9fa8d540", diff --git a/modules/nf-core/samtools/faidx/environment.yml b/modules/nf-core/samtools/faidx/environment.yml deleted file mode 100644 index 62054fc9..00000000 --- a/modules/nf-core/samtools/faidx/environment.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/environment-schema.json -channels: - - conda-forge - - bioconda -dependencies: - - bioconda::htslib=1.21 - - bioconda::samtools=1.21 diff --git a/modules/nf-core/samtools/faidx/main.nf b/modules/nf-core/samtools/faidx/main.nf deleted file mode 100644 index 28c0a81c..00000000 --- a/modules/nf-core/samtools/faidx/main.nf +++ /dev/null @@ -1,50 +0,0 @@ -process SAMTOOLS_FAIDX { - tag "$fasta" - label 'process_single' - - conda "${moduleDir}/environment.yml" - container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/samtools:1.21--h50ea8bc_0' : - 'biocontainers/samtools:1.21--h50ea8bc_0' }" - - input: - tuple val(meta), path(fasta) - tuple val(meta2), path(fai) - - output: - tuple val(meta), path ("*.{fa,fasta}") , emit: fa , optional: true - tuple val(meta), path ("*.fai") , emit: fai, optional: true - tuple val(meta), path ("*.gzi") , emit: gzi, optional: true - path "versions.yml" , emit: versions - - when: - task.ext.when == null || task.ext.when - - script: - def args = task.ext.args ?: '' - """ - samtools \\ - faidx \\ - $fasta \\ - $args - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - samtools: \$(echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//') - END_VERSIONS - """ - - stub: - def match = (task.ext.args =~ /-o(?:utput)?\s(.*)\s?/).findAll() - def fastacmd = match[0] ? "touch ${match[0][1]}" : '' - """ - ${fastacmd} - touch ${fasta}.fai - - cat <<-END_VERSIONS > versions.yml - - "${task.process}": - samtools: \$(echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//') - END_VERSIONS - """ -} diff --git a/modules/nf-core/samtools/faidx/meta.yml b/modules/nf-core/samtools/faidx/meta.yml deleted file mode 100644 index 6721b2cb..00000000 --- a/modules/nf-core/samtools/faidx/meta.yml +++ /dev/null @@ -1,80 +0,0 @@ -name: samtools_faidx -description: Index FASTA file -keywords: - - index - - fasta - - faidx -tools: - - samtools: - description: | - SAMtools is a set of utilities for interacting with and post-processing - short DNA sequence read alignments in the SAM, BAM and CRAM formats, written by Heng Li. - These files are generated as output by short read aligners like BWA. - homepage: http://www.htslib.org/ - documentation: http://www.htslib.org/doc/samtools.html - doi: 10.1093/bioinformatics/btp352 - licence: ["MIT"] - identifier: biotools:samtools -input: - - - meta: - type: map - description: | - Groovy Map containing reference information - e.g. [ id:'test' ] - - fasta: - type: file - description: FASTA file - pattern: "*.{fa,fasta}" - - - meta2: - type: map - description: | - Groovy Map containing reference information - e.g. [ id:'test' ] - - fai: - type: file - description: FASTA index file - pattern: "*.{fai}" -output: - - fa: - - meta: - type: map - description: | - Groovy Map containing sample information - e.g. [ id:'test', single_end:false ] - - "*.{fa,fasta}": - type: file - description: FASTA file - pattern: "*.{fa}" - - fai: - - meta: - type: map - description: | - Groovy Map containing sample information - e.g. [ id:'test', single_end:false ] - - "*.fai": - type: file - description: FASTA index file - pattern: "*.{fai}" - - gzi: - - meta: - type: map - description: | - Groovy Map containing sample information - e.g. [ id:'test', single_end:false ] - - "*.gzi": - type: file - description: Optional gzip index file for compressed inputs - pattern: "*.gzi" - - versions: - - versions.yml: - type: file - description: File containing software versions - pattern: "versions.yml" -authors: - - "@drpatelh" - - "@ewels" - - "@phue" -maintainers: - - "@drpatelh" - - "@ewels" - - "@phue" diff --git a/modules/nf-core/samtools/faidx/tests/main.nf.test b/modules/nf-core/samtools/faidx/tests/main.nf.test deleted file mode 100644 index 17244ef2..00000000 --- a/modules/nf-core/samtools/faidx/tests/main.nf.test +++ /dev/null @@ -1,122 +0,0 @@ -nextflow_process { - - name "Test Process SAMTOOLS_FAIDX" - script "../main.nf" - process "SAMTOOLS_FAIDX" - - tag "modules" - tag "modules_nfcore" - tag "samtools" - tag "samtools/faidx" - - test("test_samtools_faidx") { - - when { - process { - """ - input[0] = [ [ id:'test', single_end:false ], // meta map - file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) ] - - input[1] = [[],[]] - """ - } - } - - then { - assertAll( - { assert process.success }, - { assert snapshot(process.out).match() } - ) - } - } - - test("test_samtools_faidx_bgzip") { - - when { - process { - """ - input[0] = [ [ id:'test', single_end:false ], // meta map - file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta.gz', checkIfExists: true)] - - input[1] = [[],[]] - """ - } - } - - then { - assertAll( - { assert process.success }, - { assert snapshot(process.out).match() } - ) - } - } - - test("test_samtools_faidx_fasta") { - - config "./nextflow.config" - - when { - process { - """ - input[0] = [ [ id:'test', single_end:false ], // meta map - file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) ] - - input[1] = [ [ id:'test', single_end:false ], // meta map - file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta.fai', checkIfExists: true) ] - """ - } - } - - then { - assertAll( - { assert process.success }, - { assert snapshot(process.out).match() } - ) - } - } - - test("test_samtools_faidx_stub_fasta") { - - config "./nextflow2.config" - - when { - process { - """ - input[0] = [ [ id:'test', single_end:false ], // meta map - file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) ] - - input[1] = [ [ id:'test', single_end:false ], // meta map - file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta.fai', checkIfExists: true) ] - """ - } - } - - then { - assertAll( - { assert process.success }, - { assert snapshot(process.out).match() } - ) - } - } - - test("test_samtools_faidx_stub_fai") { - - when { - process { - """ - input[0] = [ [ id:'test', single_end:false ], // meta map - file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) ] - - input[1] = [[],[]] - """ - } - } - - then { - assertAll( - { assert process.success }, - { assert snapshot(process.out).match() } - ) - } - } -} \ No newline at end of file diff --git a/modules/nf-core/samtools/faidx/tests/main.nf.test.snap b/modules/nf-core/samtools/faidx/tests/main.nf.test.snap deleted file mode 100644 index 1bbb3ec2..00000000 --- a/modules/nf-core/samtools/faidx/tests/main.nf.test.snap +++ /dev/null @@ -1,249 +0,0 @@ -{ - "test_samtools_faidx": { - "content": [ - { - "0": [ - - ], - "1": [ - [ - { - "id": "test", - "single_end": false - }, - "genome.fasta.fai:md5,9da2a56e2853dc8c0b86a9e7229c9fe5" - ] - ], - "2": [ - - ], - "3": [ - "versions.yml:md5,6bbe80a2e14bd61202ca63e12d66027f" - ], - "fa": [ - - ], - "fai": [ - [ - { - "id": "test", - "single_end": false - }, - "genome.fasta.fai:md5,9da2a56e2853dc8c0b86a9e7229c9fe5" - ] - ], - "gzi": [ - - ], - "versions": [ - "versions.yml:md5,6bbe80a2e14bd61202ca63e12d66027f" - ] - } - ], - "meta": { - "nf-test": "0.9.0", - "nextflow": "24.04.4" - }, - "timestamp": "2024-09-16T07:57:47.450887871" - }, - "test_samtools_faidx_bgzip": { - "content": [ - { - "0": [ - - ], - "1": [ - [ - { - "id": "test", - "single_end": false - }, - "genome.fasta.gz.fai:md5,9da2a56e2853dc8c0b86a9e7229c9fe5" - ] - ], - "2": [ - [ - { - "id": "test", - "single_end": false - }, - "genome.fasta.gz.gzi:md5,7dea362b3fac8e00956a4952a3d4f474" - ] - ], - "3": [ - "versions.yml:md5,6bbe80a2e14bd61202ca63e12d66027f" - ], - "fa": [ - - ], - "fai": [ - [ - { - "id": "test", - "single_end": false - }, - "genome.fasta.gz.fai:md5,9da2a56e2853dc8c0b86a9e7229c9fe5" - ] - ], - "gzi": [ - [ - { - "id": "test", - "single_end": false - }, - "genome.fasta.gz.gzi:md5,7dea362b3fac8e00956a4952a3d4f474" - ] - ], - "versions": [ - "versions.yml:md5,6bbe80a2e14bd61202ca63e12d66027f" - ] - } - ], - "meta": { - "nf-test": "0.9.0", - "nextflow": "24.04.4" - }, - "timestamp": "2024-09-16T07:58:04.804905659" - }, - "test_samtools_faidx_fasta": { - "content": [ - { - "0": [ - [ - { - "id": "test", - "single_end": false - }, - "extract.fa:md5,6a0774a0ad937ba0bfd2ac7457d90f36" - ] - ], - "1": [ - - ], - "2": [ - - ], - "3": [ - "versions.yml:md5,6bbe80a2e14bd61202ca63e12d66027f" - ], - "fa": [ - [ - { - "id": "test", - "single_end": false - }, - "extract.fa:md5,6a0774a0ad937ba0bfd2ac7457d90f36" - ] - ], - "fai": [ - - ], - "gzi": [ - - ], - "versions": [ - "versions.yml:md5,6bbe80a2e14bd61202ca63e12d66027f" - ] - } - ], - "meta": { - "nf-test": "0.9.0", - "nextflow": "24.04.4" - }, - "timestamp": "2024-09-16T07:58:23.831268154" - }, - "test_samtools_faidx_stub_fasta": { - "content": [ - { - "0": [ - [ - { - "id": "test", - "single_end": false - }, - "extract.fa:md5,9da2a56e2853dc8c0b86a9e7229c9fe5" - ] - ], - "1": [ - - ], - "2": [ - - ], - "3": [ - "versions.yml:md5,6bbe80a2e14bd61202ca63e12d66027f" - ], - "fa": [ - [ - { - "id": "test", - "single_end": false - }, - "extract.fa:md5,9da2a56e2853dc8c0b86a9e7229c9fe5" - ] - ], - "fai": [ - - ], - "gzi": [ - - ], - "versions": [ - "versions.yml:md5,6bbe80a2e14bd61202ca63e12d66027f" - ] - } - ], - "meta": { - "nf-test": "0.9.0", - "nextflow": "24.04.4" - }, - "timestamp": "2024-09-16T07:58:35.600243706" - }, - "test_samtools_faidx_stub_fai": { - "content": [ - { - "0": [ - - ], - "1": [ - [ - { - "id": "test", - "single_end": false - }, - "genome.fasta.fai:md5,9da2a56e2853dc8c0b86a9e7229c9fe5" - ] - ], - "2": [ - - ], - "3": [ - "versions.yml:md5,6bbe80a2e14bd61202ca63e12d66027f" - ], - "fa": [ - - ], - "fai": [ - [ - { - "id": "test", - "single_end": false - }, - "genome.fasta.fai:md5,9da2a56e2853dc8c0b86a9e7229c9fe5" - ] - ], - "gzi": [ - - ], - "versions": [ - "versions.yml:md5,6bbe80a2e14bd61202ca63e12d66027f" - ] - } - ], - "meta": { - "nf-test": "0.9.0", - "nextflow": "24.04.4" - }, - "timestamp": "2024-09-16T07:58:54.705460167" - } -} \ No newline at end of file diff --git a/modules/nf-core/samtools/faidx/tests/nextflow.config b/modules/nf-core/samtools/faidx/tests/nextflow.config deleted file mode 100644 index f76a3ba0..00000000 --- a/modules/nf-core/samtools/faidx/tests/nextflow.config +++ /dev/null @@ -1,7 +0,0 @@ -process { - - withName: SAMTOOLS_FAIDX { - ext.args = 'MT192765.1 -o extract.fa' - } - -} diff --git a/modules/nf-core/samtools/faidx/tests/nextflow2.config b/modules/nf-core/samtools/faidx/tests/nextflow2.config deleted file mode 100644 index 33ebbd5d..00000000 --- a/modules/nf-core/samtools/faidx/tests/nextflow2.config +++ /dev/null @@ -1,6 +0,0 @@ -process { - - withName: SAMTOOLS_FAIDX { - ext.args = '-o extract.fa' - } -} diff --git a/modules/nf-core/samtools/faidx/tests/tags.yml b/modules/nf-core/samtools/faidx/tests/tags.yml deleted file mode 100644 index e4a83948..00000000 --- a/modules/nf-core/samtools/faidx/tests/tags.yml +++ /dev/null @@ -1,2 +0,0 @@ -samtools/faidx: - - modules/nf-core/samtools/faidx/** From d9b4bd59b1aaaa26fbc76c4e28c3c7462bb4ba59 Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Wed, 2 Jul 2025 13:27:59 +0100 Subject: [PATCH 10/28] We pass a BAM file, not a CRAM file --- modules/local/blobtk_depth.nf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/local/blobtk_depth.nf b/modules/local/blobtk_depth.nf index 1d96340f..721ae1d7 100644 --- a/modules/local/blobtk_depth.nf +++ b/modules/local/blobtk_depth.nf @@ -8,7 +8,7 @@ process BLOBTK_DEPTH { container "docker.io/genomehubs/blobtk:0.6.5" input: - tuple val(meta), path(cram), path(bai) + tuple val(meta), path(bam), path(bai) output: tuple val(meta), path('*.coverage.bedGraph') , emit: bedgraph @@ -22,7 +22,7 @@ process BLOBTK_DEPTH { def prefix = task.ext.prefix ?: "${meta.id}" """ blobtk depth \\ - -c ${cram} \\ + -b ${bam} \\ $args \\ -O ${prefix}.coverage.bedGraph \\ From 316262b6ad77db2b617f1a6663c879255ceda6e6 Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Wed, 2 Jul 2025 13:28:28 +0100 Subject: [PATCH 11/28] Too many names like "ch_bams" is confusing ! --- subworkflows/local/convert_stats.nf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/subworkflows/local/convert_stats.nf b/subworkflows/local/convert_stats.nf index 8e7267e7..d4123d5a 100644 --- a/subworkflows/local/convert_stats.nf +++ b/subworkflows/local/convert_stats.nf @@ -46,14 +46,14 @@ workflow CONVERT_STATS { run_crumble: meta.datatype == "hic" || meta.datatype == "illumina" || meta.datatype == "pacbio" no_crumble: true } - | set { ch_bams } + | set { crumble_selector } - CRUMBLE ( ch_bams.run_crumble, [], [] ) + CRUMBLE ( crumble_selector.run_crumble, [], [] ) ch_versions = ch_versions.mix( CRUMBLE.out.versions ) // Convert BAM to CRAM CRUMBLE.out.bam - | mix( ch_bams.no_crumble ) + | mix( crumble_selector.no_crumble ) | map { meta, bam -> [meta, bam, []] } | set { ch_bams_for_conversion } From 684384d4e1499dbc6225b075df5cadb4fd51befa Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Wed, 2 Jul 2025 15:03:30 +0100 Subject: [PATCH 12/28] Simplified the CHANGE_NAME module - `process_*` don't exist in this pipeline. The defaults resources are already for single CPU use - No need to invoke a container, the host must have `ln` - Therefore no need to report versions --- modules/local/change_name.nf | 19 +------------------ subworkflows/local/convert_stats.nf | 1 - 2 files changed, 1 insertion(+), 19 deletions(-) diff --git a/modules/local/change_name.nf b/modules/local/change_name.nf index 2136e2da..d5e0d848 100644 --- a/modules/local/change_name.nf +++ b/modules/local/change_name.nf @@ -1,43 +1,26 @@ process CHANGE_NAME { tag "$meta.id" - label 'process_single' - - container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/ubuntu:20.04' : - 'docker.io/ubuntu:20.04' }" + executor 'local' input: tuple val(meta), path(file) output: tuple val(meta), path("*.${file.extension}"), emit: file - path "versions.yml", emit: versions when: task.ext.when == null || task.ext.when script: - def VERSION = "9.1" // WARN: Version information not provided by tool on CLI. Please update this string when bumping container versions. def prefix = task.ext.prefix ?: "${meta.id}" def new_fn = "${prefix}.${file.extension}" """ ln -s ${file} ${new_fn} - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - coreutils: $VERSION - END_VERSIONS """ stub: - def VERSION = "9.1" // WARN: Version information not provided by tool on CLI. Please update this string when bumping container versions. """ touch ${new_fn} - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - coreutils: $VERSION - END_VERSIONS """ } diff --git a/subworkflows/local/convert_stats.nf b/subworkflows/local/convert_stats.nf index d4123d5a..f50592c3 100644 --- a/subworkflows/local/convert_stats.nf +++ b/subworkflows/local/convert_stats.nf @@ -89,7 +89,6 @@ workflow CONVERT_STATS { ch_bams_to_index = ch_bams_for_conversion.map{ it -> [it[0], it[1]]} // Change name of BAM files to final name for publishing ch_bam = CHANGE_NAME ( ch_bams_to_index ).file - ch_versions = ch_versions.mix( CHANGE_NAME.out.versions.first() ) // Reindex BAM SAMTOOLS_INDEX ( ch_bam ) From 4199fd78a117cdeaaf5a7f7d2728a0d18b7a7c3b Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Wed, 2 Jul 2025 15:05:30 +0100 Subject: [PATCH 13/28] It's the CRAM output that we want --- subworkflows/local/convert_stats.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subworkflows/local/convert_stats.nf b/subworkflows/local/convert_stats.nf index f50592c3..e2b50671 100644 --- a/subworkflows/local/convert_stats.nf +++ b/subworkflows/local/convert_stats.nf @@ -106,7 +106,7 @@ workflow CONVERT_STATS { // Optionally insert params.header information to bams if ( params.header ) { ch_bam = SAMTOOLS_REHEADER_BAM ( ch_bam, ch_header.first() ).bam - ch_cram = SAMTOOLS_REHEADER_CRAM ( ch_cram, ch_header.first() ).bam + ch_cram = SAMTOOLS_REHEADER_CRAM ( ch_cram, ch_header.first() ).cram ch_versions = ch_versions.mix ( SAMTOOLS_REHEADER_BAM.out.versions ) .mix ( SAMTOOLS_REHEADER_CRAM.out.versions ) } From d12030d3dd2e2c4b80fcc42e820962a85782b4ff Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Wed, 2 Jul 2025 15:05:48 +0100 Subject: [PATCH 14/28] No conversion is happening here --- subworkflows/local/convert_stats.nf | 1 - 1 file changed, 1 deletion(-) diff --git a/subworkflows/local/convert_stats.nf b/subworkflows/local/convert_stats.nf index e2b50671..ca9887dd 100644 --- a/subworkflows/local/convert_stats.nf +++ b/subworkflows/local/convert_stats.nf @@ -51,7 +51,6 @@ workflow CONVERT_STATS { CRUMBLE ( crumble_selector.run_crumble, [], [] ) ch_versions = ch_versions.mix( CRUMBLE.out.versions ) - // Convert BAM to CRAM CRUMBLE.out.bam | mix( crumble_selector.no_crumble ) | map { meta, bam -> [meta, bam, []] } From a10f97a78198e29ea7ac40afaf389a2e320b27a7 Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Wed, 2 Jul 2025 15:15:37 +0100 Subject: [PATCH 15/28] Allow rerunning --- modules/local/change_name.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/local/change_name.nf b/modules/local/change_name.nf index d5e0d848..a6aba96b 100644 --- a/modules/local/change_name.nf +++ b/modules/local/change_name.nf @@ -15,7 +15,7 @@ process CHANGE_NAME { def prefix = task.ext.prefix ?: "${meta.id}" def new_fn = "${prefix}.${file.extension}" """ - ln -s ${file} ${new_fn} + ln -sf ${file} ${new_fn} """ stub: From 03f6d98229e2ec84652d7b4189a38e866857e561 Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Wed, 2 Jul 2025 15:43:14 +0100 Subject: [PATCH 16/28] Our convention is not to prefix workflow inputs with "ch_" --- subworkflows/local/convert_stats.nf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/subworkflows/local/convert_stats.nf b/subworkflows/local/convert_stats.nf index ca9887dd..cc32345d 100644 --- a/subworkflows/local/convert_stats.nf +++ b/subworkflows/local/convert_stats.nf @@ -23,7 +23,7 @@ workflow CONVERT_STATS { take: bam // channel: [ val(meta), /path/to/bam, /path/to/bai ] fasta // channel: [ val(meta), /path/to/fasta ] - ch_header + header // channel: /path/to/header.sam main: ch_versions = Channel.empty() @@ -104,8 +104,8 @@ workflow CONVERT_STATS { // Optionally insert params.header information to bams if ( params.header ) { - ch_bam = SAMTOOLS_REHEADER_BAM ( ch_bam, ch_header.first() ).bam - ch_cram = SAMTOOLS_REHEADER_CRAM ( ch_cram, ch_header.first() ).cram + ch_bam = SAMTOOLS_REHEADER_BAM ( ch_bam, header.first() ).bam + ch_cram = SAMTOOLS_REHEADER_CRAM ( ch_cram, header.first() ).cram ch_versions = ch_versions.mix ( SAMTOOLS_REHEADER_BAM.out.versions ) .mix ( SAMTOOLS_REHEADER_CRAM.out.versions ) } From 89a8bd846776779197435dbeeb5f0c9cdd9db7f5 Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Wed, 2 Jul 2025 15:47:02 +0100 Subject: [PATCH 17/28] More straightforward sub-workflow - CHANGE_NAME is always called and is the only place where the published file name is set - All downstream modules use the renamed BAMs as input and copy its name to the output --- conf/modules.config | 6 ++-- modules/local/change_name.nf | 1 + subworkflows/local/convert_stats.nf | 47 +++++++++++------------------ 3 files changed, 21 insertions(+), 33 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index 0db62c81..3cfe46d6 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -131,7 +131,7 @@ process { withName: '.*:CONVERT_STATS:SAMTOOLS_CRAM' { beforeScript = { "export REF_PATH=spoof"} - ext.prefix = { "${fasta.baseName}.${meta.datatype}.${meta.id}" } + ext.prefix = { "${input.baseName}" } ext.args = '--output-fmt cram --write-index' } @@ -171,7 +171,7 @@ process { } withName: 'BLOBTK_DEPTH' { - ext.prefix = { "${meta.fasta}.${meta.datatype}.${meta.id}" } + ext.prefix = { "${bam.baseName}" } } withName: 'BGZIP_BEDGRAPH' { @@ -179,7 +179,7 @@ process { } withName: 'CHANGE_NAME' { - ext.prefix = { "${meta.fasta}.${meta.datatype}.${meta.id}" } + ext.prefix = { "${fasta.baseName}.${meta.datatype}.${meta.id}" } } withName: FASTQC { diff --git a/modules/local/change_name.nf b/modules/local/change_name.nf index a6aba96b..f2b6b1ca 100644 --- a/modules/local/change_name.nf +++ b/modules/local/change_name.nf @@ -4,6 +4,7 @@ process CHANGE_NAME { input: tuple val(meta), path(file) + tuple val(meta2), path(fasta) output: tuple val(meta), path("*.${file.extension}"), emit: file diff --git a/subworkflows/local/convert_stats.nf b/subworkflows/local/convert_stats.nf index cc32345d..043c41cd 100644 --- a/subworkflows/local/convert_stats.nf +++ b/subworkflows/local/convert_stats.nf @@ -32,12 +32,6 @@ workflow CONVERT_STATS { // Split outfmt parameter into a list def outfmt_options = params.outfmt.split(',').collect { it.trim() } - // Reserve fasta base name for prefix of CHANGE_NAME and BLOBTK_DEPTH - bam - .combine( fasta ) - .map{ meta, bam, meta1, fasta -> [ meta +[fasta:fasta.baseName], bam]} - .set { bam } - // (Optionally) Compress the quality scores of Illumina and PacBio CCS alignments if ( params.compression == "crumble" ) { bam @@ -53,30 +47,32 @@ workflow CONVERT_STATS { CRUMBLE.out.bam | mix( crumble_selector.no_crumble ) - | map { meta, bam -> [meta, bam, []] } | set { ch_bams_for_conversion } } else { - bam - | map { meta, bam -> [meta, bam, []] } - | set { ch_bams_for_conversion } + ch_bams_for_conversion = bam } + // Change name of BAM files to final name for publishing + CHANGE_NAME ( ch_bams_for_conversion, fasta ) + + CHANGE_NAME.out.file + | map { meta, bam -> [meta, bam, []] } + | set { ch_renamed_bams } + + // (Optionally) convert to CRAM if it's specified in outfmt ch_cram = Channel.empty() ch_crai = Channel.empty() if ("cram" in outfmt_options) { - SAMTOOLS_CRAM ( ch_bams_for_conversion, fasta, [] ) + SAMTOOLS_CRAM ( ch_renamed_bams, fasta, [] ) ch_versions = ch_versions.mix( SAMTOOLS_CRAM.out.versions.first() ) // Combine CRAM and CRAI into one channel ch_cram = SAMTOOLS_CRAM.out.cram ch_crai = SAMTOOLS_CRAM.out.crai - ch_data_for_stats = ch_cram.join( ch_crai ) - } else { - ch_data_for_stats = ch_bams_for_conversion } @@ -85,23 +81,16 @@ workflow CONVERT_STATS { ch_bai = Channel.empty() if ("bam" in outfmt_options) { - ch_bams_to_index = ch_bams_for_conversion.map{ it -> [it[0], it[1]]} - // Change name of BAM files to final name for publishing - ch_bam = CHANGE_NAME ( ch_bams_to_index ).file - // Reindex BAM - SAMTOOLS_INDEX ( ch_bam ) + SAMTOOLS_INDEX ( CHANGE_NAME.out.file ) ch_versions = ch_versions.mix( SAMTOOLS_INDEX.out.versions.first() ) // Set the BAM and BAI channels for emission + ch_bam = CHANGE_NAME.out.file ch_bai = SAMTOOLS_INDEX.out.bai.mix(SAMTOOLS_INDEX.out.csi) - - // If using BAM for stats, use the reindexed BAM - if ( !("cram" in outfmt_options) ) { - ch_data_for_stats = ch_bam.join ( SAMTOOLS_INDEX.out.bai ) - } } + // Optionally insert params.header information to bams if ( params.header ) { ch_bam = SAMTOOLS_REHEADER_BAM ( ch_bam, header.first() ).bam @@ -112,7 +101,7 @@ workflow CONVERT_STATS { // Calculate read depth - BLOBTK_DEPTH ( ch_bams_for_conversion ) + BLOBTK_DEPTH ( ch_renamed_bams ) ch_versions = ch_versions.mix( BLOBTK_DEPTH.out.versions.first() ) BGZIP_BEDGRAPH ( BLOBTK_DEPTH.out.bedgraph ) @@ -120,17 +109,15 @@ workflow CONVERT_STATS { // Calculate statistics - SAMTOOLS_STATS ( ch_data_for_stats, fasta ) + SAMTOOLS_STATS ( ch_renamed_bams, [[], []] ) ch_versions = ch_versions.mix( SAMTOOLS_STATS.out.versions.first() ) - // Calculate statistics based on flag values - SAMTOOLS_FLAGSTAT ( ch_data_for_stats ) + SAMTOOLS_FLAGSTAT ( ch_renamed_bams ) ch_versions = ch_versions.mix( SAMTOOLS_FLAGSTAT.out.versions.first() ) - // Calculate index statistics - SAMTOOLS_IDXSTATS ( ch_data_for_stats ) + SAMTOOLS_IDXSTATS ( ch_renamed_bams ) ch_versions = ch_versions.mix( SAMTOOLS_IDXSTATS.out.versions.first() ) From ac819b45cdf464362a0135302bbe613811d6ed01 Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Wed, 2 Jul 2025 17:22:19 +0100 Subject: [PATCH 18/28] Trick for not publishing the BAM when CRAM output is selected We don't publish anything from CHANGE_NAME itself. Instead we make SAMTOOLS_INDEX publish its input file. --- conf/modules.config | 9 ++++++--- modules.json | 3 ++- modules/nf-core/samtools/index/main.nf | 1 + .../samtools/index/samtools-index.diff | 20 +++++++++++++++++++ 4 files changed, 29 insertions(+), 4 deletions(-) create mode 100644 modules/nf-core/samtools/index/samtools-index.diff diff --git a/conf/modules.config b/conf/modules.config index 3cfe46d6..201f38f8 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -158,14 +158,17 @@ process { ext.args = { (meta.datatype == "pacbio" ? "-y pbccs " : "") + "-O bam" } } - withName: '.*:CONVERT_STATS:SAMTOOLS_.*|BGZIP_BEDGRAPH|CHANGE_NAME' { + withName: '.*:CONVERT_STATS:SAMTOOLS_.*|BGZIP_BEDGRAPH' { beforeScript = { "export REF_PATH=spoof" } publishDir = [ path: { "${params.outdir}/read_mapping/${meta.datatype}${meta.merged ? '/merged' : ''}" }, mode: params.publish_dir_mode, - // Only publish BAM/CRAM from SAMTOOLS_REHEADER if params.header is true, else from: CHANGE_NAME and SAMTOOLS_CRAM + // Only publish BAM/CRAM from SAMTOOLS_REHEADER if params.header is true, else from: SAMTOOLS_INDEX (BAM) and SAMTOOLS_CRAM (CRAM) saveAs: { filename -> - filename == 'versions.yml' || ((task.process =~ /CHANGE_NAME|SAMTOOLS_CRAM/) && params.header) ? null : filename + filename == 'versions.yml' + || ((task.process =~ /SAMTOOLS_INDEX|SAMTOOLS_CRAM/) && params.header) + || ((task.process =~ /SAMTOOLS_CRAM/) && filename.endsWith(".bam")) + ? null : filename } ] } diff --git a/modules.json b/modules.json index 32d0ca5c..ec2d606a 100644 --- a/modules.json +++ b/modules.json @@ -58,7 +58,8 @@ "samtools/index": { "branch": "master", "git_sha": "46eca555142d6e597729fcb682adcc791796f514", - "installed_by": ["modules"] + "installed_by": ["modules"], + "patch": "modules/nf-core/samtools/index/samtools-index.diff" }, "samtools/merge": { "branch": "master", diff --git a/modules/nf-core/samtools/index/main.nf b/modules/nf-core/samtools/index/main.nf index e002585b..2f4c52f3 100644 --- a/modules/nf-core/samtools/index/main.nf +++ b/modules/nf-core/samtools/index/main.nf @@ -11,6 +11,7 @@ process SAMTOOLS_INDEX { tuple val(meta), path(input) output: + tuple val(meta), path("${input}"), emit: input tuple val(meta), path("*.bai") , optional:true, emit: bai tuple val(meta), path("*.csi") , optional:true, emit: csi tuple val(meta), path("*.crai"), optional:true, emit: crai diff --git a/modules/nf-core/samtools/index/samtools-index.diff b/modules/nf-core/samtools/index/samtools-index.diff new file mode 100644 index 00000000..c086fede --- /dev/null +++ b/modules/nf-core/samtools/index/samtools-index.diff @@ -0,0 +1,20 @@ +Changes in component 'nf-core/samtools/index' +'modules/nf-core/samtools/index/meta.yml' is unchanged +Changes in 'samtools/index/main.nf': +--- modules/nf-core/samtools/index/main.nf ++++ modules/nf-core/samtools/index/main.nf +@@ -11,6 +11,7 @@ + tuple val(meta), path(input) + + output: ++ tuple val(meta), path("${input}"), emit: input + tuple val(meta), path("*.bai") , optional:true, emit: bai + tuple val(meta), path("*.csi") , optional:true, emit: csi + tuple val(meta), path("*.crai"), optional:true, emit: crai + +'modules/nf-core/samtools/index/environment.yml' is unchanged +'modules/nf-core/samtools/index/tests/csi.nextflow.config' is unchanged +'modules/nf-core/samtools/index/tests/main.nf.test' is unchanged +'modules/nf-core/samtools/index/tests/tags.yml' is unchanged +'modules/nf-core/samtools/index/tests/main.nf.test.snap' is unchanged +************************************************************ From 50346658dadb1199277ad3a6b839c62cbbcbe717 Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Thu, 3 Jul 2025 09:38:12 +0100 Subject: [PATCH 19/28] More explicit name --- subworkflows/local/convert_stats.nf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/subworkflows/local/convert_stats.nf b/subworkflows/local/convert_stats.nf index 043c41cd..6fa90353 100644 --- a/subworkflows/local/convert_stats.nf +++ b/subworkflows/local/convert_stats.nf @@ -47,15 +47,15 @@ workflow CONVERT_STATS { CRUMBLE.out.bam | mix( crumble_selector.no_crumble ) - | set { ch_bams_for_conversion } + | set { ch_bams_for_renaming } } else { - ch_bams_for_conversion = bam + ch_bams_for_renaming = bam } // Change name of BAM files to final name for publishing - CHANGE_NAME ( ch_bams_for_conversion, fasta ) + CHANGE_NAME ( ch_bams_for_renaming, fasta ) CHANGE_NAME.out.file | map { meta, bam -> [meta, bam, []] } From c48ba522806ad35b7f4ab9eccc3c53d0ba173ee7 Mon Sep 17 00:00:00 2001 From: sainsachiko Date: Tue, 8 Jul 2025 10:28:35 +0100 Subject: [PATCH 20/28] Input both file and index for stats process --- subworkflows/local/convert_stats.nf | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/subworkflows/local/convert_stats.nf b/subworkflows/local/convert_stats.nf index 6fa90353..b78d754f 100644 --- a/subworkflows/local/convert_stats.nf +++ b/subworkflows/local/convert_stats.nf @@ -73,6 +73,7 @@ workflow CONVERT_STATS { // Combine CRAM and CRAI into one channel ch_cram = SAMTOOLS_CRAM.out.cram ch_crai = SAMTOOLS_CRAM.out.crai + ch_for_stats = ch_cram.join( ch_crai ) } @@ -88,6 +89,11 @@ workflow CONVERT_STATS { // Set the BAM and BAI channels for emission ch_bam = CHANGE_NAME.out.file ch_bai = SAMTOOLS_INDEX.out.bai.mix(SAMTOOLS_INDEX.out.csi) + + if ( !('cram' in outfmt_options) ) { + ch_for_stats = ch_bam.join( ch_bai ) + } + } @@ -107,17 +113,16 @@ workflow CONVERT_STATS { BGZIP_BEDGRAPH ( BLOBTK_DEPTH.out.bedgraph ) ch_versions = ch_versions.mix( BGZIP_BEDGRAPH.out.versions.first() ) - // Calculate statistics - SAMTOOLS_STATS ( ch_renamed_bams, [[], []] ) + SAMTOOLS_STATS (ch_for_stats, [[], []] ) ch_versions = ch_versions.mix( SAMTOOLS_STATS.out.versions.first() ) // Calculate statistics based on flag values - SAMTOOLS_FLAGSTAT ( ch_renamed_bams ) + SAMTOOLS_FLAGSTAT ( ch_for_stats ) ch_versions = ch_versions.mix( SAMTOOLS_FLAGSTAT.out.versions.first() ) // Calculate index statistics - SAMTOOLS_IDXSTATS ( ch_renamed_bams ) + SAMTOOLS_IDXSTATS ( ch_for_stats ) ch_versions = ch_versions.mix( SAMTOOLS_IDXSTATS.out.versions.first() ) From 76a46020c39f7b69d7ab64576527a8e816618c13 Mon Sep 17 00:00:00 2001 From: sainsachiko Date: Tue, 8 Jul 2025 11:26:05 +0100 Subject: [PATCH 21/28] Publish cram index when provide header --- conf/modules.config | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conf/modules.config b/conf/modules.config index 201f38f8..19844e53 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -166,7 +166,8 @@ process { // Only publish BAM/CRAM from SAMTOOLS_REHEADER if params.header is true, else from: SAMTOOLS_INDEX (BAM) and SAMTOOLS_CRAM (CRAM) saveAs: { filename -> filename == 'versions.yml' - || ((task.process =~ /SAMTOOLS_INDEX|SAMTOOLS_CRAM/) && params.header) + || ((task.process == 'CHANGE_NAME') || (task.process == 'SAMTOOLS_CRAM' && !file.name.endsWith('.cram')) + && params.header) || ((task.process =~ /SAMTOOLS_CRAM/) && filename.endsWith(".bam")) ? null : filename } From 8cde4262717b1c9e1c4abc88ce6cca2b07bb1c21 Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Wed, 9 Jul 2025 06:42:37 +0100 Subject: [PATCH 22/28] Remove trailing whitespace --- conf/modules.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/modules.config b/conf/modules.config index 19844e53..2e815e4f 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -166,7 +166,7 @@ process { // Only publish BAM/CRAM from SAMTOOLS_REHEADER if params.header is true, else from: SAMTOOLS_INDEX (BAM) and SAMTOOLS_CRAM (CRAM) saveAs: { filename -> filename == 'versions.yml' - || ((task.process == 'CHANGE_NAME') || (task.process == 'SAMTOOLS_CRAM' && !file.name.endsWith('.cram')) + || ((task.process == 'CHANGE_NAME') || (task.process == 'SAMTOOLS_CRAM' && !file.name.endsWith('.cram')) && params.header) || ((task.process =~ /SAMTOOLS_CRAM/) && filename.endsWith(".bam")) ? null : filename From 2b4f66ad6f1d93c4b64cd051828b1f57dd23d16e Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Mon, 14 Jul 2025 21:16:41 +0100 Subject: [PATCH 23/28] Remove a trailing empty line --- modules/local/change_name.nf | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/local/change_name.nf b/modules/local/change_name.nf index f2b6b1ca..a1d6a0f4 100644 --- a/modules/local/change_name.nf +++ b/modules/local/change_name.nf @@ -24,4 +24,3 @@ process CHANGE_NAME { touch ${new_fn} """ } - From 295e5b65559dbfc39fe137850c88bb48214a7aa2 Mon Sep 17 00:00:00 2001 From: sainsachiko Date: Tue, 15 Jul 2025 14:10:48 +0100 Subject: [PATCH 24/28] Merge pacbio chunked bam after alignment --- subworkflows/local/align_pacbio.nf | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/subworkflows/local/align_pacbio.nf b/subworkflows/local/align_pacbio.nf index dddfeec6..30171141 100644 --- a/subworkflows/local/align_pacbio.nf +++ b/subworkflows/local/align_pacbio.nf @@ -10,6 +10,7 @@ include { SAMTOOLS_SORMADUP as CONVERT_CRAM } from '../../modules/local/samtools include { CREATE_CRAM_FILTER_INPUT } from '../../subworkflows/local/create_cram_filter_input' include { MINIMAP2_ALIGN } from '../../modules/nf-core/minimap2/align/main' include { MERGE_OUTPUT } from '../../subworkflows/local/merge_output' +include { SAMTOOLS_MERGE } from '../../modules/nf-core/samtools/merge/main' workflow ALIGN_PACBIO { take: @@ -52,13 +53,29 @@ workflow ALIGN_PACBIO { MINIMAP2_ALIGN ( FILTER_PACBIO.out.fastq, fasta, true, "csi", false, false ) ch_versions = ch_versions.mix ( MINIMAP2_ALIGN.out.versions.first() ) + MINIMAP2_ALIGN.out.bam + |map { meta, bam -> [meta.id, meta, bam] } + |groupTuple() + |map { id, meta, bam -> + def newMeta = meta[0].findAll { key, value -> key != 'chunk_id' } + [newMeta, bam] + } + |set { collected_files_for_merge } + + + // Merge chunked aligned bams from minimap align + SAMTOOLS_MERGE ( + collected_files_for_merge, + fasta, + [ [], [] ] + ) + // // SUBWORKFLOW: Merge all alignment output by sample name // - ch_sort = MERGE_OUTPUT( MINIMAP2_ALIGN.out.bam ).bam + ch_sort = MERGE_OUTPUT( SAMTOOLS_MERGE.out.bam ).bam ch_versions = ch_versions.mix ( MERGE_OUTPUT.out.versions) - emit: bam = ch_sort // channel: [ val(meta), /path/to/bam ] versions = ch_versions // channel: [ versions.yml ] From 493b28ffb62afdee2c7e6126662ea295d97ab033 Mon Sep 17 00:00:00 2001 From: sainsachiko Date: Tue, 15 Jul 2025 16:15:46 +0100 Subject: [PATCH 25/28] Do not publish BAM from SAMTOOLS_INDEX if re-header BAMs --- conf/modules.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/modules.config b/conf/modules.config index 2e815e4f..783586b0 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -166,7 +166,7 @@ process { // Only publish BAM/CRAM from SAMTOOLS_REHEADER if params.header is true, else from: SAMTOOLS_INDEX (BAM) and SAMTOOLS_CRAM (CRAM) saveAs: { filename -> filename == 'versions.yml' - || ((task.process == 'CHANGE_NAME') || (task.process == 'SAMTOOLS_CRAM' && !file.name.endsWith('.cram')) + || ((task.process =~ /SAMTOOLS_INDEX/ && filename.endsWith(".bam") ) || (task.process =~ /SAMTOOLS_CRAM/ && filename.endsWith('.cram')) && params.header) || ((task.process =~ /SAMTOOLS_CRAM/) && filename.endsWith(".bam")) ? null : filename From f4a660bcf4870dbf47fc7f30620d6789e31227a6 Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Tue, 15 Jul 2025 17:17:27 +0100 Subject: [PATCH 26/28] [lint] Removed trailing whitespace --- subworkflows/local/align_pacbio.nf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/subworkflows/local/align_pacbio.nf b/subworkflows/local/align_pacbio.nf index 30171141..7459b78c 100644 --- a/subworkflows/local/align_pacbio.nf +++ b/subworkflows/local/align_pacbio.nf @@ -56,9 +56,9 @@ workflow ALIGN_PACBIO { MINIMAP2_ALIGN.out.bam |map { meta, bam -> [meta.id, meta, bam] } |groupTuple() - |map { id, meta, bam -> + |map { id, meta, bam -> def newMeta = meta[0].findAll { key, value -> key != 'chunk_id' } - [newMeta, bam] + [newMeta, bam] } |set { collected_files_for_merge } From 6df042c69a07a1b9c79a39906e2a6ff4e0860369 Mon Sep 17 00:00:00 2001 From: sainsachiko Date: Wed, 16 Jul 2025 09:58:54 +0100 Subject: [PATCH 27/28] Publish bam file when no header --- conf/modules.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index 783586b0..3b894951 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -166,8 +166,8 @@ process { // Only publish BAM/CRAM from SAMTOOLS_REHEADER if params.header is true, else from: SAMTOOLS_INDEX (BAM) and SAMTOOLS_CRAM (CRAM) saveAs: { filename -> filename == 'versions.yml' - || ((task.process =~ /SAMTOOLS_INDEX/ && filename.endsWith(".bam") ) || (task.process =~ /SAMTOOLS_CRAM/ && filename.endsWith('.cram')) - && params.header) + || (task.process =~ /SAMTOOLS_INDEX/ && filename.endsWith(".bam") && params.header ) + || (task.process =~ /SAMTOOLS_CRAM/ && filename.endsWith('.cram') && params.header ) || ((task.process =~ /SAMTOOLS_CRAM/) && filename.endsWith(".bam")) ? null : filename } From dad4ee7ef3f252942be377dd4e9d21b410506818 Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Wed, 16 Jul 2025 11:31:44 +0100 Subject: [PATCH 28/28] [lint] Removed trailing whitespace --- conf/modules.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/modules.config b/conf/modules.config index 3b894951..3281c312 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -166,7 +166,7 @@ process { // Only publish BAM/CRAM from SAMTOOLS_REHEADER if params.header is true, else from: SAMTOOLS_INDEX (BAM) and SAMTOOLS_CRAM (CRAM) saveAs: { filename -> filename == 'versions.yml' - || (task.process =~ /SAMTOOLS_INDEX/ && filename.endsWith(".bam") && params.header ) + || (task.process =~ /SAMTOOLS_INDEX/ && filename.endsWith(".bam") && params.header ) || (task.process =~ /SAMTOOLS_CRAM/ && filename.endsWith('.cram') && params.header ) || ((task.process =~ /SAMTOOLS_CRAM/) && filename.endsWith(".bam")) ? null : filename