From d685d3aebbe95560eb44d9f3b387ff520e770afd Mon Sep 17 00:00:00 2001 From: Abhinav Sharma Date: Sat, 26 Oct 2024 16:25:15 +0200 Subject: [PATCH 1/4] initiate tbpore modules [ci skip] --- .../tbpore/modules/local/tbpore/download.nf | 91 +++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 modules/tbpore/modules/local/tbpore/download.nf diff --git a/modules/tbpore/modules/local/tbpore/download.nf b/modules/tbpore/modules/local/tbpore/download.nf new file mode 100644 index 0000000..07da4cb --- /dev/null +++ b/modules/tbpore/modules/local/tbpore/download.nf @@ -0,0 +1,91 @@ +// TODO nf-core: If in doubt look at other nf-core/modules to see how we are doing things! :) +// https://github.com/nf-core/modules/tree/master/modules/nf-core/ +// You can also ask for help via your pull request or on the #modules channel on the nf-core Slack workspace: +// https://nf-co.re/join +// TODO nf-core: A module file SHOULD only define input and output files as command-line parameters. +// All other parameters MUST be provided using the "task.ext" directive, see here: +// https://www.nextflow.io/docs/latest/process.html#ext +// where "task.ext" is a string. +// Any parameters that need to be evaluated in the context of a particular sample +// e.g. single-end/paired-end data MUST also be defined and evaluated appropriately. +// TODO nf-core: Software that can be piped together SHOULD be added to separate module files +// unless there is a run-time, storage advantage in implementing in this way +// e.g. it's ok to have a single module for bwa to output BAM instead of SAM: +// bwa mem | samtools view -B -T ref.fasta +// TODO nf-core: Optional inputs are not currently supported by Nextflow. However, using an empty +// list (`[]`) instead of a file can be used to work around this issue. + +process TBPORE_DOWNLOAD { + tag "$meta.id" + label 'process_high' + + // TODO nf-core: List required Conda package(s). + // Software MUST be pinned to channel (i.e. "bioconda"), version (i.e. "1.10"). + // For Conda, the build (i.e. "h9402c20_2") must be EXCLUDED to support installation on different operating systems. + // TODO nf-core: See section in main README for further information regarding finding and adding container addresses to the section below. + conda "${moduleDir}/environment.yml" + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'https://depot.galaxyproject.org/singularity/tbpore:0.7.1--pyhdfd78af_0': + 'biocontainers/tbpore:0.7.1--pyhdfd78af_0' }" + + input: + // TODO nf-core: Where applicable all sample-specific information e.g. "id", "single_end", "read_group" + // MUST be provided as an input via a Groovy Map called "meta". + // This information may not be required in some instances e.g. indexing reference genome files: + // https://github.com/nf-core/modules/blob/master/modules/nf-core/bwa/index/main.nf + // TODO nf-core: Where applicable please provide/convert compressed files as input/output + // e.g. "*.fastq.gz" and NOT "*.fastq", "*.bam" and NOT "*.sam" etc. + tuple val(meta), path(bam) + + output: + // TODO nf-core: Named file extensions MUST be emitted for ALL output channels + tuple val(meta), path("*.bam"), emit: bam + // TODO nf-core: List additional required output channels/values here + path "versions.yml" , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + // TODO nf-core: Where possible, a command MUST be provided to obtain the version number of the software e.g. 1.10 + // If the software is unable to output a version number on the command-line then it can be manually specified + // e.g. https://github.com/nf-core/modules/blob/master/modules/nf-core/homer/annotatepeaks/main.nf + // Each software used MUST provide the software name and version number in the YAML version file (versions.yml) + // TODO nf-core: It MUST be possible to pass additional parameters to the tool as a command-line string via the "task.ext.args" directive + // TODO nf-core: If the tool supports multi-threading then you MUST provide the appropriate parameter + // using the Nextflow "task" variable e.g. "--threads $task.cpus" + // TODO nf-core: Please replace the example samtools command below with your module's command + // TODO nf-core: Please indent the command appropriately (4 spaces!!) to help with readability ;) + """ + samtools \\ + sort \\ + $args \\ + -@ $task.cpus \\ + -o ${prefix}.bam \\ + -T $prefix \\ + $bam + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + tbpore: \$(samtools --version |& sed '1!d ; s/samtools //') + END_VERSIONS + """ + + stub: + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + // TODO nf-core: A stub section should mimic the execution of the original module as best as possible + // Have a look at the following examples: + // Simple example: https://github.com/nf-core/modules/blob/818474a292b4860ae8ff88e149fbcda68814114d/modules/nf-core/bcftools/annotate/main.nf#L47-L63 + // Complex example: https://github.com/nf-core/modules/blob/818474a292b4860ae8ff88e149fbcda68814114d/modules/nf-core/bedtools/split/main.nf#L38-L54 + """ + touch ${prefix}.bam + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + tbpore: \$(samtools --version |& sed '1!d ; s/samtools //') + END_VERSIONS + """ +} From 636a86c7f61c938c72d89728ae12df7f51a01fd1 Mon Sep 17 00:00:00 2001 From: Opeyemi Alayande Date: Tue, 29 Oct 2024 09:02:47 +0100 Subject: [PATCH 2/4] testing write access --- main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.nf b/main.nf index e7e9c5f..fef5585 100644 --- a/main.nf +++ b/main.nf @@ -96,7 +96,7 @@ workflow { NFCORE_TBANALYZER.out.multiqc_report ) } - +// Everything seemed okay /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ THE END From c99b1712c1e8bf76117fbb2bbf9d09d1315196de Mon Sep 17 00:00:00 2001 From: Opeyemi Alayande Date: Sun, 3 Nov 2024 23:41:55 +0100 Subject: [PATCH 3/4] wrapped tb-pore commands in 2 processes and wrote their corresponding tests [ci skip] --- .../tbpore/modules/local/tbpore/download.nf | 91 ------------------- .../modules/local/tbpore_cluster/main.nf | 47 ++++++++++ .../local/tbpore_cluster/tests/main.nf.test | 55 +++++++++++ .../modules/local/tbpore_process/main.nf | 61 +++++++++++++ .../local/tbpore_process/tests/main.nf.test | 43 +++++++++ .../tbpore_process/tests/main.nf.test.snap | 43 +++++++++ nf-test.config | 8 ++ tests/nextflow.config | 5 + 8 files changed, 262 insertions(+), 91 deletions(-) delete mode 100644 modules/tbpore/modules/local/tbpore/download.nf create mode 100644 modules/tbpore/modules/local/tbpore_cluster/main.nf create mode 100644 modules/tbpore/modules/local/tbpore_cluster/tests/main.nf.test create mode 100644 modules/tbpore/modules/local/tbpore_process/main.nf create mode 100644 modules/tbpore/modules/local/tbpore_process/tests/main.nf.test create mode 100644 modules/tbpore/modules/local/tbpore_process/tests/main.nf.test.snap create mode 100644 nf-test.config create mode 100644 tests/nextflow.config diff --git a/modules/tbpore/modules/local/tbpore/download.nf b/modules/tbpore/modules/local/tbpore/download.nf deleted file mode 100644 index 07da4cb..0000000 --- a/modules/tbpore/modules/local/tbpore/download.nf +++ /dev/null @@ -1,91 +0,0 @@ -// TODO nf-core: If in doubt look at other nf-core/modules to see how we are doing things! :) -// https://github.com/nf-core/modules/tree/master/modules/nf-core/ -// You can also ask for help via your pull request or on the #modules channel on the nf-core Slack workspace: -// https://nf-co.re/join -// TODO nf-core: A module file SHOULD only define input and output files as command-line parameters. -// All other parameters MUST be provided using the "task.ext" directive, see here: -// https://www.nextflow.io/docs/latest/process.html#ext -// where "task.ext" is a string. -// Any parameters that need to be evaluated in the context of a particular sample -// e.g. single-end/paired-end data MUST also be defined and evaluated appropriately. -// TODO nf-core: Software that can be piped together SHOULD be added to separate module files -// unless there is a run-time, storage advantage in implementing in this way -// e.g. it's ok to have a single module for bwa to output BAM instead of SAM: -// bwa mem | samtools view -B -T ref.fasta -// TODO nf-core: Optional inputs are not currently supported by Nextflow. However, using an empty -// list (`[]`) instead of a file can be used to work around this issue. - -process TBPORE_DOWNLOAD { - tag "$meta.id" - label 'process_high' - - // TODO nf-core: List required Conda package(s). - // Software MUST be pinned to channel (i.e. "bioconda"), version (i.e. "1.10"). - // For Conda, the build (i.e. "h9402c20_2") must be EXCLUDED to support installation on different operating systems. - // TODO nf-core: See section in main README for further information regarding finding and adding container addresses to the section below. - conda "${moduleDir}/environment.yml" - container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/tbpore:0.7.1--pyhdfd78af_0': - 'biocontainers/tbpore:0.7.1--pyhdfd78af_0' }" - - input: - // TODO nf-core: Where applicable all sample-specific information e.g. "id", "single_end", "read_group" - // MUST be provided as an input via a Groovy Map called "meta". - // This information may not be required in some instances e.g. indexing reference genome files: - // https://github.com/nf-core/modules/blob/master/modules/nf-core/bwa/index/main.nf - // TODO nf-core: Where applicable please provide/convert compressed files as input/output - // e.g. "*.fastq.gz" and NOT "*.fastq", "*.bam" and NOT "*.sam" etc. - tuple val(meta), path(bam) - - output: - // TODO nf-core: Named file extensions MUST be emitted for ALL output channels - tuple val(meta), path("*.bam"), emit: bam - // TODO nf-core: List additional required output channels/values here - path "versions.yml" , emit: versions - - when: - task.ext.when == null || task.ext.when - - script: - def args = task.ext.args ?: '' - def prefix = task.ext.prefix ?: "${meta.id}" - // TODO nf-core: Where possible, a command MUST be provided to obtain the version number of the software e.g. 1.10 - // If the software is unable to output a version number on the command-line then it can be manually specified - // e.g. https://github.com/nf-core/modules/blob/master/modules/nf-core/homer/annotatepeaks/main.nf - // Each software used MUST provide the software name and version number in the YAML version file (versions.yml) - // TODO nf-core: It MUST be possible to pass additional parameters to the tool as a command-line string via the "task.ext.args" directive - // TODO nf-core: If the tool supports multi-threading then you MUST provide the appropriate parameter - // using the Nextflow "task" variable e.g. "--threads $task.cpus" - // TODO nf-core: Please replace the example samtools command below with your module's command - // TODO nf-core: Please indent the command appropriately (4 spaces!!) to help with readability ;) - """ - samtools \\ - sort \\ - $args \\ - -@ $task.cpus \\ - -o ${prefix}.bam \\ - -T $prefix \\ - $bam - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - tbpore: \$(samtools --version |& sed '1!d ; s/samtools //') - END_VERSIONS - """ - - stub: - def args = task.ext.args ?: '' - def prefix = task.ext.prefix ?: "${meta.id}" - // TODO nf-core: A stub section should mimic the execution of the original module as best as possible - // Have a look at the following examples: - // Simple example: https://github.com/nf-core/modules/blob/818474a292b4860ae8ff88e149fbcda68814114d/modules/nf-core/bcftools/annotate/main.nf#L47-L63 - // Complex example: https://github.com/nf-core/modules/blob/818474a292b4860ae8ff88e149fbcda68814114d/modules/nf-core/bedtools/split/main.nf#L38-L54 - """ - touch ${prefix}.bam - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - tbpore: \$(samtools --version |& sed '1!d ; s/samtools //') - END_VERSIONS - """ -} diff --git a/modules/tbpore/modules/local/tbpore_cluster/main.nf b/modules/tbpore/modules/local/tbpore_cluster/main.nf new file mode 100644 index 0000000..6046889 --- /dev/null +++ b/modules/tbpore/modules/local/tbpore_cluster/main.nf @@ -0,0 +1,47 @@ +process TBPORE_CLUSTER { + tag "$meta.id" + label 'process_low' + + conda "${moduleDir}/environment.yml" + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'https://depot.galaxyproject.org/singularity/tbpore:0.7.1--pyhdfd78af_0': + 'biocontainers/tbpore:0.7.1--pyhdfd78af_0' }" + + input: + tuple val(meta), path(consensus_fa) + + output: + + tuple val(meta), path("*.txt"), emit: txt + path "versions.yml" , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + + """ + tbpore cluster $args \\ + --threads $task.cpus \\ + $consensus_fa + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + tbpore: \$(tbpore --version |& sed s/tbpore, version //') + END_VERSIONS + """ + + stub: + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + + """ + touch ${prefix}.txt + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + tbpore: \$(tbpore --version |& sed s/tbpore, version //') + END_VERSIONS + """ +} diff --git a/modules/tbpore/modules/local/tbpore_cluster/tests/main.nf.test b/modules/tbpore/modules/local/tbpore_cluster/tests/main.nf.test new file mode 100644 index 0000000..0e591dc --- /dev/null +++ b/modules/tbpore/modules/local/tbpore_cluster/tests/main.nf.test @@ -0,0 +1,55 @@ +nextflow_process { + + name "Test Process TBPORE_CLUSTER" + script "../main.nf" + process "TBPORE_CLUSTER" + + setup { + run("TBPORE_PROCESS") { + script "../../tbpore_process/main.nf" + + params { + modules_testdata_base_path = 'https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/' + } + + process { + """ + + input[0] = Channel.of([ + [ id:'test', single_end:true ], + file( + params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', + checkIfExists: true + ) + ]) + + input[1] = Channel.of( + file("/.tbpore/decontamination_db/remove_contam.map-ont.mmi") + ) + """ + } + } + } + + test("sarscov2 single end [fastq]") { + + when { + params { + // define parameters here. Example: + outdir = "tests/results" + } + process { + """ + // input[0] = TBPORE_PROCESS.out.fa | collect + """ + } + } + + then { + assert process.success + assert snapshot(process.out).match() + } + + } + +} diff --git a/modules/tbpore/modules/local/tbpore_process/main.nf b/modules/tbpore/modules/local/tbpore_process/main.nf new file mode 100644 index 0000000..780f538 --- /dev/null +++ b/modules/tbpore/modules/local/tbpore_process/main.nf @@ -0,0 +1,61 @@ + +process TBPORE_PROCESS { + tag "$meta.id" + label 'process_low' + + conda "${moduleDir}/environment.yml" + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'https://depot.galaxyproject.org/singularity/tbpore:0.7.1--pyhdfd78af_0': + 'biocontainers/tbpore:0.7.1--pyhdfd78af_0' }" + + input: + tuple val(meta), path(fastq) + path db + + output: + tuple val(meta), path("*.json"), emit: json + tuple val(meta), path("*.fa") , emit: fa + tuple val(meta), path("*.bcf") , emit: bcf + tuple val(meta), path("*.txt") , emit: txt + path "versions.yml" , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + + """ + tbpore process $args \\ + --threads 6 \\ + --name $prefix \\ + --db $db \\ + $fastq + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + tbpore: \$(tbpore --version |& sed s/tbpore, version //') + END_VERSIONS + """ + + stub: + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + // TODO nf-core: A stub section should mimic the execution of the original module as best as possible + // Have a look at the following examples: + // Simple example: https://github.com/nf-core/modules/blob/818474a292b4860ae8ff88e149fbcda68814114d/modules/nf-core/bcftools/annotate/main.nf#L47-L63 + // Complex example: https://github.com/nf-core/modules/blob/818474a292b4860ae8ff88e149fbcda68814114d/modules/nf-core/bedtools/split/main.nf#L38-L54 + """ + touch ${prefix}.json + touch ${prefix}.fa + touch ${prefix}.bcf + touch ${prefix}.txt + + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + tbpore: \$(tbpore --version |& sed s/tbpore, version //') + END_VERSIONS + """ +} diff --git a/modules/tbpore/modules/local/tbpore_process/tests/main.nf.test b/modules/tbpore/modules/local/tbpore_process/tests/main.nf.test new file mode 100644 index 0000000..25eb488 --- /dev/null +++ b/modules/tbpore/modules/local/tbpore_process/tests/main.nf.test @@ -0,0 +1,43 @@ +nextflow_process { + + name "Test Process TBPORE_PROCESS" + script "../main.nf" + process "TBPORE_PROCESS" + + tag TBPORE_PROCESS + + test("sarscov2 single end fastq") { + + when { + params { + //define parameters here. Example: + modules_testdata_base_path = 'https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/' + outdir = "tests/results" + } + process { + """ + input[0] = Channel.of([ + [ id:'test', single_end:true ], + file( + params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', + checkIfExists: true + ) + ]) + input[1] = Channel.fromPath( + file("/.tbpore/decontamination_db/remove_contam.map-ont.mmi") + ) + """ + } + } + + then { + assertAll( + + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + + } + +} diff --git a/modules/tbpore/modules/local/tbpore_process/tests/main.nf.test.snap b/modules/tbpore/modules/local/tbpore_process/tests/main.nf.test.snap new file mode 100644 index 0000000..ca2413c --- /dev/null +++ b/modules/tbpore/modules/local/tbpore_process/tests/main.nf.test.snap @@ -0,0 +1,43 @@ +{ + "real data fastq": { + "content": [ + { + "0": [ + + ], + "1": [ + + ], + "2": [ + + ], + "3": [ + + ], + "4": [ + + ], + "bcf": [ + + ], + "fa": [ + + ], + "json": [ + + ], + "txt": [ + + ], + "versions": [ + + ] + } + ], + "meta": { + "nf-test": "0.9.1", + "nextflow": "24.04.4" + }, + "timestamp": "2024-11-03T12:51:08.385030023" + } +} \ No newline at end of file diff --git a/nf-test.config b/nf-test.config new file mode 100644 index 0000000..870799d --- /dev/null +++ b/nf-test.config @@ -0,0 +1,8 @@ +config { + + testsDir "tests" + workDir ".nf-test" + configFile "tests/nextflow.config" + profile "" + +} diff --git a/tests/nextflow.config b/tests/nextflow.config new file mode 100644 index 0000000..c19b1ad --- /dev/null +++ b/tests/nextflow.config @@ -0,0 +1,5 @@ +/* +======================================================================================== + Nextflow config file for running tests +======================================================================================== +*/ From e20d3783558b573a94f1ca653937b065ace9d3a8 Mon Sep 17 00:00:00 2001 From: Opeyemi Alayande Date: Sun, 3 Nov 2024 23:43:13 +0100 Subject: [PATCH 4/4] wrapped tb-pore commands in 2 processes and wrote their corresponding tests [ci skip] --- modules/tbpore/modules/local/tbpore_process/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/tbpore/modules/local/tbpore_process/main.nf b/modules/tbpore/modules/local/tbpore_process/main.nf index 780f538..f6a4d66 100644 --- a/modules/tbpore/modules/local/tbpore_process/main.nf +++ b/modules/tbpore/modules/local/tbpore_process/main.nf @@ -28,7 +28,7 @@ process TBPORE_PROCESS { """ tbpore process $args \\ - --threads 6 \\ + --threads $task.cpus \\ --name $prefix \\ --db $db \\ $fastq