Release 6.1.0

.github/workflows/Update-Version-and-create-Realeases-PR.yml

@@ -63,9 +63,10 @@ jobs:
           $(pdm venv activate)
           npm install -g doctoc
           for pipeline in ampliconseq chipseq covseq dnaseq methylseq longread_dnaseq nanopore_covseq rnaseq rnaseq_denovo_assembly rnaseq_light; do
-            SUMMARY=$(genpipes $pipeline --help | awk '/Summary:/, /^Steps:/' | sed '$d' | sed '1d' | sed '/Attributes:/,$d')
-            HELPTEXT=`genpipes $pipeline --help | sed -e '/Summary:/, /^Steps:/{/^Steps:/!d}' | sed -e '/Returns:/,+1d'`
-            echo -e "[TOC]\n${SUMMARY}\nUsage\n-----\n\n\`\`\`\n#!text\n${HELPTEXT}\n" > genpipes/pipelines/$pipeline/README.md
+            SUMMARY=$(genpipes $pipeline --help | awk '/Summary:/ {flag=1; next} /Steps:/ {exit} flag')
+            USAGE=$(genpipes ampliconseq --help | awk '/Summary:/ {exit} {print}')
+            HELPTEXT=$(genpipes $pipeline --help | awk '/Steps:/ {flag=1} flag')
+            echo -e "${SUMMARY}\n\nUsage\n-----\n\n\`\`\`text\n${USAGE}\n\n${HELPTEXT}\n" > genpipes/pipelines/$pipeline/README.md
             doctoc genpipes/pipelines/$pipeline/README.md
           done
       - name: Update autocomplete files

README.md

@@ -10,7 +10,7 @@ GenPipes consists of Python scripts which create a list of jobs running Bash com
 On this page:
 
 * [Software requirement](#software-requirement)
-* [Quick setup for Abacus, Beluga, Narval, Graham and Cedar users](#quick-setup-for-abacus-beluga-narval-graham-and-cedar-users)
+* [Quick setup for Abacus, Rorqual, Narval, Nibi and Fir users](#quick-setup-for-abacus-rorqual-narval-nibi-and-fir-users)
 * [Download and setup for external users](#download-and-setup-for-external-users)
     * [Download](#download)
     * [Installation](#installation)
@@ -45,7 +45,7 @@ GenPipes has been tested with Python 3.11.1 and 3.12.2
 It may work with other versions of python, but this has not been extensively tested. 
 
 
-Quick setup for Abacus, Beluga, Narval, Graham and Cedar users
+Quick setup for Abacus, Rorqual, Narval, Nibi and Fir users
 --------------------------------------------------------------
 Genomes and modules used by the pipelines are already installed on a CVMFS partition mounted on all those clusters in `/cvmfs/soft.mugqic/root`.
 To access them, add the following lines to your *$HOME/.bash_profile*:
@@ -57,23 +57,23 @@ export MUGQIC_INSTALL_HOME=/cvmfs/soft.mugqic/root
 module use $MUGQIC_INSTALL_HOME/modulefiles
 ```
 
-For MUGQIC analysts, add the following lines to your *$HOME/.bash_profile*:
+For C3G analysts, add the following lines to your *$HOME/.bash_profile*:
 
 ```bash
 umask 0006
-# MUGQIC genomes and modules for MUGQIC analysts
+# MUGQIC genomes and modules for C3G analysts
 HOST=`hostname`;
 DNSDOMAIN=`dnsdomainname`;
 export MUGQIC_INSTALL_HOME=/cvmfs/soft.mugqic/root
 if [[ $HOST == abacus* || $DNSDOMAIN == ferrier.genome.mcgill.ca ]]; then
   export MUGQIC_INSTALL_HOME_DEV=/lb/project/mugqic/analyste_dev
 elif [[ $HOST == ip* || $DNSDOMAIN == m  ]]; then
   export MUGQIC_INSTALL_HOME_DEV=/project/6007512/C3G/analyste_dev
-elif [[ $HOST == cedar* || $DNSDOMAIN == cedar.computecanada.ca ]]; then
+elif [[ $HOST == fir* || $DNSDOMAIN == fir.alliancecan.ca ]]; then
   export MUGQIC_INSTALL_HOME_DEV=/project/6007512/C3G/analyste_dev
-elif [[ $HOST == beluga* || $DNSDOMAIN == beluga.computecanada.ca ]]; then
+elif [[ $HOST == rorqual* || $DNSDOMAIN == rorqual.alliancecan.ca ]]; then
   export MUGQIC_INSTALL_HOME_DEV=/project/6007512/C3G/analyste_dev
-elif [[ $HOST == narval* || $DNSDOMAIN == narval.computecanada.ca ]]; then
+elif [[ $HOST == narval* || $DNSDOMAIN == narval.alliancecan.ca ]]; then
   export MUGQIC_INSTALL_HOME_DEV=/project/rrg-bourqueg-ad/C3G/analyste_dev
 fi
 module use $MUGQIC_INSTALL_HOME/modulefiles $MUGQIC_INSTALL_HOME_DEV/modulefiles
@@ -501,12 +501,12 @@ You can also add a list of other configuration files to `--config`.
 Files are read in the list order and each parameter value is overwritten if redefined in the next file.
 
 This is useful to customize settings for a specific cluster or genome.
-Each cluster has a special configuration file available (for example, `beluga.ini` and `narval.ini`) in the common_ini directory.
+Each cluster has a special configuration file available (for example, `rorqual.ini` and `narval.ini`) in the common_ini directory.
 And various genome settings are available in `$MUGQIC_INSTALL_HOME/genomes/species/`.
 
-For example, to run the DNA-Seq pipeline on the beluga cluster with *Mus musculus* reference genome:
+For example, to run the DNA-Seq pipeline on the rorqual cluster with *Mus musculus* reference genome:
 ```bash
-genpipes dnaseq --config $GENPIPES_INIS/dnaseq/dnaseq.base.ini $GENPIPES_INIS/common_ini/beluga.ini $MUGQIC_INSTALL_HOME/genomes/species/Mus_musculus.GRCm38/Mus_musculus.GRCm38.ini ...
+genpipes dnaseq --config $GENPIPES_INIS/dnaseq/dnaseq.base.ini $GENPIPES_INIS/common_ini/rorqual.ini $MUGQIC_INSTALL_HOME/genomes/species/Mus_musculus.GRCm38/Mus_musculus.GRCm38.ini ...
 ```
 
 
@@ -615,7 +615,7 @@ id  name    status  user    node    priority    submit_time eligible_time   star
 
 A Note about non-Alliance Clusters
 ------------
-The default scheduler in GenPipes is the SLURM scheduler. Beluga, Narval, Cedar and Graham use the SLURM scheduler. To use GenPipes on abacus, don't forget to add the "-j pbs" option.
+The default scheduler in GenPipes is the SLURM scheduler. Rorqual, Narval, Fir and Nibi use the SLURM scheduler. To use GenPipes on abacus, don't forget to add the "-j pbs" option.
 
 Call home
 ---------

genpipes/__version__.py

@@ -1 +1 @@
-__version__ = '6.0.0'
+__version__ = '6.0.0.dev'

genpipes/bfx/annotsv.py

@@ -90,14 +90,12 @@ def html(
   --datatableDir $KNOTANNOTSV_HOME/DataTables \\
   --annotSVfile {input_tsv} \\
   -genomeBuild {genome_build} \\
-  --outDir {output_dir} \\
-  --outPrefix {output_prefix}""".format(
+  --outDir {output_dir}""".format(
             html_options=global_conf.global_get(ini_section, 'html_options', required=False),
             config=global_conf.global_get(ini_section, 'knotAnnotSV_config', required=False) if global_conf.global_get(ini_section, 'knotAnnotSV_config', required=False) else "$KNOTANNOTSV_CONFIG",
             input_tsv=input_tsv,
             genome_build=global_conf.global_get(ini_section, 'assembly'),
-            output_dir=output_dir,
-            output_prefix=output_prefix
+            output_dir=output_dir
         )
     )
 
@@ -130,14 +128,12 @@ def excel(
   --annotSVfile {input_tsv} \\
   -genomeBuild {genome_build} \\
   --geneCountThreshold {genecount_threshold} \\
-  --outDir {output_dir} \\
-  --outPrefix {output_prefix}""".format(
+  --outDir {output_dir}""".format(
             excel_options=global_conf.global_get(ini_section, 'excel_options', required=False),
             config=global_conf.global_get(ini_section, 'knotAnnotSV_config', required=False) if global_conf.global_get(ini_section, 'knotAnnotSV_config', required=False) else "$KNOTANNOTSV_CONFIG",
             input_tsv=input_tsv,
             genome_build=global_conf.global_get(ini_section, 'assembly'),
             genecount_threshold=global_conf.global_get(ini_section, 'genecount_threshold'),
-            output_dir=output_dir,
-            output_prefix=output_prefix
+            output_dir=output_dir
         )
     )

genpipes/bfx/bcftools.py

@@ -155,15 +155,15 @@ def concat(inputs, output, options=None):
         )
     )
 
-def view(input, output, filter_options=None):
+def view(input, output, filter_options=None, ini_section="bcftools_view"):
     """
     Generalized view 
     """
     return Job(
         [input],
         [output],
         [
-            ['bcftools_view', 'module_bcftools']
+            [ini_section, 'module_bcftools']
         ],
         command="""\
 bcftools \\
@@ -318,3 +318,24 @@ def split(input, output, options, ini_section='bcftools_split'):
         output=output
         )
     )
+
+def reheader(input, output, options, ini_section='bcftools_reheader'):
+    """
+    modify header of vcf or change sample names
+    """
+
+    return Job(
+        [input],
+        [output],
+        [
+            [ini_section, 'module_bcftools']
+        ],
+        command="""\
+bcftools \\
+    reheader {options} \\
+    {output} {input}""".format(
+        options=options,
+        input=input if input else "",
+        output="-o " + output if output else ""
+    )
+    )

genpipes/bfx/clair3.py

@@ -0,0 +1,75 @@
+################################################################################
+# Copyright (C) 2025 C3G, The Victor Phillip Dahdaleh Institute of Genomic Medicine at McGill University
+#
+# This file is part of GenPipes.
+#
+# GenPipes is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# GenPipes is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with GenPipes.  If not, see <http://www.gnu.org/licenses/>.
+################################################################################
+
+import os
+
+# MUGQIC Modules
+from ..core.config import global_conf
+from ..core.job import Job
+
+
+def run(
+    input_bam,
+    output_dir,
+    sample_name,
+    platform,
+    region=None,
+    ini_section='clair3'
+    ):
+    """
+    Call variants with Clair3
+
+    :return: a job for Clair3 variant calling
+    """
+
+    outputs = [
+        os.path.join(output_dir, "pileup.vcf.gz"),
+        os.path.join(output_dir, "full_alignment.vcf.gz"),
+        os.path.join(output_dir, "merge_output.vcf.gz"),
+        os.path.join(output_dir, "phased_merge_output.vcf.gz")
+    ]
+
+    return Job(
+        [input_bam],
+        outputs,
+        [
+            [ini_section, "module_clair3"],
+        ],
+        command="""\
+run_clair3.sh {other_options} \\
+  --bam_fn={input_bam} \\
+  --ref_fn={genome_fasta} \\
+  --threads={threads} \\
+  --platform="{platform}" \\
+  --model_path={model_path} \\
+  {region} {sites_to_call} \\
+  --sample_name={sample_name} \\
+  --output={output}""".format(
+            other_options=global_conf.global_get(ini_section, 'other_options', required=False),
+            input_bam=input_bam,
+            genome_fasta=global_conf.global_get(ini_section, 'genome_fasta'),
+            threads=global_conf.global_get(ini_section, 'threads'),
+            platform=platform,
+            model_path=global_conf.global_get(ini_section, 'model_path', param_type='dirpath'),
+            region=region if region else "",
+            sites_to_call="--vcf_fn=" + global_conf.global_get(ini_section, 'sites_to_call', required=False, param_type='filepath') if global_conf.global_get(ini_section, 'sites_to_call', required=False) else "",
+            sample_name=sample_name,
+            output=output_dir
+        )
+    )

genpipes/bfx/clairS.py

@@ -0,0 +1,78 @@
+################################################################################
+# Copyright (C) 2025 C3G, The Victor Phillip Dahdaleh Institute of Genomic Medicine at McGill University
+#
+# This file is part of GenPipes.
+#
+# GenPipes is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# GenPipes is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with GenPipes.  If not, see <http://www.gnu.org/licenses/>.
+################################################################################
+
+import os
+
+# MUGQIC Modules
+from ..core.config import global_conf
+from ..core.job import Job
+
+
+def run(
+    normal_bam,
+    tumor_bam,
+    output_dir,
+    tumor_name,
+    platform,
+    region=None,
+    ini_section='clairS'
+    ):
+    """
+    Call somatic small variants with ClairS
+
+    :return: a job for ClairS somatic small variant calling
+    """
+
+    outputs = [
+        os.path.join(output_dir, "output.vcf.gz"),
+        os.path.join(output_dir, "clair3_normal_germline_output.vcf.gz"),
+        os.path.join(output_dir, "indel.vcf.gz"),
+        os.path.join(output_dir, "snv.vcf.gz")
+    ]
+
+    return Job(
+        [normal_bam, tumor_bam],
+        outputs,
+        [
+            [ini_section, "module_clairS"],
+        ],
+        command="""\
+run_clairs {other_options} \\
+  --normal_bam_fn={normal_bam} \\
+  --tumor_bam_fn {tumor_bam} \\
+  --ref_fn={genome_fasta} \\
+  --threads={threads} \\
+  --platform="{platform}" \\
+  --clair3_model_path={model_path} \\
+  {region} {sites_to_call} \\
+  --sample_name={tumor_name} \\
+  --output_dir={output}""".format(
+            other_options=global_conf.global_get(ini_section, 'other_options', required=False),
+            normal_bam=normal_bam,
+            tumor_bam=tumor_bam,
+            genome_fasta=global_conf.global_get(ini_section, 'genome_fasta'),
+            threads=global_conf.global_get(ini_section, 'threads'),
+            platform=platform,
+            model_path=global_conf.global_get(ini_section, 'model_path', param_type='dirpath'),
+            region=region if region else "",
+            sites_to_call="--vcf_fn=" + global_conf.global_get(ini_section, 'sites_to_call', required=False, param_type='filepath') if global_conf.global_get(ini_section, 'sites_to_call', required=False) else "",
+            tumor_name=tumor_name,
+            output=output_dir
+        )
+    )