Merge dev (0.7.1)

CHANGELOG.md

@@ -15,14 +15,28 @@ situations.
 
 ## [0.7.1] Unreleased
 ### Added
+- Slurm profile for PDC Dardel. Uses Snakemake's grouping feature to group
+  multiple HUMAnN jobs together on high memory nodes to better optimize core
+  billing for high-memory jobs such as HUMAnN.
 
 ### Fixed
+- Slurm profile: Fixed the incorrect rule names used for allocations for host
+  removal steps.
 
 ### Changed
+- Moved container specifications into config file. Now using more detailed
+  container specifications for most rules. This also enables easier use of
+  local copies of containers (e.g. in HPC environments without external network
+  access).
+- Changed compression of kraken2 host removal artifacts to use gzip instead of
+  pigz due to the latter not being present in the kraken2 biocontainer.
+- HUMANnN3: Updated to version 3.9, compatible with MetaPhlAn4 vJun23 database.
 
 ### Deprecated
 
 ### Removed
+- Removed mentions of assembly workflow from docs and config.
+- Removed outdated amrplusplus scripts left behind in the scripts subdirectory.
 
 
 ## [0.7.0] 2023-06-13

config/config.yaml

@@ -17,7 +17,6 @@ input_fn_pattern: "{sample}_{readpair}.fq.gz"
 samplesheet: ""           # Three-column samplesheet with sample_id,fastq_1,fastq_2 columns. Used instead of inputdir
 outdir: "output_dir"
 logdir: "output_dir/logs"
-dbdir: "databases"        # Databases will be downloaded to this dir
 report: "StaG_report-"    # Filename prefix for report file ("-{datetime}.html" automatically appended)
 email: ""                 # Email to send status message after completed/failed run.
 
@@ -29,6 +28,25 @@ s3_endpoint_url: "https://s3.ki.se"  # Use https://s3.amazonaws.com for Amazon S
 keep_local: False                    # Keep local copies of remote input files, default False.
 
 
+#########################
+# Container images
+#########################
+containers:
+    bbmap: "docker://quay.io/biocontainers/bbmap:39.06--h92535d8_0"
+    bowtie2: "docker://quay.io/biocontainers/bowtie2:2.5.1--py38he00c5e5_2"
+    bracken: "docker://quay.io/biocontainers/bracken:2.9--py39h1f90b4d_0"
+    fastp: "docker://quay.io/biocontainers/fastp:0.23.4--hadf994f_2"
+    humann: "docker://quay.io/biocontainers/humann:3.9--py312hdfd78af_0"
+    kaiju: "docker://quai.io/biocontainers/kaiju:1.10.1--h43eeafb_0"
+    kraken2: "docker://quay.io/biocontainers/kraken2:2.1.3--pl5321hdcf5f25_0"
+    krakenuniq: "docker://quay.io/biocontainers/krakenuniq:1.0.4--pl5321h6dccd9a_1"
+    krona: "docker://quay.io/biocontainers/krona:2.8.1--pl5321hdfd78af_1"
+    metaphlan: "docker://quay.io/biocontainers/metaphlan:4.1.0--pyhca03a8a_0"
+    multiqc: "docker://quay.io/biocontainers/multiqc:1.21--pyhdfd78af_0"
+    samtools: "docker://quay.io/biocontainers/samtools:1.19.2--h50ea8bc_1"
+    stag: "oras://ghcr.io/ctmrbio/stag-mwc:stag-mwc-develop"
+
+
 #########################
 # Pipeline steps included
 #########################
@@ -52,8 +70,6 @@ functional_profile:
 mappers:
     bbmap: False
     bowtie2: False
-assembly: False
-binning: False
 
 
 #########################
@@ -190,8 +206,3 @@ bowtie2:
           attribute_type: ""   # Attribute type to summarize counts for, default is "gene_id" (any attribute in the GTF file's attribute field can be used)
           extra: ""            # Extra featureCount command line parameters
 
-
-#########################
-# Assembly
-#########################
-# Assembly workflow was removed in StaG v0.7.0

docs/source/conf.py

@@ -56,9 +56,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '0.7.0'
+version = '0.7.1'
 # The full version, including alpha/beta/rc tags.
-release = '0.7.0'
+release = '0.7.1-dev'
 
 # reStructuredText prolog contains a string of reStructuredText that will be
 # included at the beginning of every source file that is read.

docs/source/overview.rst

@@ -15,6 +15,3 @@ The following image shows a simplified graph of the workflow of |full_name|:
     :alt: StaG mwc
     :align: left
 
-Note that several parts of the workflow are not yet implemented, most notably
-the structure of the sections performing metagenomic assembly are not yet
-finalized.

profiles/ctmr_gandalf/config.yaml

@@ -48,9 +48,9 @@ default-resources:
   - time="01:00:00"
   - mem_mb=10240
 set-threads:
-  - fastp=20
-  - kraken2_remove_host=20
-  - bowtie2_remove_host=20
+  - fastp=16
+  - kraken2_host_removal=20
+  - bowtie2_host_removal=20
   - bbcountunique=4
   - sketch=8
   - kaiju=32
@@ -70,10 +70,10 @@ set-threads:
 set-resources:
   - fastp:mem_mb=20240
   - fastp:time="02:00:00"
-  - kraken2_remove_host:mem_mb=10240
-  - kraken2_remove_host:time="02:00:00"
-  - bowtie2_remove_host:mem_mb=10240
-  - bowtie2_remove_host:time="06:00:00"
+  - kraken2_host_removal:mem_mb=10240
+  - kraken2_host_removal:time="03:00:00"
+  - bowtie2_host_removal:mem_mb=10240
+  - bowtie2_host_removal:time="08:00:00"
   - kaiju:mem_mb=10240
   - kaiju:time="10:00:00"
   - kraken2:mem_mb=10240

profiles/pdc_dardel/config.yaml

@@ -0,0 +1,101 @@
+# Snakemake profile for StaG for use on PDC Dardel
+# Fredrik Boulund 2024
+
+#######################################
+# General Snakemake settings
+#######################################
+configfile: config/config.yaml
+keep-going: False
+rerun-incomplete: True
+printshellcmds: True
+scheduler: greedy
+conda-frontend: conda
+use-conda: True
+conda-prefix: /cfs/klemming/projects/supr/naiss2023-6-372/sing/smk_conda_envs
+#use-singularity: True
+#singularity-prefix: /cfs/klemming/projects/supr/naiss2023-6-372/sing
+#singularity-args: "-B /cfs/klemming/projects/supr/naiss2023-6-372/"
+groups: humann=group0
+group-components: group0=4
+
+#######################################
+# Cluster settings
+#######################################
+cluster:
+  mkdir -p slurm_logs/{rule} &&
+  sbatch
+    --account={resources.account}
+    --partition={resources.partition}
+    --nodes=1
+    --ntasks-per-node=1
+    --cpus-per-task={threads}
+    --mem={resources.mem_mb}
+    --time={resources.time}
+    --job-name={rule}-{wildcards}
+    --output=slurm_logs/{rule}/{rule}-{wildcards}-%j.out
+    --parsable
+cluster-cancel: scancel
+cluster-cancel-nargs: 50
+cluster-status: status-sacct.sh
+restart-times: 0
+max-jobs-per-second: 10
+max-status-checks-per-second: 1
+latency-wait: 60
+jobs: 500
+
+#######################################
+# Resource settings
+#######################################
+local-cores: 2
+default-resources:
+  - threads=2
+  - account=naiss2023-5-552
+  - partition=shared
+  - time="01:00:00"
+  - mem_mb="1G"
+set-threads:
+  - fastp=10
+  - kraken2_host_removal=20
+  - bowtie2_host_removal=10
+  - bt2_sort_bam_files=4
+  - bbcountunique=4
+  - sketch=8
+  - kaiju=32
+  - kraken2=16
+  - krakenuniq=24
+  - metaphlan=12
+  - strainphlan=8
+  - bracken=2
+  - humann=30
+  - bbmap=16
+  - bowtie2=16
+  - assembly=20
+  - consolidate_bins=20
+  - blobology=20
+set-resources:
+  - fastp:mem_mb=4G
+  - fastp:time=02:00:00
+  - kraken2_host_removal:mem_mb=8G
+  - kraken2_host_removal:time=02:00:00
+  - bowtie2_host_removal:mem_mb=8G
+  - bowtie2_host_removal:time=06:00:00
+  - bt2_sort_bam_files:mem_mb=30G
+  - bt2_sort_bam_files:time=01:00:00
+  - kaiju:mem_mb=8G
+  - kaiju:time=10:00:00
+  - kraken2:mem_mb=160G
+  - kraken2:time=04:00:00
+  - krakenuniq:mem_mb=100G
+  - krakenuniq:time=06:00:00
+  - metaphlan:mem_mb=30G
+  - metaphlan:time=04:00:00
+  - strainphlan:mem_mb=10G
+  - strainphlan:time=02:00:00
+  - humann:partition=memory
+  - humann:mem_mb=850G
+  - humann:time=23:00:00
+  - humann:tmpdir=/cfs/klemming/scratch/b/boulund/
+  - bbmap:mem_mb=16G
+  - bbmap:time=04:00:00
+  - bowtie2:mem_mb=16G
+  - bowtie2:time=04:00:00

profiles/pdc_dardel/status-sacct.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+# Check status of Slurm job
+
+jobid="$1"
+
+if [[ "$jobid" == Submitted ]]
+then
+  echo smk-simple-slurm: Invalid job ID: "$jobid" >&2
+  echo smk-simple-slurm: Did you remember to add the flag --parsable to your sbatch call? >&2
+  exit 1
+fi
+
+output=`sacct -j "$jobid" --format State --noheader | head -n 1 | awk '{print $1}'`
+
+if [[ $output =~ ^(COMPLETED).* ]]
+then
+  echo success
+elif [[ $output =~ ^(RUNNING|PENDING|COMPLETING|CONFIGURING|SUSPENDED).* ]]
+then
+  echo running
+else
+  echo failed
+fi

workflow/Snakefile

@@ -21,8 +21,7 @@ from scripts.common import UserMessages, SampleSheet
 
 user_messages = UserMessages()
 
-stag_version = "0.7.0"
-singularity_branch_tag = "-master"  # Replace with "-master" before publishing new version
+stag_version = "0.7.1"
 
 configfile: "config/config.yaml"
 report: "report/workflow.rst"
@@ -31,17 +30,20 @@ citations = {publications["StaG"], publications["Snakemake"]}
 INPUTDIR = Path(config["inputdir"])
 OUTDIR = Path(config["outdir"])
 LOGDIR = Path(config["logdir"])
-DBDIR = Path(config["dbdir"])
 all_outputs = []
 
 if config["samplesheet"]:
-    samplesheet = SampleSheet(config["samplesheet"], keep_local=config["keep_local"], endpoint_url=config["s3_endpoint_url"])
+    samplesheet = SampleSheet(
+            samplesheet=config["samplesheet"],
+            keep_local=config["keep_local"],
+            endpoint_url=config["s3_endpoint_url"],
+    )
     SAMPLES = samplesheet.samples
     INPUT_read1 = lambda w: samplesheet.sample_info[w.sample]["read1"]
     INPUT_read2 = lambda w: samplesheet.sample_info[w.sample]["read2"]
 else:
     SAMPLES = set(glob_wildcards(INPUTDIR/config["input_fn_pattern"]).sample)
-    INPUT_read1 = INPUTDIR/config["input_fn_pattern"].format(sample="{sample}", readpair="1"),
+    INPUT_read1 = INPUTDIR/config["input_fn_pattern"].format(sample="{sample}", readpair="1")
     INPUT_read2 = INPUTDIR/config["input_fn_pattern"].format(sample="{sample}", readpair="2")
 
 onstart:
@@ -60,7 +62,8 @@ onstart:
         if config["samplesheet"]:
             print(f"Found these samples in '{config['samplesheet']}':\n{SAMPLES}")
         else:
-            print(f"Found these samples in '{config['inputdir']}' using input filename pattern '{config['input_fn_pattern']}':\n{SAMPLES}")
+            print(f"Found these samples in '{config['inputdir']}' using "
+                   "input filename pattern '{config['input_fn_pattern']}':\n{SAMPLES}")
 
 
 #############################
@@ -100,10 +103,6 @@ include: "rules/functional_profiling/humann.smk"
 include: "rules/mappers/bbmap.smk"
 include: "rules/mappers/bowtie2.smk"
 
-#############################
-# Assembly
-#############################
-
 #############################
 # MultiQC
 #############################

workflow/envs/humann.yaml

@@ -5,4 +5,4 @@ channels:
     - bioconda
     - defaults
 dependencies:
-    - humann=3.7
+    - humann=3.9

workflow/envs/krakenuniq.yaml

@@ -4,4 +4,4 @@ channels:
     - bioconda
     - defaults
 dependencies:
-    - krakenuniq =1.0.3
+    - krakenuniq =1.0.4

workflow/envs/metaphlan.yaml

@@ -4,5 +4,5 @@ channels:
     - bioconda
     - defaults
 dependencies:
-    - metaphlan =4.0.6
+    - metaphlan =4.1.0
     - krona =2.8.1

workflow/envs/stag-mwc.yaml

@@ -5,21 +5,20 @@ channels:
     - defaults
 dependencies:
     - python =3.10.9
-    - fastp =0.23.2
-    - bbmap =39.01
-    - kaiju =1.9.2
-    - kraken2 =2.1.2
-    - bracken =2.8
+    - bbmap =39.06
+    - bracken =2.9
+    - fastp =0.23.4
+    - kaiju =1.10.1
+    - kraken2 =2.1.3
     - krona =2.8.1
     - matplotlib =3.7.1
-    - multiqc =1.14
+    - multiqc =1.21
     - pandas =2.0.0
+    - pigz =2.6
+    - sambamba =1.0
+    - samtools =1.19.2
     - seaborn =0.12.2
     - subread =2.0.3
-    - sambamba =1.0
-    - samtools =1.6
-    - groot =1.1.2
-    - pigz =2.6
     - pip
     - pip:
         - fastcluster

workflow/rules/functional_profiling/humann.smk

@@ -48,7 +48,7 @@ rule humann:
     conda:
         "../../envs/humann.yaml"
     container:
-        "oras://ghcr.io/ctmrbio/stag-mwc:biobakery"+singularity_branch_tag
+        config["containers"]["humann"]
     threads: 20
     params:
         outdir=f"{OUTDIR}/humann/",
@@ -96,7 +96,7 @@ rule normalize_humann_tables:
     conda:
         "../../envs/humann.yaml"
     container:
-        "oras://ghcr.io/ctmrbio/stag-mwc:biobakery"+singularity_branch_tag
+        config["containers"]["humann"]
     threads: 1
     params:
         method=h_config["norm_method"],
@@ -143,7 +143,7 @@ rule humann_join_tables:
     conda:
         "../../envs/humann.yaml"
     container:
-        "oras://ghcr.io/ctmrbio/stag-mwc:biobakery"+singularity_branch_tag
+        config["containers"]["humann"]
     threads: 
         1
     params:
@@ -174,3 +174,4 @@ rule humann_join_tables:
             >> {log.stdout} \
             2>> {log.stderr}
         """
+