Misc output cleanup

CHANGELOG.md

@@ -15,12 +15,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Annotation input is now provided via the samplesheet instead of `--annotation_set` (see [usage.md](docs/usage.md))
 - Added support for PacBio k-mer directories provided as `tar.gz` archives
 - Updated output structure (see [output.md](docs/output.md))
+- Removed the `--write_to_portal` and `--genome_notes_api` options as the
+  Genome Note platform is now expected to _pull_ data from the Genome
+  After-Party.
 
 ### Parameters
 
-| Old parameter    | New parameter |
-| ---------------- | ------------- |
-| --annotation_set |               |
+| Old parameter      | New parameter |
+| ------------------ | ------------- |
+| --annotation_set   |               |
+| --write_to_portal  |               |
+| --genome_notes_api |               |
 
 > **NB:** Parameter has been **updated** if both old and new parameter information is present. </br> **NB:** Parameter has been **added** if just the new parameter information is present. </br> **NB:** Parameter has been **removed** if new parameter information isn't present.
 

conf/modules.config

@@ -96,7 +96,16 @@ process {
         publishDir = [
             path: { "${params.outdir}/genome_stats/${meta.datatype}/${meta.sample}/merqury" },
             mode: params.publish_dir_mode,
-            saveAs: { filename -> filename.equals('versions.yml') ? null : filename.replaceAll(/_only.bed$/, '.only.bed') },
+            saveAs: { filename -> filename.equals('versions.yml') || filename.endsWith("_only.bed") ? null : filename },
+        ]
+    }
+
+    withName: BGZIPTABIX {
+        ext.prefix = { "${input.baseName.replaceFirst(/_only$/, '.only')}" }
+        publishDir = [
+            path: { "${params.outdir}/genome_stats/${meta.datatype}/${meta.sample}/merqury" },
+            mode: params.publish_dir_mode,
+            saveAs: { filename -> filename.equals('versions.yml') ? null : filename },
         ]
     }
 

conf/test.config

@@ -47,9 +47,7 @@ params {
     biosample_hic              = 'SAMEA112198479'
     biosample_rna              = 'SAMEA112232914'
 
-    // Genome Notes Portal
-    write_to_portal            = false
-    genome_notes_api           = "https://notes-staging.tol.sanger.ac.uk/api/v1"
+    // Genome Note document
     note_template              = "${projectDir}/assets/genome_note_template.docx"
 
     // Contact Map Options

conf/test_full.config

@@ -37,9 +37,7 @@ params {
     biosample_hic              = 'SAMEA7519968'
     biosample_rna              = null
 
-    // Genome Notes Portal
-    write_to_portal            = false
-    genome_notes_api           = "https://notes-staging.tol.sanger.ac.uk/api/v1"
+    // Genome Note document
     note_template              = "${projectDir}/assets/genome_note_template.docx"
 
     // HiGlass Options

docs/usage.md

@@ -266,23 +266,11 @@ NXF_OPTS='-Xms1g -Xmx4g'
 
 ## For internal Sanger use only
 
-If you wish to run the optional step that writes genome metadata key value-pairs to a genome notes databases you will need to set the parameter "write_to_portal" to true and provide the base url for the REST API that writes to the database.
-
-```bash
-   --write_to_portal_db 'true'
-   --genome_notes_api '[URL for Genome Notes Portal API]'
-```
-
-You will also need to set a nextflow secret to store the API key belonging to your user.
-
-```bash
-  nextflow secrets set TOL_API_KEY '[API key]'
-```
-
 If you want to populate a genome notes template file with the key-value pairs generated by this pipeline you will need to pass the path to the template file as the "note_template" parameter. Templates may be either docx or xml format.
+See the provided `assets/genome_note_template.docx` for an example
 
 ```bash
-   --note_template '[URL for Genome Notes Portal API]'
+   --note_template '/path/to/template'
 ```
 
 If you wish to run the optional step that writes the .mcool and .genome files produced by the contact_maps subworkflow to a kubernetes hosted higlass server you will need to set the parameter "upload_higlass_data" to true and provide the configuration information for the kubernetes deployment.

modules.json

@@ -23,7 +23,8 @@
                     "blobtk/plot": {
                         "branch": "master",
                         "git_sha": "02616d1725cc2f97670e25d14d7d1c29d412dffc",
-                        "installed_by": ["modules"]
+                        "installed_by": ["modules"],
+                        "patch": "modules/nf-core/blobtk/plot/blobtk-plot.diff"
                     },
                     "busco/busco": {
                         "branch": "master",
@@ -154,6 +155,11 @@
                         "git_sha": "ac0fc48df659b3a6cca0fc2b0346af41162caabb",
                         "installed_by": ["modules"]
                     },
+                    "bgziptabix": {
+                        "branch": "main",
+                        "git_sha": "d87a6e1f034a64bfec4befec1d5d9f398c4239ec",
+                        "installed_by": ["modules"]
+                    },
                     "restructurebuscodir": {
                         "branch": "main",
                         "git_sha": "8dac7d789fb5ec033c38e69c2553e71cae0dd4c3",

modules/local/createtable.nf

@@ -39,7 +39,7 @@ process CREATETABLE {
         ${mco} \\
         ${hic} \\
         ${fst} \\
-        --outcsv ${prefix}.csv
+        --outcsv ${prefix}.template_parameters.csv
 
     cat <<-END_VERSIONS > versions.yml
     "${task.process}":

modules/sanger-tol/bgziptabix/environment.yml

@@ -0,0 +1,8 @@
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/environment-schema.json
+channels:
+  - conda-forge
+  - bioconda
+
+dependencies:
+  - bioconda::htslib=1.21

modules/sanger-tol/bgziptabix/main.nf

@@ -0,0 +1,45 @@
+process BGZIPTABIX {
+    tag "${meta.id}"
+    label 'process_single'
+
+    conda "${moduleDir}/environment.yml"
+    container "${workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container
+        ? 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/92/92859404d861ae01afb87e2b789aebc71c0ab546397af890c7df74e4ee22c8dd/data'
+        : 'community.wave.seqera.io/library/htslib:1.21--ff8e28a189fbecaa'}"
+
+    input:
+    tuple val(meta), path(input), val(max_seq_length)
+    tuple val(column_numbers), val(header_lines), val(extension)
+
+    output:
+    tuple val(meta), path("*.gz"), path("*.gzi"), emit: gz_index
+    tuple val(meta), path("*.tbi"), emit: tbi, optional: true
+    tuple val(meta), path("*.csi"), emit: csi, optional: true
+    tuple val("${task.process}"), val('bgzip'), eval("bgzip --version | sed '1!d;s/.* //'"), topic: versions, emit: versions_bgzip
+    tuple val("${task.process}"), val('tabix'), eval("tabix -h 2>&1 | grep -oP 'Version:\\s*\\K[^\\s]+'"), topic: versions, emit: versions_tabix
+
+    when:
+    task.ext.when == null || task.ext.when
+
+    script:
+    def args = task.ext.args ?: ''
+    def args2 = task.ext.args2 ?: ''
+    def prefix = task.ext.prefix ?: "${meta.id}"
+    def input_data = column_numbers ? "<(cut -f${column_numbers} ${input} | tail -n+${header_lines + 1})" : input
+    extension ?= input.extension
+    """
+    bgzip --threads ${task.cpus} --index ${args} ${input_data} --output ${prefix}.${extension}.gz
+    [[ ${max_seq_length} -lt \$(( 2 ** 29 )) ]] && tabix --threads ${task.cpus} ${args2} ${prefix}.${extension}.gz
+    [[ ${max_seq_length} -lt \$(( 2 ** 32 )) ]] && tabix --threads ${task.cpus} --csi ${args2} ${prefix}.${extension}.gz
+    """
+
+    stub:
+    def prefix = task.ext.prefix ?: "${meta.id}"
+    extension ?= input.extension
+    """
+    echo "" | gzip > ${prefix}.${extension}.gz
+    touch ${prefix}.${extension}.gz.gzi
+    touch ${prefix}.${extension}.gz.tbi
+    touch ${prefix}.${extension}.gz.csi
+    """
+}

modules/sanger-tol/bgziptabix/meta.yml

@@ -0,0 +1,142 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/meta-schema.json
+name: "bgziptabix"
+description: bgzip a sorted tab-delimited genome file and then create tabix
+  indexes in TBI and CSI format when the coordinate / sequence length is within
+  the supported range of each index format.
+keywords:
+  - bgzip
+  - compress
+  - index
+  - tabix
+  - vcf
+tools:
+  - bgzip:
+      description: |
+        Bgzip compresses or decompresses files in a similar manner to, and compatible with, gzip.
+      homepage: https://www.htslib.org/doc/tabix.html
+      documentation: http://www.htslib.org/doc/bgzip.html
+      doi: 10.1093/bioinformatics/btp352
+      licence: ["MIT"]
+      identifier: biotools:tabix
+  - tabix:
+      description: Generic indexer for TAB-delimited genome position files.
+      homepage: https://www.htslib.org/doc/tabix.html
+      documentation: https://www.htslib.org/doc/tabix.1.html
+      doi: 10.1093/bioinformatics/btq671
+      licence: ["MIT"]
+      identifier: biotools:tabix
+input:
+  - - meta:
+        type: map
+        description: |
+          Groovy Map containing sample information
+          e.g. [ id:'test', single_end:false ]
+    - input:
+        type: file
+        description: Sorted tab-delimited genome file
+        ontologies: []
+    - max_seq_length:
+        type: integer
+        description: Maximum length (coordinate) of a sequence in this genome (TSV)
+  - - column_numbers:
+        type: string
+        description: |
+          Optional. Columns to extract from the input file, in a format understood by `cut`
+          e.g. '1-3,5'
+    - header_lines:
+        type: integer
+        description: |
+          Number of header lines to skip from the input file. Only considered when column_numbers
+    - extension:
+        type: string
+        description: |
+          Extension to name the output file, in case it differs from the input
+          file - for instance, turning a TSV into BED
+output:
+  gz_index:
+    - - meta:
+          type: map
+          description: |
+            Groovy Map containing sample information
+            e.g. [ id:'test', single_end:false ]
+      - "*.gz":
+          type: file
+          description: bgzipped tab-delimited genome file
+          pattern: "*.gz"
+          ontologies:
+            - edam: http://edamontology.org/format_3989
+      - "*.gzi":
+          type: file
+          description: bgzip index file (GZI format)
+          pattern: "*.gzi"
+          ontologies: []
+  tbi:
+    - - meta:
+          type: map
+          description: |
+            Groovy Map containing sample information
+            e.g. [ id:'test', single_end:false ]
+      - "*.tbi":
+          type: file
+          description: Tabix index file (TBI format)
+          pattern: "*.tbi"
+          ontologies:
+            - edam: http://edamontology.org/format_3616
+  csi:
+    - - meta:
+          type: map
+          description: |
+            Groovy Map containing sample information
+            e.g. [ id:'test', single_end:false ]
+      - "*.csi":
+          type: file
+          description: Tabix index file (CSI format)
+          pattern: "*.csi"
+          ontologies: []
+  versions_bgzip:
+    - - ${task.process}:
+          type: string
+          description: The process the versions were collected from
+      - bgzip:
+          type: string
+          description: The tool name
+      - bgzip --version | sed '1!d;s/.* //':
+          type: eval
+          description: The expression to obtain the version of the tool
+  versions_tabix:
+    - - ${task.process}:
+          type: string
+          description: The process the versions were collected from
+      - tabix:
+          type: string
+          description: The tool name
+      - tabix -h 2>&1 | grep -oP 'Version:\s*\K[^\s]+':
+          type: eval
+          description: The expression to obtain the version of the tool
+topics:
+  versions:
+    - - ${task.process}:
+          type: string
+          description: The process the versions were collected from
+      - bgzip:
+          type: string
+          description: The tool name
+      - bgzip --version | sed '1!d;s/.* //':
+          type: eval
+          description: The expression to obtain the version of the tool
+    - - ${task.process}:
+          type: string
+          description: The process the versions were collected from
+      - tabix:
+          type: string
+          description: The tool name
+      - tabix -h 2>&1 | grep -oP 'Version:\s*\K[^\s]+':
+          type: eval
+          description: The expression to obtain the version of the tool
+authors:
+  - "@maxulysse"
+  - "@DLBPointon"
+  - "@muffato"
+maintainers:
+  - "@DLBPointon"
+  - "@muffato"