Improve fai again and address review comments

CHANGELOG.md

@@ -21,6 +21,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - [#137](https://github.com/nf-core/bamtofastq/pull/137) - Recover help message
 - [#141](https://github.com/nf-core/bamtofastq/pull/141) - Update local sbwf and module to adhere to directory structure, update nf-test plugins
 - [#142](https://github.com/nf-core/bamtofastq/pull/142) - Add tests to local subworkflows, improve `.fai` generation
+- [#143](https://github.com/nf-core/bamtofastq/pull/143) - Improve `fai` again and address review comments
 
 ### `Dependencies`
 

assets/schema_input.json

@@ -31,7 +31,7 @@
                 "type": "string",
                 "errorMessage": "Input filetypes must either be 'bam' or 'cram'",
                 "meta": ["filetype"],
-                "enums": ["bam", "cram"]
+                "enum": ["bam", "cram"]
             }
         },
         "required": ["sample_id", "mapped", "file_type"]

modules.json

@@ -71,7 +71,7 @@
                 "nf-core": {
                     "utils_nextflow_pipeline": {
                         "branch": "master",
-                        "git_sha": "05954dab2ff481bcb999f24455da29a5828af08d",
+                        "git_sha": "1a545fcbd762911c21a64ced3dbef99b2b51ac75",
                         "installed_by": ["subworkflows"]
                     },
                     "utils_nfcore_pipeline": {
@@ -81,7 +81,7 @@
                     },
                     "utils_nfschema_plugin": {
                         "branch": "master",
-                        "git_sha": "fdc08b8b1ae74f56686ce21f7ea11ad11990ce57",
+                        "git_sha": "ce3424257de288c0ca25f097a68d3a289da23f12",
                         "installed_by": ["subworkflows"]
                     }
                 }

modules/local/checkpairedend/meta.yaml

@@ -56,7 +56,7 @@ output:
       - samtools:
           type: string
           description: The tool name
-      - samtools --version 2>&1 | sed -n '1p' | awk '{print $2}':
+      - samtools version | sed '1!d;s/.* //':
           type: eval
           description: The expression to obtain the version of samtools
 
@@ -68,7 +68,7 @@ topics:
       - samtools:
           type: string
           description: The tool name
-      - samtools --version 2>&1 | sed -n '1p' | awk '{print $2}':
+      - samtools version | sed '1!d;s/.* //':
           type: eval
           description: The expression to obtain the version of samtools
 

subworkflows/local/prepare_indices/meta.yaml → subworkflows/local/prepare_indices/meta.yml

@@ -36,10 +36,24 @@ input:
         Optional channel emitting reference FASTA files (required for CRAM
         indexing when reference is not embedded in header).
       structure:
-        - fasta:
-            type: file
-            description: Reference genome FASTA file
-            pattern: "*.{fa,fna,fasta}"
+        - fasta_fai:
+            type: channel
+            description: |
+              Channel emitting tuples: [meta, fasta, fai]. `meta.index` should be true if a
+              pre-built `.fai` is supplied, false to rebuild it via `samtools faidx`. Pass
+              `[ [id:'none', index:false], [], [] ]` if no reference is needed.
+            structure:
+              - meta:
+                  type: map
+                  description: Metadata map with reference id and `index` flag
+              - fasta:
+                  type: file
+                  description: Reference genome FASTA file
+                  pattern: "*.{fa,fna,fasta}"
+              - fai:
+                  type: file
+                  description: Reference genome FASTA index file (optional, rebuilt if absent)
+                  pattern: "*.fai"
 output:
   - ch_input_indexed:
       type: channel

subworkflows/local/utils_nfcore_bamtofastq_pipeline/main.nf

@@ -8,14 +8,13 @@
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 */
 
-include { UTILS_NFSCHEMA_PLUGIN     } from '../../nf-core/utils_nfschema_plugin'
-include { paramsSummaryMap          } from 'plugin/nf-schema'
-include { samplesheetToList         } from 'plugin/nf-schema'
-include { paramsHelp                } from 'plugin/nf-schema'
-include { completionEmail           } from '../../nf-core/utils_nfcore_pipeline'
-include { completionSummary         } from '../../nf-core/utils_nfcore_pipeline'
-include { UTILS_NFCORE_PIPELINE     } from '../../nf-core/utils_nfcore_pipeline'
-include { UTILS_NEXTFLOW_PIPELINE   } from '../../nf-core/utils_nextflow_pipeline'
+include { UTILS_NFSCHEMA_PLUGIN   } from '../../nf-core/utils_nfschema_plugin'
+include { paramsSummaryMap        } from 'plugin/nf-schema'
+include { samplesheetToList       } from 'plugin/nf-schema'
+include { completionEmail         } from '../../nf-core/utils_nfcore_pipeline'
+include { completionSummary       } from '../../nf-core/utils_nfcore_pipeline'
+include { UTILS_NFCORE_PIPELINE   } from '../../nf-core/utils_nfcore_pipeline'
+include { UTILS_NEXTFLOW_PIPELINE } from '../../nf-core/utils_nextflow_pipeline'
 
 /*
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -88,6 +87,7 @@ workflow PIPELINE_INITIALISATION {
         before_text,
         after_text,
         command,
+        null,
     )
 
     //
@@ -136,7 +136,7 @@ workflow PIPELINE_COMPLETION {
     plaintext_email // boolean: Send plain-text email instead of HTML
     outdir //    path: Path to output directory where results will be published
     monochrome_logs // boolean: Disable ANSI colour codes in log output
-    multiqc_report  //  string: Path to MultiQC report
+    multiqc_report //  string: Path to MultiQC report
 
     main:
     summary_params = paramsSummaryMap(workflow, parameters_schema: "nextflow_schema.json")
@@ -159,11 +159,10 @@ workflow PIPELINE_COMPLETION {
         }
 
         completionSummary(monochrome_logs)
-
     }
 
     workflow.onError {
-        log.error "Pipeline failed. Please refer to troubleshooting docs for common issues: https://nf-co.re/docs/running/troubleshooting"
+        log.error("Pipeline failed. Please refer to troubleshooting docs for common issues: https://nf-co.re/docs/running/troubleshooting")
     }
 }
 
@@ -207,26 +206,27 @@ def genomeExistsError() {
 // Generate methods description for MultiQC
 //
 def toolCitationText() {
-    // TODO nf-core: Optionally add in-text citation tools to this list.
     // Can use ternary operators to dynamically construct based conditions, e.g. params["run_xyz"] ? "Tool (Foo et al. 2023)" : "",
     // Uncomment function in methodsDescriptionText to render in MultiQC report
     def citation_text = [
         "Tools used in the workflow included:",
         "FastQC (Andrews 2010),",
-        "MultiQC (Ewels et al. 2016)",
-        ".",
+        "fastq_utils (Fonseca & Manning 2023),",
+        "MultiQC (Ewels et al. 2016),",
+        "SAMtools (Danecek et al. 2021)",
     ].join(' ').trim()
 
     return citation_text
 }
 
 def toolBibliographyText() {
-    // TODO nf-core: Optionally add bibliographic entries to this list.
     // Can use ternary operators to dynamically construct based conditions, e.g. params["run_xyz"] ? "<li>Author (2023) Pub name, Journal, DOI</li>" : "",
     // Uncomment function in methodsDescriptionText to render in MultiQC report
     def reference_text = [
         "<li>Andrews S, (2010) FastQC, URL: https://www.bioinformatics.babraham.ac.uk/projects/fastqc/).</li>",
-        "<li>Ewels, P., Magnusson, M., Lundin, S., & Käller, M. (2016). MultiQC: summarize analysis results for multiple tools and samples in a single report. Bioinformatics , 32(19), 3047–3048. doi: /10.1093/bioinformatics/btw354</li>",
+        "<li>Nuno Fonseca & Jonathan Manning (2023) fastq_utils v0.25.2, Zenodo. https://doi.org/10.5281/zenodo.7755574</li>",
+        "<li>Ewels, P., Magnusson, M., Lundin, S., & Käller, M. (2016). MultiQC: summarize analysis results for multiple tools and samples in a single report. Bioinformatics , 32(19), 3047–3048. doi: 10.1093/bioinformatics/btw354</li>",
+        "<li>Danecek, P., Bonfield, J. K., Liddle, J., Marshall, J., Ohan, V., Pollard, M. O., Whitwham, A., Keane, T., McCarthy, S. A., Davies, R. D., & Li, H. (2021). Twelve years of SAMtools and BCFtools. GigaScience, 10(2), giab008. doi: 10.1093/gigascience/giab008</li>",
     ].join(' ').trim()
 
     return reference_text
@@ -256,12 +256,8 @@ def methodsDescriptionText(mqc_methods_yaml) {
     meta["nodoi_text"] = meta.manifest_map.doi ? "" : "<li>If available, make sure to update the text to include the Zenodo DOI of version of the pipeline used. </li>"
 
     // Tool references
-    meta["tool_citations"] = ""
-    meta["tool_bibliography"] = ""
-
-    // TODO nf-core: Only uncomment below if logic in toolCitationText/toolBibliographyText has been filled!
-    // meta["tool_citations"] = toolCitationText().replaceAll(", \\.", ".").replaceAll("\\. \\.", ".").replaceAll(", \\.", ".")
-    // meta["tool_bibliography"] = toolBibliographyText()
+    meta["tool_citations"] = toolCitationText().replaceAll(", \\.", ".").replaceAll("\\. \\.", ".").replaceAll(", \\.", ".")
+    meta["tool_bibliography"] = toolBibliographyText()
 
 
     def methods_text = mqc_methods_yaml.text

tests/fai.nf.test

@@ -0,0 +1,50 @@
+nextflow_pipeline {
+
+    name "Test pipeline"
+    script "../main.nf"
+    tag "pipeline"
+    tag "pipeline_bamtofastq"
+    tag "cpu"
+
+    test("-profile test | with fasta and fasta_fai") {
+
+        when {
+            params {
+                outdir    = "$outputDir"
+                fasta     = 'https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/homo_sapiens/genome/genome.fasta'
+                fasta_fai = 'https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/homo_sapiens/genome/genome.fasta.fai'
+            }
+        }
+
+        then {
+            // stable_name: All files + folders in ${params.outdir}/ with a stable name
+            def stable_name = getAllFilesFromDir(params.outdir, relative: true, includeDir: true, ignore: ['pipeline_info/*.{html,json,txt}'])
+            // stable_path: All files in ${params.outdir}/ with stable content
+            def stable_path = getAllFilesFromDir(params.outdir, ignoreFile: 'tests/.nftignore')
+            // bam_files: All bam files
+            def bam_files  = getAllFilesFromDir(params.outdir, include: ['**/*.bam'])
+            // cram_files: All cram files
+            def cram_files  = getAllFilesFromDir(params.outdir, include: ['**/*.cram'])
+            // Fasta file for cram verification with nft-bam
+            def fasta_base  = 'https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/'
+            def fasta       = fasta_base + 'genomics/homo_sapiens/genome/genome.fasta'
+            assert workflow.success
+            assertAll(
+                { assert snapshot(
+                    // Number of successful tasks
+                    workflow.trace.succeeded().size(),
+                    // pipeline versions.yml file for multiqc from which Nextflow and pipeline versions are removed (all from the workflow key)
+                    removeFromYamlMap("${outputDir}/pipeline_info/nf_core_bamtofastq_software_mqc_versions.yml", "Workflow"),
+                    // All stable path name, with a relative path
+                    stable_name.isEmpty() ? 'No stable paths' : stable_name,
+                    // All files with stable contents
+                    stable_path.isEmpty() ? 'No stable content' : stable_path,
+                    // All cram files
+                    bam_files.isEmpty() ? 'No BAM files' : bam_files.collect { file -> file.getName() + ":md5," +  bam(file.toString()).readsMD5 },
+                    // All cram files
+                    cram_files.isEmpty() ? 'No CRAM files' : cram_files.collect { file -> file.getName() + ":md5," +  cram(file.toString(), fasta).readsMD5 },
+                ).match() }
+            )
+        }
+    }
+}