Handle extra header for reads

.github/workflows/download_pipeline.yml

@@ -32,7 +32,7 @@ jobs:
         run: |
           echo "REPO_LOWERCASE=${GITHUB_REPOSITORY,,}" >> "$GITHUB_OUTPUT"
           echo "REPOTITLE_LOWERCASE=$(basename ${GITHUB_REPOSITORY,,})" >> "$GITHUB_OUTPUT"
-          echo "REPO_BRANCH=${{ github.event.inputs.testbranch || 'dev' }}" >> "$GITHUB_OUTPUT"
+          echo "REPO_BRANCH=${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || github.event.inputs.testbranch || 'dev' }}" >> "$GITHUB_OUTPUT"
 
   download:
     runs-on: ubuntu-latest

.nf-core.yml

@@ -47,4 +47,4 @@ template:
     - igenomes
     - multiqc
     - fastqc
-  version: 2.0.3
+  version: 2.1.0-dev

CHANGELOG.md

@@ -3,6 +3,12 @@
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [[v2.1.0-dev](https://github.com/sanger-tol/readmapping/releases/tag/2.1.0-dev)] - 2.1.0-dev - [release-date]
+
+### Enhancements & fixes
+
+- Support extra_header at read-file level to provide `@RG` and `@PG` lines (see [usage](docs/usage.md))
+
 ## [[v2.0.3](https://github.com/sanger-tol/readmapping/releases/tag/2.0.3)] - Hungarian Horntail (patch 3) - [2026-05-15]
 
 ### Enhancements & fixes

CITATION.cff

@@ -57,7 +57,7 @@ license: MIT
 message: If you use this software, please cite it using the metadata from this file
   and all references from CITATIONS.md .
 repository-code: https://github.com/sanger-tol/readmapping
-title: sanger-tol/readmapping v2.0.3 - Hungarian Horntail (patch 3)
+title: sanger-tol/readmapping 2.1.0-dev - 2.1.0-dev
 type: software
 url: https://pipelines.tol.sanger.ac.uk/readmapping
-version: 2.0.3
+version: 2.1.0-dev

assets/samplesheet.csv

@@ -1,8 +1,8 @@
-specimen,run,datatype,datafile,library,barcode
-SAMEA7524438,ERR6688599,illumina,https://tolit.cog.sanger.ac.uk/test-data/Meles_meles/genomic_data/mMelMel1/illumina/31231_3_1.subset.cram,,
-SAMEA7524439,ERR6688600,illumina,https://tolit.cog.sanger.ac.uk/test-data/Meles_meles/genomic_data/mMelMel2/illumina/31231_4_1.subset.fastq.gz,,
-SAMEA7524440,ERR6688402,hic,https://tolit.cog.sanger.ac.uk/test-data/Meles_meles/genomic_data/mMelMel3/hic/35528_2_1.subset.cram,,
-SAMEA7524440,PAE35587,ont,https://tolit.cog.sanger.ac.uk/test-data/Meles_meles/genomic_data/mMelMel3/ont/PAE35587_pass_1f1f0707_115.subset.fastq.gz,,
-SAMEA7524440,PAE35587_test,ont,https://tolit.cog.sanger.ac.uk/test-data/Meles_meles/genomic_data/mMelMel3/ont/PAE35587_pass_1f1f0707_115.subset.addrg.bam,,
-SAMEA7524440,ERR6939248,pacbio,https://tolit.cog.sanger.ac.uk/test-data/Meles_meles/genomic_data/mMelMel3/pacbio/m64094_200910_173211.ccs.bc1022_BAK8B_OA--bc1022_BAK8B_OA.subset.bam,,
-SAMEA7524440,ERR6939249,pacbio,https://tolit.cog.sanger.ac.uk/test-data/Meles_meles/genomic_data/mMelMel3/pacbio/m64094_200911_174739.ccs.bc1022_BAK8B_OA--bc1022_BAK8B_OA.subset.fastq.gz,,bc1022
+specimen,run,datatype,datafile,library,barcode,extra_header
+SAMEA7524438,ERR6688599,illumina,https://tolit.cog.sanger.ac.uk/test-data/Meles_meles/genomic_data/mMelMel1/illumina/31231_3_1.subset.cram,,,
+SAMEA7524439,ERR6688600,illumina,https://tolit.cog.sanger.ac.uk/test-data/Meles_meles/genomic_data/mMelMel2/illumina/31231_4_1.subset.fastq.gz,,,
+SAMEA7524440,ERR6688402,hic,https://tolit.cog.sanger.ac.uk/test-data/Meles_meles/genomic_data/mMelMel3/hic/35528_2_1.subset.cram,,,https://tolit.cog.sanger.ac.uk/test-data/Meles_meles/genomic_data/mMelMel3/hic/35528_2_1.subset.header_no_rg.sam
+SAMEA7524440,PAE35587,ont,https://tolit.cog.sanger.ac.uk/test-data/Meles_meles/genomic_data/mMelMel3/ont/PAE35587_pass_1f1f0707_115.subset.fastq.gz,,,
+SAMEA7524440,PAE35587_test,ont,https://tolit.cog.sanger.ac.uk/test-data/Meles_meles/genomic_data/mMelMel3/ont/PAE35587_pass_1f1f0707_115.subset.addrg.bam,,,
+SAMEA7524440,ERR6939248,pacbio,https://tolit.cog.sanger.ac.uk/test-data/Meles_meles/genomic_data/mMelMel3/pacbio/m64094_200910_173211.ccs.bc1022_BAK8B_OA--bc1022_BAK8B_OA.subset.header_no_rg.bam,,,https://tolit.cog.sanger.ac.uk/test-data/Meles_meles/genomic_data/mMelMel3/pacbio/m64094_200910_173211.ccs.bc1022_BAK8B_OA--bc1022_BAK8B_OA.subset.header.sam
+SAMEA7524440,ERR6939249,pacbio,https://tolit.cog.sanger.ac.uk/test-data/Meles_meles/genomic_data/mMelMel3/pacbio/m64094_200911_174739.ccs.bc1022_BAK8B_OA--bc1022_BAK8B_OA.subset.fastq.gz,,bc1022,https://tolit.cog.sanger.ac.uk/test-data/Meles_meles/genomic_data/mMelMel3/pacbio/m64094_200911_174739.ccs.bc1022_BAK8B_OA--bc1022_BAK8B_OA.subset.header.sam

assets/samplesheet_long_read.csv

@@ -1,4 +1,4 @@
-specimen,run,datatype,datafile,library,barcode
-SAMEA7524440,PAE35587,ont,https://tolit.cog.sanger.ac.uk/test-data/Meles_meles/genomic_data/mMelMel3/ont/PAE35587_pass_1f1f0707_115.subset.fastq.gz,,
-SAMEA7524440,ERR6939248,pacbio,https://tolit.cog.sanger.ac.uk/test-data/Meles_meles/genomic_data/mMelMel3/pacbio/m64094_200910_173211.ccs.bc1022_BAK8B_OA--bc1022_BAK8B_OA.subset.bam,,bc1022
-SAMEA7524440,ERR6939249,pacbio,https://tolit.cog.sanger.ac.uk/test-data/Meles_meles/genomic_data/mMelMel3/pacbio/m64094_200911_174739.ccs.bc1022_BAK8B_OA--bc1022_BAK8B_OA.subset.fastq.gz,,bc1022
+specimen,run,datatype,datafile,library,barcode,extra_header
+SAMEA7524440,PAE35587,ont,https://tolit.cog.sanger.ac.uk/test-data/Meles_meles/genomic_data/mMelMel3/ont/PAE35587_pass_1f1f0707_115.subset.fastq.gz,,,
+SAMEA7524440,ERR6939248,pacbio,https://tolit.cog.sanger.ac.uk/test-data/Meles_meles/genomic_data/mMelMel3/pacbio/m64094_200910_173211.ccs.bc1022_BAK8B_OA--bc1022_BAK8B_OA.subset.bam,,bc1022,
+SAMEA7524440,ERR6939249,pacbio,https://tolit.cog.sanger.ac.uk/test-data/Meles_meles/genomic_data/mMelMel3/pacbio/m64094_200911_174739.ccs.bc1022_BAK8B_OA--bc1022_BAK8B_OA.subset.fastq.gz,,bc1022,https://tolit.cog.sanger.ac.uk/test-data/Meles_meles/genomic_data/mMelMel3/pacbio/m64094_200911_174739.ccs.bc1022_BAK8B_OA--bc1022_BAK8B_OA.subset.header.sam

assets/samplesheet_short_reads.csv

@@ -1,4 +1,4 @@
-specimen,run,datatype,datafile,library,barcode
+specimen,run,datatype,datafile,library,barcode,extra_header
 SAMEA7524438,ERR6688599,illumina,https://tolit.cog.sanger.ac.uk/test-data/Meles_meles/genomic_data/mMelMel1/illumina/31231_3_1.subset.bam,,
 SAMEA7524439,ERR6688600,illumina,https://tolit.cog.sanger.ac.uk/test-data/Meles_meles/genomic_data/mMelMel2/illumina/31231_4_1.subset.fastq.gz,,
-SAMEA7524440,ERR6688402,hic,https://tolit.cog.sanger.ac.uk/test-data/Meles_meles/genomic_data/mMelMel3/hic/35528_2_1.subset.cram,,
+SAMEA7524440,ERR6688402,hic,https://tolit.cog.sanger.ac.uk/test-data/Meles_meles/genomic_data/mMelMel3/hic/35528_2_1.subset.cram,,,https://tolit.cog.sanger.ac.uk/test-data/Meles_meles/genomic_data/mMelMel3/hic/35528_2_1.subset.header_no_rg.sam

assets/samplesheet_uli.csv

@@ -1,3 +1,3 @@
-specimen,run,datatype,datafile,library,barcode
-SAMEA114784749,ERR14209104,pacbio,https://tolit.cog.sanger.ac.uk/test-data/Acropora_cervicornis/genomic_data/m84093_241116_151316_s2.hifi_reads.bc2028.subset.bam,uli,bc2028
-SAMEA114784749,ERR14209104_test,pacbio,https://tolit.cog.sanger.ac.uk/test-data/Acropora_cervicornis/genomic_data/m84093_241116_151316_s2.hifi_reads.bc2028.subset.bam,,bc2028
+specimen,run,datatype,datafile,library,barcode,extra_header
+SAMEA114784749,ERR14209104,pacbio,https://tolit.cog.sanger.ac.uk/test-data/Acropora_cervicornis/genomic_data/m84093_241116_151316_s2.hifi_reads.bc2028.subset.bam,uli,bc2028,
+SAMEA114784749,ERR14209104_test,pacbio,https://tolit.cog.sanger.ac.uk/test-data/Acropora_cervicornis/genomic_data/m84093_241116_151316_s2.hifi_reads.bc2028.subset.bam,,bc2028,

assets/schema_input.json

@@ -45,6 +45,14 @@
                 "errorMessage": "Sample barcode cannot contain spaces",
                 "default": "",
                 "meta": ["barcode"]
+            },
+            "extra_header": {
+                "type": "string",
+                "format": "file-path",
+                "exists": true,
+                "pattern": "^\\S+\\.(sam|sam.gz)$",
+                "errorMessage": "SAM header file cannot contain spaces and must be in SAM format",
+                "meta": ["extra_header"]
             }
         },
         "required": ["specimen", "run", "datatype", "datafile"]

conf/modules.config

@@ -113,6 +113,7 @@ process {
     }
 
     withName: SAMTOOLS_ADDREPLACERG {
+        ext.args =  { "${meta.read_group}" }
         ext.prefix = { "${input.baseName}_addRG" }
     }
 

docs/usage.md

@@ -32,22 +32,23 @@ The samplesheet can have as many columns as you desire, however, there is a stri
 A final samplesheet file consisting of both HiC and PacBio data may look something like the one below.
 
 ```console
-specimen,run,datatype,datafile,library,barcode
-specimen1,run1,hic,hic1.cram,,
-specimen1,run2,hic,hic2.cram,,
-specimen2,run3,hic,hic3.cram,,
-specimen2,run4,pacbio,pacbio1.bam,uli,
-specimen3,run5,pacbio,pacbio2.bam,,
+specimen,run,datatype,datafile,library,barcode,extra_header
+specimen1,run1,hic,hic1.cram,,,
+specimen1,run2,hic,hic2.cram,,,
+specimen2,run3,hic,hic3.cram,,,
+specimen2,run4,pacbio,pacbio1.bam,uli,,headers/pacbio_rg.sam
+specimen3,run5,pacbio,pacbio2.bam,,,
 ```
 
-| Column     | Description                                                                                                                                                                                                                                 |
-| ---------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `specimen` | Identifier of the specimen. Usually a BioSpecimen accession, i,e. `SAMEA7521529`.                                                                                                                                                           |
-| `run`      | Identifier of the sequencing run. Usually the accession number of the data in INSDC. For example,`ERR9248445` (hic), `ERR9284044` (pacbio).                                                                                                 |
-| `datatype` | Type of sequencing data. Must be one of `hic`, `illumina`, `pacbio`, `pacbio_clr`, or `ont`.                                                                                                                                                |
-| `datafile` | Full path to read data file. Must be `bam`, `cram`, `fastq.gz` or `fq.gz` for `illumina` and `hic`. Must be `bam`, `fastq.gz` or `fq.gz` for `pacbio`, `pacbio_clr`, and `ont`. Note that FASTQ inputs should be interleaved if paired-end. |
-| `library`  | (Optional) The library value is a unique identifier which is assigned to read group (`@RG`) ID. If the library name is not specified, the pipeline will auto-create library name using the data filename provided in the samplesheet.       |
-| `barcode`  | (Optional) Barcode identifier used to trim barcode adapter for PacBio reads. If empty, barcode-specific adapter sequences will not be trimmed.                                                                                              |
+| Column         | Description                                                                                                                                                                                                                                                                           |
+| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `specimen`     | Identifier of the specimen. Usually a BioSpecimen accession, i,e. `SAMEA7521529`.                                                                                                                                                                                                     |
+| `run`          | Identifier of the sequencing run. Usually the accession number of the data in INSDC. For example,`ERR9248445` (hic), `ERR9284044` (pacbio).                                                                                                                                           |
+| `datatype`     | Type of sequencing data. Must be one of `hic`, `illumina`, `pacbio`, `pacbio_clr`, or `ont`.                                                                                                                                                                                          |
+| `datafile`     | Full path to read data file. Must be `bam`, `cram`, `fastq.gz` or `fq.gz` for `illumina` and `hic`. Must be `bam`, `fastq.gz` or `fq.gz` for `pacbio`, `pacbio_clr`, and `ont`. Note that FASTQ inputs should be interleaved if paired-end.                                           |
+| `library`      | (Optional) The library value is a unique identifier which is assigned to read group (`@RG`) ID. If the library name is not specified, the pipeline will auto-create library name using the data filename provided in the samplesheet.                                                 |
+| `barcode`      | (Optional) Barcode identifier used to trim barcode adapter for PacBio reads. If empty, barcode-specific adapter sequences will not be trimmed.                                                                                                                                        |
+| `extra_header` | (Optional) Path to a SAM file containing additional SAM header lines including `@RG`, `@PG`. This information will be injected into output BAM/CRAM headers. `@RG` lines are overwritten, while `@PG` lines are added to the existing headers (auto add `.extra_header` to `@PG ID`). |
 
 An [example samplesheet](../assets/samplesheet.csv) has been provided with the pipeline.
 

modules/local/samtools/reheader/samtools_replaceheader.nf

@@ -8,8 +8,8 @@ process SAMTOOLS_REHEADER {
         : 'community.wave.seqera.io/library/htslib_samtools:1.23.1--5b6bb4ede7e612e5'}"
 
     input:
-    tuple val(meta), path(file, stageAs: "input/*")
-    path(header)
+    tuple val(meta), path(file, stageAs: "input/*"), path(sample_extra_header, stageAs: "extra_header/*")
+    path(header_template)
 
     output:
     tuple val(meta), path("${prefix}.bam") , optional:true, emit: bam
@@ -22,17 +22,24 @@ process SAMTOOLS_REHEADER {
     script:
     prefix = task.ext.prefix ?: "${meta.id}"
     suffix = file.getExtension()
+    def sq_template = header_template ? " | grep -v ^@SQ && grep '^@SQ' ${header_template}" : ""
+    // Suffix all PG IDs from sample extra_header to keep them distinct from native file-header PG IDs.
+    // Also rewrite matching PP references so the extra-header PG chain remains intact after suffixing.
+    def pg_extra = sample_extra_header ? "awk 'BEGIN { FS = OFS = \"\\t\" } FNR == NR { if (\$1 == \"@PG\") for (i = 1; i <= NF; i++) if (\$i ~ /^ID:/) { ids[substr(\$i, 4)] = 1; break } next } \$1 == \"@PG\" { for (i = 1; i <= NF; i++) { if (\$i ~ /^ID:/) { \$i = \$i \".extra_header\" } else if (\$i ~ /^PP:/) { pp = substr(\$i, 4); if (pp in ids) \$i = \"PP:\" pp \".extra_header\" } } print }' ${sample_extra_header} ${sample_extra_header} || true" : "true"
 
     if ("$file" == "${prefix}.${suffix}") error "Input and output names are the same, use \"task.ext.prefix\" to disambiguate!"
     """
     # Replace SQ lines with those from external template
-    ( samtools view --no-PG --header-only ${file} | \\
-    grep -v ^@SQ && grep ^@SQ ${header} ) > temp.header.sam
+    ( samtools view --no-PG --header-only ${file} \\
+        ${sq_template} ) > temp.header.sam
 
     # custom sort for readability (retain order of insertion but sort groups by tag)
+    # Add PG lines from sample header
+    # Sort order: HD, SQ, RG, extra PG, PG, other
     ( grep ^@HD temp.header.sam || true && \\
     grep ^@SQ temp.header.sam || true && \\
     grep ^@RG temp.header.sam || true && \\
+    ${pg_extra} && \\
     grep ^@PG temp.header.sam || true && \\
     grep -v -E '^@HD|^@SQ|^@RG|^@PG' temp.header.sam || true; \\
     ) > temp.sorted.header.sam

nextflow.config

@@ -322,7 +322,7 @@ manifest {
     mainScript      = 'main.nf'
     defaultBranch   = 'main'
     nextflowVersion = '!>=25.04.0'
-    version         = '2.0.3'
+    version         = '2.1.0-dev'
     doi             = '10.5281/zenodo.6563577'
 }
 

nf-test.config

@@ -1,3 +1,7 @@
+// Force UTC so DT fields in BAM/CRAM headers don't shift across DST
+System.setProperty('user.timezone', 'UTC')
+TimeZone.setDefault(TimeZone.getTimeZone('UTC'))
+
 config {
     // location for all nf-test tests
     testsDir "."