Skip to content

Make pipeline configs strict syntax compatible #1017

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
nvnieuwk wants to merge 6 commits into from
Closed

Make pipeline configs strict syntax compatible #1017

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
cf61456
make pipeline configs strict syntax compatible
nvnieuwk Feb 18, 2026
447ffb2
Merge branch 'master' into strict/pipeline-configs
jfy133 Mar 19, 2026
ba59eb7
Merge branch 'master' into strict/pipeline-configs
maxulysse Apr 2, 2026
1e8bcc0
Merge branch 'master' into strict/pipeline-configs
maxulysse Apr 7, 2026
fe78c2c
Merge branch 'master' into strict/pipeline-configs
maxulysse Apr 7, 2026
681879e
Merge branch 'master' into strict/pipeline-configs
maxulysse Apr 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion conf/pipeline/deepmodeloptim/crg.config
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ profiles {
singularity {
enabled = true
envWhitelist = "CUDA_VISIBLE_DEVICES,NVIDIA_VISIBLE_DEVICES"
if (params.use_gpu) { singularity.runOptions = '--nv' }
runOptions = params.get('use_gpu', false) ? '--nv' : ''
pullTimeout = "60m"
}
}
Expand Down
141 changes: 73 additions & 68 deletions conf/pipeline/eager/eva.config
View file
Open in desktop

Large diffs are not rendered by default.

90 changes: 48 additions & 42 deletions conf/pipeline/eager/maestro.config
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,12 @@
* nextflow run nf-core/eager -profile maestro,<qos>,maestro,<genome> (where <qos> is long or normal and <genome> is nuclear, mitocondrial or unlimitedtime)
*/

validation {
ignoreParams = [
'eager_maestro_check_max_func'
]
}

params {

config_profile_name = 'nf-core/eager nuclear/mitocondrial - human profiles'
Expand All @@ -21,41 +27,41 @@ profiles {
maxRetries = 2

withName: makeBWAIndex {
cpus = { check_max(8 * task.attempt, 'cpus') }
memory = { check_max(8.GB * task.attempt, 'memory') }
time = { check_max(12.h * task.attempt, 'time') }
cpus = { params.eager_maestro_check_max_func(8 * task.attempt, 'cpus') }
memory = { params.eager_maestro_check_max_func(8.GB * task.attempt, 'memory') }
time = { params.eager_maestro_check_max_func(12.h * task.attempt, 'time') }
}
withName: adapter_removal {
cpus = { check_max(8 * task.attempt, 'cpus') }
memory = { check_max(16.GB * task.attempt, 'memory') }
time = { check_max(12.h * task.attempt, 'time') }
cpus = { params.eager_maestro_check_max_func(8 * task.attempt, 'cpus') }
memory = { params.eager_maestro_check_max_func(16.GB * task.attempt, 'memory') }
time = { params.eager_maestro_check_max_func(12.h * task.attempt, 'time') }
}
withName: bwa {
cpus = { check_max(40 * task.attempt, 'cpus') }
memory = { check_max(40.GB * task.attempt, 'memory') }
cpus = { params.eager_maestro_check_max_func(40 * task.attempt, 'cpus') }
memory = { params.eager_maestro_check_max_func(40.GB * task.attempt, 'memory') }
time = 24.h
cache = 'deep'
}
withName: markduplicates {
errorStrategy = { task.exitStatus in [143, 137, 104, 134, 139] ? 'retry' : 'finish' }
cpus = { check_max(16 * task.attempt, 'cpus') }
memory = { check_max(16.GB * task.attempt, 'memory') }
time = { check_max(12.h * task.attempt, 'time') }
cpus = { params.eager_maestro_check_max_func(16 * task.attempt, 'cpus') }
memory = { params.eager_maestro_check_max_func(16.GB * task.attempt, 'memory') }
time = { params.eager_maestro_check_max_func(12.h * task.attempt, 'time') }
}
withName: damageprofiler {
cpus = 1
memory = { check_max(8.GB * task.attempt, 'memory') }
time = { check_max(6.h * task.attempt, 'time') }
memory = { params.eager_maestro_check_max_func(8.GB * task.attempt, 'memory') }
time = { params.eager_maestro_check_max_func(6.h * task.attempt, 'time') }
}
withName: fastp {
cpus = 8
memory = { check_max(8.GB * task.attempt, 'memory') }
time = { check_max(6.h * task.attempt, 'time') }
memory = { params.eager_maestro_check_max_func(8.GB * task.attempt, 'memory') }
time = { params.eager_maestro_check_max_func(6.h * task.attempt, 'time') }
}
withName: fastqc {
cpus = 2
memory = { check_max(8.GB * task.attempt, 'memory') }
time = { check_max(6.h * task.attempt, 'time') }
memory = { params.eager_maestro_check_max_func(8.GB * task.attempt, 'memory') }
time = { params.eager_maestro_check_max_func(6.h * task.attempt, 'time') }
}
}
}
Expand All @@ -66,40 +72,40 @@ profiles {
maxRetries = 2

withName: makeBWAIndex {
cpus = { check_max(8 * task.attempt, 'cpus') }
memory = { check_max(8.GB * task.attempt, 'memory') }
time = { check_max(12.h * task.attempt, 'time') }
cpus = { params.eager_maestro_check_max_func(8 * task.attempt, 'cpus') }
memory = { params.eager_maestro_check_max_func(8.GB * task.attempt, 'memory') }
time = { params.eager_maestro_check_max_func(12.h * task.attempt, 'time') }
}
withName: adapter_removal {
cpus = { check_max(8 * task.attempt, 'cpus') }
memory = { check_max(16.GB * task.attempt, 'memory') }
time = { check_max(12.h * task.attempt, 'time') }
cpus = { params.eager_maestro_check_max_func(8 * task.attempt, 'cpus') }
memory = { params.eager_maestro_check_max_func(16.GB * task.attempt, 'memory') }
time = { params.eager_maestro_check_max_func(12.h * task.attempt, 'time') }
}
withName: bwa {
cpus = { check_max(5 * task.attempt, 'cpus') }
memory = { check_max(5.GB * task.attempt, 'memory') }
cpus = { params.eager_maestro_check_max_func(5 * task.attempt, 'cpus') }
memory = { params.eager_maestro_check_max_func(5.GB * task.attempt, 'memory') }
time = 24.h
}
withName: markduplicates {
errorStrategy = { task.exitStatus in [143, 137, 104, 134, 139] ? 'retry' : 'finish' }
cpus = { check_max(5 * task.attempt, 'cpus') }
memory = { check_max(5.GB * task.attempt, 'memory') }
time = { check_max(6.h * task.attempt, 'time') }
cpus = { params.eager_maestro_check_max_func(5 * task.attempt, 'cpus') }
memory = { params.eager_maestro_check_max_func(5.GB * task.attempt, 'memory') }
time = { params.eager_maestro_check_max_func(6.h * task.attempt, 'time') }
}
withName: damageprofiler {
cpus = 1
memory = { check_max(5.GB * task.attempt, 'memory') }
time = { check_max(3.h * task.attempt, 'time') }
memory = { params.eager_maestro_check_max_func(5.GB * task.attempt, 'memory') }
time = { params.eager_maestro_check_max_func(3.h * task.attempt, 'time') }
}
withName: fastp {
cpus = 8
memory = { check_max(5.GB * task.attempt, 'memory') }
time = { check_max(3.h * task.attempt, 'time') }
memory = { params.eager_maestro_check_max_func(5.GB * task.attempt, 'memory') }
time = { params.eager_maestro_check_max_func(3.h * task.attempt, 'time') }
}
withName: fastqc {
cpus = 2
memory = { check_max(8.GB * task.attempt, 'memory') }
time = { check_max(6.h * task.attempt, 'time') }
memory = { params.eager_maestro_check_max_func(8.GB * task.attempt, 'memory') }
time = { params.eager_maestro_check_max_func(6.h * task.attempt, 'time') }
}
}
}
Expand All @@ -117,31 +123,31 @@ profiles {
// Function to ensure that resource requirements don't go beyond
// a maximum limit
// FOR DSL1 PIPELINE ONLY!
def check_max(obj, type) {
params.eager_maestro_check_max_func = { obj, type ->
if (type == 'memory') {
try {
if (obj.compareTo(params.max_memory as nextflow.util.MemoryUnit) == 1)
return params.max_memory as nextflow.util.MemoryUnit
if (obj.compareTo(params.max_memory as MemoryUnit) == 1)
return params.max_memory as MemoryUnit
else
return obj
} catch (all) {
} catch (_) {
println " ### ERROR ### Max memory '${params.max_memory}' is not valid! Using default value: $obj"
return obj
}
} else if (type == 'time') {
try {
if (obj.compareTo(params.max_time as nextflow.util.Duration) == 1)
return params.max_time as nextflow.util.Duration
if (obj.compareTo(params.max_time as Duration) == 1)
return params.max_time as Duration
else
return obj
} catch (all) {
} catch (_) {
println " ### ERROR ### Max time '${params.max_time}' is not valid! Using default value: $obj"
return obj
}
} else if (type == 'cpus') {
try {
return Math.min( obj, params.max_cpus as int )
} catch (all) {
} catch (_) {
println " ### ERROR ### Max cpus '${params.max_cpus}' is not valid! Using default value: $obj"
return obj
}
Expand Down
2 changes: 1 addition & 1 deletion conf/pipeline/methylseq/ku_sund_danhead.config
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,6 @@ process {
stageInMode = 'copy'
}
withName: 'NFCORE_METHYLSEQ:METHYLSEQ:BISMARK:BISMARK_ALIGN' {
multicore = 1
ext.args = '--multicore 1'
}
}
2 changes: 1 addition & 1 deletion conf/pipeline/multiplesequencealign/crg.config
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ profiles {
singularity {
enabled = true
envWhitelist = "CUDA_VISIBLE_DEVICES,NVIDIA_VISIBLE_DEVICES"
if (params.use_gpu) { singularity.runOptions = '--nv' }
runOptions = params.get('use_gpu', false) ? '--nv' : ''
pullTimeout = "60m"
}
}
Expand Down
8 changes: 1 addition & 7 deletions conf/pipeline/proteinfold/nci_gadi.config
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,6 @@ profiles {
// Define process resource limits
process {
executor = 'pbspro'
project = System.getenv("PROJECT")
storage = params.storage_account?.trim() ? params.storage_account : "scratch/${params.project}+gdata/${params.project}"
module = 'singularity'
cache = 'lenient'
stageInMode = 'symlink'
Expand All @@ -22,7 +20,6 @@ profiles {
withName: 'RUN_ALPHAFOLD2|RUN_ALPHAFOLD2_PRED|RUN_ALPHAFOLD2_MSA' {
queue = params.use_gpu ? 'gpuvolta' : 'normal'
cpus = 48
gpus = 4
time = '4h'
memory = 380.GB
}
Expand All @@ -31,7 +28,6 @@ profiles {
container = "nf-core/proteinfold_colabfold:1.1.1"
queue = params.use_gpu ? 'gpuvolta' : 'normal'
cpus = 48
gpus = 4
time = '4h'
memory = 380.GB
}
Expand All @@ -40,18 +36,16 @@ profiles {
container = "nf-core/proteinfold_esmfold:1.1.1"
queue = params.use_gpu ? 'gpuvolta' : 'normal'
cpus = 48
gpus = 4
time = '4h'
memory = 380.GB
}
}

// Write custom trace file with outputs required for SU calculation
def trace_timestamp = new java.util.Date().format('yyyy-MM-dd_HH-mm-ss')
trace {
enabled = true
overwrite = false
file = "./gadi-nf-core-trace-${trace_timestamp}.txt"
file = "./gadi-nf-core-trace-${new java.util.Date().format('yyyy-MM-dd_HH-mm-ss')}.txt"
fields = 'name,status,exit,duration,realtime,cpus,%cpu,memory,%mem,rss'
}
}
Expand Down
26 changes: 14 additions & 12 deletions conf/pipeline/sarek/cfc.config
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
// Profile config names for nf-core/configs

validation.ignoreParams = ['sarek_cfc_check_resource_func']

params {
// Specific nf-core/configs params
config_profile_contact = 'Friederike Hanssen (@FriederikeHanssen)'
Expand All @@ -9,34 +11,34 @@ params {
// Specific nf-core/sarek process configuration
process {
withName:'StrelkaSingle|Strelka|StrelkaBP|MantaSingle|Manta' {
cpus = { check_resource( 20 * task.attempt) }
memory = { check_resource( 59.GB * task.attempt) }
cpus = { params.sarek_cfc_check_resource_func( 20 * task.attempt) }
memory = { params.sarek_cfc_check_resource_func( 59.GB * task.attempt) }
}
withName:'MSIsensor_scan|MSIsensor_msi' {
memory = { check_resource( 55.GB * task.attempt ) }
memory = { params.sarek_cfc_check_resource_func( 55.GB * task.attempt ) }

}
withName:BamQC {
memory = { check_resource( 372.GB * task.attempt) }
memory = { params.sarek_cfc_check_resource_func( 372.GB * task.attempt) }
}

withName:MapReads{
cpus = { check_resource( 20 * task.attempt ) }
memory = { check_resource( 59.GB * task.attempt) }
cpus = { params.sarek_cfc_check_resource_func( 20 * task.attempt ) }
memory = { params.sarek_cfc_check_resource_func( 59.GB * task.attempt) }
}
}

def check_resource(obj) {
params.sarek_cfc_check_resource_func = { obj ->
try {
if (obj.getClass() == nextflow.util.MemoryUnit && obj.compareTo(params.max_memory as nextflow.util.MemoryUnit) == 1)
return params.max_memory as nextflow.util.MemoryUnit
else if (obj.getClass() == nextflow.util.Duration && obj.compareTo(params.max_time as nextflow.util.Duration) == 1)
return params.max_time as nextflow.util.Duration
if (obj.getClass() == nextflow.util.MemoryUnit && obj.compareTo(params.max_memory as MemoryUnit) == 1)
return params.max_memory as MemoryUnit
else if (obj.getClass() == nextflow.util.Duration && obj.compareTo(params.max_time as Duration) == 1)
return params.max_time as Duration
else if (obj.getClass() == java.lang.Integer)
return Math.min(obj, params.max_cpus as int)
else
return obj
} catch (all) {
} catch (_) {
println " ### ERROR ### Max params max_memory:'${params.max_memory}', max_time:'${params.max_time}' or max_cpus:'${params.max_cpus}' is not valid! Using default value: $obj"
}
}
4 changes: 0 additions & 4 deletions conf/pipeline/sarek/icr_davros.config
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,4 @@
process {
errorStrategy = { task.exitStatus in [104, 134, 137, 139, 141, 143, 255] ? 'retry' : 'finish' }
maxRetries = 5
withName: MapReads {
memory = { check_resource(12.GB) }
time = { check_resource(48.h * task.attempt) }
}
}
8 changes: 4 additions & 4 deletions conf/pipeline/scflow/imperial.config
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,13 @@ params {
config_profile_contact = 'NA'

// Analysis Resource Params
ctd_folder = "/rds/general/user/${USER}/projects/ukdrmultiomicsproject/live/Analyses/scFlowResources/refs/ctd"
ensembl_mappings = "/rds/general/user/${USER}/projects/ukdrmultiomicsproject/live/Analyses/scFlowResources/src/ensembl-ids/ensembl_mappings.tsv"
ctd_folder = "/rds/general/user/${System.getenv('USER')}/projects/ukdrmultiomicsproject/live/Analyses/scFlowResources/refs/ctd"
ensembl_mappings = "/rds/general/user/${System.getenv('USER')}/projects/ukdrmultiomicsproject/live/Analyses/scFlowResources/src/ensembl-ids/ensembl_mappings.tsv"
}

singularity {
enabled = true
autoMounts = true
cacheDir = "/rds/general/user/${USER}/projects/ukdrmultiomicsproject/live/.singularity-cache"
runOptions = "-B /rds/,/rdsgpfs/,/rds/general/user/${USER}/ephemeral/tmp/:/tmp,/rds/general/user/${USER}/ephemeral/tmp/:/var/tmp"
cacheDir = "/rds/general/user/${System.getenv('USER')}/projects/ukdrmultiomicsproject/live/.singularity-cache"
runOptions = "-B /rds/,/rdsgpfs/,/rds/general/user/${System.getenv('USER')}/ephemeral/tmp/:/tmp,/rds/general/user/${System.getenv('USER')}/ephemeral/tmp/:/var/tmp"
}
Loading