peka-eclip/nextflow.config at main · ulelab/peka-eclip · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
#!/usr/bin/env nextflow

params {

    input = "./"
    outdir = "./results_umi"
    tracedir = "${params.outdir}/pipeline_info/"

}

// Enable by default nextflow tracing and place in trace dir location
timeline {
  enabled = true
  file = "${params.tracedir}/execution_timeline.html"
}
report {
  enabled = true
  file = "${params.tracedir}/execution_report.html"
}
trace {
  enabled = true
  file = "${params.tracedir}/execution_trace.txt"
  fields = "task_id,hash,native_id,name,status,exit,submit,start,complete,duration,realtime,cpus,time,memory,%cpu,peak_rss,peak_vmem,rchar,wchar"
}
dag {
  enabled = true
  file = "${params.tracedir}/pipeline_dag.svg"
}

singularity {
  enabled = true
  autoMounts = true
}

// Load some standard modules before running script
process {
  executor = 'slurm'
}

executor {
    queueSize = 300
}

// Crick specific computing max resource levels
params {
  max_memory = 224.GB
  max_cpus = 32
  max_time = 72.h
}

// Create process scaling defaults for running on a cluster
process {

  // Memory and time increases with the number of retrys
  cpus = { check_max( 2, 'cpus' ) }
  memory = { check_max( 4.GB * task.attempt, 'memory' ) }
  time = { check_max( 2.h * task.attempt, 'time' ) }

  errorStrategy = { task.exitStatus in [143,137,104,134,139] ? 'retry' : 'terminate' }
  maxRetries = 1
  maxErrors = '-1'

  // Process-specific resource requirements
  withLabel: low_memory {
    memory = { check_max( 16.GB * task.attempt, 'memory' ) }
  }
  withLabel: mid_memory {
    cpus = { check_max (4, 'cpus')}
    memory = { check_max( 28.GB * task.attempt, 'memory' ) }
    time = { check_max( 8.h * task.attempt, 'time' ) }
  }
  withLabel: high_memory {
    cpus = { check_max (10, 'cpus')}
    memory = { check_max( 70.GB * task.attempt, 'memory' ) }
    time = { check_max( 8.h * task.attempt, 'time' ) }
  }

}

// Function to ensure that resource requirements don't go beyond
// a maximum limit
def check_max(obj, type) {
  if (type == 'memory') {
    try {
      if (obj.compareTo(params.max_memory as nextflow.util.MemoryUnit) == 1)
        return params.max_memory as nextflow.util.MemoryUnit
      else
        return obj
    } catch (all) {
      println "   ### ERROR ###   Max memory '${params.max_memory}' is not valid! Using default value: $obj"
      return obj
    }
  } else if (type == 'time') {
    try {
      if (obj.compareTo(params.max_time as nextflow.util.Duration) == 1)
        return params.max_time as nextflow.util.Duration
      else
        return obj
    } catch (all) {
      println "   ### ERROR ###   Max time '${params.max_time}' is not valid! Using default value: $obj"
      return obj
    }
  } else if (type == 'cpus') {
    try {
      return Math.min( obj, params.max_cpus as int )
    } catch (all) {
      println "   ### ERROR ###   Max cpus '${params.max_cpus}' is not valid! Using default value: $obj"
      return obj
    }
  } else if (type == 'gpus') {
    try {
      return Math.min( obj, params.max_gpus as int )
    } catch (all) {
      println "   ### ERROR ###   Max gpus '${params.max_gpus}' is not valid! Using default value: $obj"
      return obj as String
    }
  }
}