diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9f7f6c149..e40e5dc09 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -121,6 +121,7 @@ jobs: - "gis" - "googlebatch" - "hasta" + - "hazel" - "hki_genie" - "hypatia" - "humantechnopole" diff --git a/conf/hazel.config b/conf/hazel.config new file mode 100644 index 000000000..f961ddb71 --- /dev/null +++ b/conf/hazel.config @@ -0,0 +1,64 @@ +// NC State University - Hazel HPC Cluster +// nf-core institutional config +// Scheduler: SLURM | Containers: Singularity/Apptainer +// +// To activate: nextflow run -profile hazel +// +// Submit an issue or PR to add this to the nf-core/configs repo: +// https://github.com/nf-core/configs + +params { + config_profile_name = 'NC State University - Hazel HPC' + config_profile_description = 'nf-core config for the Hazel HPC cluster at NC State University' + config_profile_contact = 'Seth Weaver (@sweavs111)' + config_profile_url = 'https://brc.ncsu.edu/' + + max_memory = 128.GB + max_cpus = 24 + max_time = 120.h + + // Default GPU allocation. Override at runtime with --hazel_gpu 'gpu::' + hazel_gpu = 'gpu:h100:1' + + // Default partitions. Override at runtime with --partition or --gpu_partition + partition = 'compute' + gpu_partition = 'gpu' +} + +validation { + ignoreParams = ['hazel_gpu', 'partition', 'gpu_partition'] +} + +// ─── Singularity ──────────────────────────────────────────────────────────── +singularity { + enabled = true + autoMounts = true + cacheDir = System.getenv('APPTAINER_CACHEDIR') ?: "/share/${System.getenv('GROUP')}/${System.getenv('USER')}/tmp" +} + +// ─── Executor tuning ──────────────────────────────────────────────────────── +executor { + queueSize = 16 + pollInterval = '1 min' + queueStatInterval = '5 min' + submitRateLimit = '10 sec' +} + +// ─── Process defaults ──────────────────────────────────────────────────────── +process { + beforeScript = 'module load singularity' + executor = 'slurm' + queue = params.partition + + resourceLimits = [ + memory: params.max_memory, + cpus: params.max_cpus, + time: params.max_time + ] + + // GPU jobs use gpu partition + withLabel: 'process_gpu' { + queue = params.gpu_partition + clusterOptions = { "--gres=${params.hazel_gpu}" } + } +} diff --git a/docs/hazel.md b/docs/hazel.md new file mode 100644 index 000000000..183b3ec9f --- /dev/null +++ b/docs/hazel.md @@ -0,0 +1,48 @@ +# nf-core/configs: NC State Hazel HPC Configuration + +nf-core pipelines have been configured for use on the Hazel HPC at NC State University. + +To use, run the pipeline with `-profile hazel`. + +Example: + +`nextflow run nf-core/ -profile hazel` + +Running this command will download the `hazel` config from the nf-core repository, and will submit nextflow processes as jobs to the `slurm` job scheduler. Therefore, the `nextflow run` command must be run from the `login node`. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. These images are cached in `/share/$GROUP/$USER/tmp` so they don't have to be re-downloaded on subsequent runs. + +## Loading required environment modules + +Before running the pipeline you will need to load Nextflow using the environment module system on hazel. Some of these modules are managed by the BRC, and are not in the default module path. See this resource for information about using these modules: https://hurwitzlab.github.io/COS_Compute_Handbook/chapters/14_loading_brc_modules.html + +```bash +## Load Nextflow environment modules +module purge # Optional, unload any currently loaded modules. +module load nextflow/26.04.3 # Required +module load nf-core # Optional, manages and downloads nf-core pipelines for common use. Requires singularity. +``` + +## Job submission partitions + +By default, using this profile will submit jobs to the `compute` partition. To specify a different partition for submission, use `--partition ` in the `nextflow run` command. + +Nextflow processes with the `process_gpu` label will be submitted to the `gpu` partition by default; override with `--gpu_partition `. To request a different GPU type or multiple GPUs, use `--hazel_gpu 'gpu::'` at runtime. + + +## Overriding Hazel config options + +Nextflow-level options passed to your `nextflow run` command will override options contained in the Hazel institutional profile: + +```bash +nextflow run nf-core/ -profile hazel --max_memory 256.GB # raise memory ceiling +nextflow run nf-core/ -profile hazel --hazel_gpu 'gpu:a100:2' # different GPU +``` + +For persistent overrides, use a local config file with `-c my_overrides.config`. + +:::note +You will need an account to use the Hazel HPC in order to run the pipeline. If in doubt contact IT or your PI to get access. +::: + +:::note +Nextflow will need to submit the jobs via the job scheduler to the HPC cluster and as such the commands above will have to be executed on one of the login nodes. If in doubt contact IT. +::: diff --git a/nfcore_custom.config b/nfcore_custom.config index 0167fc914..43e3b380d 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -190,6 +190,9 @@ profiles { hasta { includeConfig "${params.custom_config_base}/conf/hasta.config" } + hazel { + includeConfig "${params.custom_config_base}/conf/hazel.config" + } hki_genie { includeConfig "${params.custom_config_base}/conf/hki_genie.config" }