switch to using new python run script

Author: Raymond Lim

Makefile.inc

@@ -161,12 +161,9 @@ space +=
 $(space) := 
 $(space) +=
 
-
-
 # $(call strip-suffix,filename)
 strip-suffix = $(firstword $(subst ., ,$(1)))
 
-
 LINK = ln -svf $(notdir $1) $(2) && touch $1
 
 NOW := $(shell date +"%F")
@@ -188,164 +185,31 @@ QUEUE ?=
 
 WALLTIMES = MEM SHORT LONG
 ### PBS
-# mem for PBS: multiply mem_per_cpu by ncpu for request
-# $(call PBS_MEM,ncpu,mem_per_cpu)
-PBS_SHORT = -l mem=$2 -l walltime=1:00:00
-PBS_MEM = -l mem=$2 -l walltime=24:00:00
-PBS_LONG = -l mem=$2 -l walltime=128:00:00
-PBS_PARALLEL_MEM_WALLTIME = -l nodes=1:ppn=$1 -l walltime=$3  -l mem=$(shell unit=`echo $2 | sed 's/[^A-Za-z]//g'`; echo "$1 * $2" | sed 's/[A-Za-z]//g; s/\..*//;' | bc | sed 's/$$/'$$unit'/')
-PBS_PARALLEL_MEM = $(call PBS_PARALLEL_MEM_WALLTIME,$1,$3,12:00:00)
-PBS_PARALLEL_SHORT = $(call PBS_PARALLEL_MEM_WALLTIME,$1,$3,1:00:00)
-PBS_PARALLEL_LONG = $(call PBS_PARALLEL_MEM_WALLTIME,$1,$3,256:00:00)
-PBS_QSUB_ARGS = -o $(LOG) -j oe -d $(PWD)
-PBS_QSUB_SCRIPT = python modules/scripts/qsub_pbs.py
-PBS_DOCKER_MEM = -l mem=$2 -l walltime=24:00:00 -l nodes=1:docker -l mem=$2
-PBS_DOCKER_SHORT = -l mem=$2 -l walltime=1:00:00 -l nodes=1:docker -l mem=$2
-PBS_DOCKER_LONG = -l mem=$2 -l walltime=128:00:00 -l nodes=1:docker -l mem=$2
-PBS_DOCKER_PARALLEL_MEM_WALLTIME = -l nodes=1:docker:ppn=$1 -l walltime=$3 -l mem=$(shell unit=`echo $2 | sed 's/^[A-Za-z]//g'`; echo "$1 * $2" | sed 's/[A-Za-z]//g; s/\..*//;' | bc | sed 's/$$/'$$unit'/')
-PBS_DOCKER_PARALLEL_MEM = $(call PBS_DOCKER_PARALLEL_MEM_WALLTIME,$1,$3,12:00:00)
-PBS_DOCKER_PARALLEL_SHORT = $(call PBS_DOCKER_PARALLEL_MEM_WALLTIME,$1,$3,1:00:00)
-PBS_DOCKER_PARALLEL_LONG = $(call PBS_DOCKER_PARALLEL_MEM_WALLTIME,$1,$3,128:00:00)
 PBS_NODES = gpu-2-14 cpu-6-1 gpu-1-4
 
-
 #### SGE
-# mem for SGE
-# $(call SGE_MEM,ncpu,mem_per_cpu)
 SGE_PARALLEL_ENV = smp
-SGE_PARALLEL_MEM = -pe $(SGE_PARALLEL_ENV) $1 -l virtual_free=$2,h_vmem=$3
-SGE_PARALLEL_SHORT = $(SGE_PARALLEL_MEM)
-SGE_PARALLEL_LONG = $(SGE_PARALLEL_MEM)
-SGE_SHORT = $(SGE_MEM)
-SGE_MEM = -l virtual_free=$1,h_vmem=$2
-SGE_LONG = $(SGE_MEM)
-QSUB_PRIORITY ?= 0
-SGE_QSUB_ARGS = -V -wd $(PWD) -now n -notify -o $(LOG) -j y -S $(SHELL) -p $(QSUB_PRIORITY) -b n
 SGE_NODES = e01 e02 e06
 SGE_QSUB_SCRIPT = python modules/scripts/qsub.py
 
-
-
-## DRMAA qsub script
-QSUB_SCRIPT_OPTS = -o $@ -s $($(CLUSTER_ENGINE)_NODES)
-QS = $($(CLUSTER_ENGINE)_QSUB_SCRIPT) $(QSUB_SCRIPT_OPTS) $1 --args $(foreach q,$(strip $2), -q $q)
-QSUB = $(call QS,,$(QUEUE))
-QSUB_QUEUE = $(call QS,,$1)
-# check output file for non-zero size
-QSUB_CHECK = $(call QS,-c)
-QSUB_CHECK_QUEUE = $(call QS,-c,-q $1)
-
-
 MAKE_PID := $(shell echo $$PPID)
 JOB_FLAG := $(filter -j%, $(subst -j ,-j,$(shell ps T | grep "^\s*$(MAKE_PID).*$(MAKE)")))
 JOBS := $(subst -j,,$(JOB_FLAG))
 
-#### no cluster
-RUN_FIRST = umask $(UMASK); $(if $(and $(findstring true,$(USE_CLUSTER)),$(JOBS)),sleep $$[ ( $$RANDOM % 20 ) + 1 ]s;) $(MKDIRS)
-ECHO_FIRST = echo -n "umask $(UMASK); set -o pipefail; $(call ECHO_ACTIVATE_ENV,$(JRFLAB_MODULES_ENV)); ";
-ECHO_FIRST_ENV = echo -n "umask $(UMASK); set -o pipefail; $(call ECHO_ACTIVATE_ENV,$1); ";
-
 # run locally
+RUN_FIRST = umask $(UMASK); $(if $(and $(findstring true,$(USE_CLUSTER)),$(JOBS)),sleep $$[ ( $$RANDOM % 20 ) + 1 ]s;) $(MKDIRS)
 INIT = $(RUN_FIRST); $(call ACTIVATE_ENV,$(JRFLAB_MODULES_ENV)); umask $(UMASK); set -o pipefail;
 INIT_ENV = $(RUN_FIRST); $(call ACTIVATE_ENV,$1); umask $(UMASK); set -o pipefail;
 
 CLUSTER_ENGINE ?= SGE
 USE_CLUSTER ?= true
-ifeq ($(USE_CLUSTER),false)
-LSCRIPT = $(RUN_FIRST); ($(ECHO_FIRST) echo $1) | $(SHELL) >& $(LOG)
-LSCRIPT_ENV = $(RUN_FIRST); ($(call ECHO_FIRST_ENV,$1)) echo $2) | $(SHELL) >& $(LOG)
-LSCRIPT_PARALLEL_MEM = $(RUN_FIRST); ($(ECHO_FIRST) echo $4) | $(SHELL) >& $(LOG)
-LSCRIPT_ENV_PARALLEL_MEM = $(RUN_FIRST); ($(call ECHO_FIRST_ENV,$1) echo $5) | $(SHELL) >& $(LOG)
-LSCRIPT_PARALLEL_SHORT = $(LSCRIPT_PARALLEL_MEM)
-LSCRIPT_ENV_PARALLEL_SHORT = $(LSCRIPT_ENV_PARALLEL_MEM)
-LSCRIPT_PARALLEL_LONG = $(LSCRIPT_PARALLEL_MEM)
-LSCRIPT_ENV_PARALLEL_LONG = $(LSCRIPT_ENV_PARALLEL_MEM)
-LSCRIPT_MEM = $(RUN_FIRST); ($(ECHO_FIRST) echo $3) | $(SHELL) >& $(LOG)
-LSCRIPT_ENV_MEM = $(RUN_FIRST); ($(call ECHO_FIRST_ENV,$1) echo $4) | $(SHELL) >& $(LOG)
-LSCRIPT_SHORT = $(LSCRIPT_MEM)
-LSCRIPT_ENV_SHORT = $(LSCRIPT_ENV_MEM)
-LSCRIPT_LONG = $(LSCRIPT_MEM)
-LSCRIPT_ENV_LONG = $(LSCRIPT_ENV_MEM)
-LSCRIPT_CHECK = $(RUN_FIRST); ($(ECHO_FIRST) echo $1) | $(SHELL) >& $(LOG)
-LSCRIPT_ENV_CHECK = $(RUN_FIRST); ($(call ECHO_FIRST_ENV,$1) echo $2) | $(SHELL) >& $(LOG)
-LSCRIPT_CHECK_PARALLEL_MEM = $(RUN_FIRST) && ($(ECHO_FIRST) echo $4) | $(SHELL) >& $(LOG)
-LSCRIPT_ENV_CHECK_PARALLEL_MEM = $(RUN_FIRST) && ($(call ECHO_FIRST_ENV,$1) echo $5) | $(SHELL) >& $(LOG)
-LSCRIPT_CHECK_PARALLEL_SHORT = $(LSCRIPT_CHECK_PARALLEL_MEM)
-LSCRIPT_ENV_CHECK_PARALLEL_SHORT = $(LSCRIPT_ENV_CHECK_PARALLEL_MEM)
-LSCRIPT_CHECK_PARALLEL_LONG = $(LSCRIPT_CHECK_PARALLEL_MEM)
-LSCRIPT_ENV_CHECK_PARALLEL_LONG = $(LSCRIPT_ENV_CHECK_PARALLEL_MEM)
-LSCRIPT_CHECK_MEM = $(RUN_FIRST); ($(ECHO_FIRST) echo $3) | $(SHELL) >& $(LOG)
-LSCRIPT_ENV_CHECK_MEM = $(RUN_FIRST); ($(call ECHO_FIRST_ENV,$1) echo $4) | $(SHELL) >& $(LOG)
-LSCRIPT_CHECK_SHORT = $(LSCRIPT_CHECK_MEM)
-LSCRIPT_ENV_CHECK_SHORT = $(LSCRIPT_ENV_CHECK_MEM)
-LSCRIPT_CHECK_LONG = $(LSCRIPT_CHECK_MEM)
-LSCRIPT_ENV_CHECK_LONG = $(LSCRIPT_ENV_CHECK_MEM)
-LSCRIPT_NAMED = $(call LSCRIPT,$2)
-LSCRIPT_ENV_NAMED = $(call LSCRIPT_ENV,$1,$3)
-LSCRIPT_NAMED_PARALLEL_MEM = $(call LSCRIPT_PARALLEL_MEM,$2,$3,$4,$5)
-LSCRIPT_ENV_NAMED_PARALLEL_MEM = $(call LSCRIPT_ENV_PARALLEL_MEM,$1,$3,$4,$5,$6)
-LSCRIPT_NAMED_PARALLEL_SHORT = $(call LSCRIPT_PARALLEL_SHORT,$2,$3,$4,$5)
-LSCRIPT_ENV_NAMED_PARALLEL_SHORT = $(call LSCRIPT_ENV_PARALLEL_MEM,$1,$3,$4,$5,$6)
-LSCRIPT_NAMED_PARALLEL_LONG = $(call LSCRIPT_PARALLEL_LONG,$2,$3,$4,$5)
-LSCRIPT_ENV_NAMED_PARALLEL_LONG = $(call LSCRIPT_ENV_PARALLEL_MEM,$1,$3,$4,$5,$6)
-LSCRIPT_NAMED_MEM = $(call LSCRIPT_MEM,$2,$3,$4)
-LSCRIPT_ENV_NAMED_MEM = $(call LSCRIPT_ENV_MEM,$1,$3,$4,$5)
-LSCRIPT_NAMED_SHORT = $(LSCRIPT_NAMED_MEM)
-LSCRIPT_ENV_NAMED_SHORT = $(LSCRIPT_ENV_NAMED_MEM)
-LSCRIPT_NAMED_LONG = $(LSCRIPT_NAMED_MEM)
-LSCRIPT_ENV_NAMED_LONG = $(LSCRIPT_ENV_NAMED_MEM)
-LSCRIPT_CHECK_NAMED = $(call LSCRIPT,$2)
-LSCRIPT_ENV_CHECK_NAMED = $(call LSCRIPT_ENV,$1,$3)
-LSCRIPT_CHECK_NAMED_PARALLEL_MEM = $(call LSCRIPT_PARALLEL_MEM,$2,$3,$4,$5)
-LSCRIPT_CHECK_NAMED_PARALLEL_LONG = $(call LSCRIPT_PARALLEL_LONG,$2,$3,$4,$5)
-LSCRIPT_CHECK_NAMED_PARALLEL_SHORT = $(call LSCRIPT_PARALLEL_SHORT,$2,$3,$4,$5)
-LSCRIPT_ENV_CHECK_NAMED_PARALLEL_MEM = $(call LSCRIPT_ENV_PARALLEL_MEM,$1,$3,$4,$5,$6)
-LSCRIPT_ENV_CHECK_NAMED_PARALLEL_LONG = $(call LSCRIPT_ENV_PARALLEL_LONG,$1,$3,$4,$5,$6)
-LSCRIPT_ENV_CHECK_NAMED_PARALLEL_SHORT = $(call LSCRIPT_ENV_PARALLEL_SHORT,$1,$3,$4,$5,$6)
-LSCRIPT_CHECK_NAMED_MEM = $(call LSCRIPT_MEM,$2,$3,$4)
-LSCRIPT_CHECK_NAMED_SHORT = $(call LSCRIPT_SHORT,$2,$3,$4)
-LSCRIPT_CHECK_NAMED_LONG = $(call LSCRIPT_LONG,$2,$3,$4)
-LSCRIPT_ENV_CHECK_NAMED_MEM = $(call LSCRIPT_ENV_MEM,$1,$3,$4,$5)
-LSCRIPT_ENV_CHECK_NAMED_SHORT = $(call LSCRIPT_ENV_MEM,$1,$3,$4,$5)
-LSCRIPT_ENV_CHECK_NAMED_LONG = $(call LSCRIPT_ENV_MEM,$1,$3,$4,$5)
-else
-#$(call LS,qsub,name,qsub_args,script)
-LS = $(RUN_FIRST); (echo "umask $(UMASK); set -o pipefail; $(call ECHO_ACTIVATE_ENV,$(JRFLAB_MODULES_ENV));" $4) | $1 -N $(PROJECT_NAME)_$2 $3
-#$(call LS,env,qsub,name,qsub_args,script)
-LS_ENV = $(RUN_FIRST); (echo "umask $(UMASK); set -o pipefail; $(call ECHO_ACTIVATE_ENV,$1);" $5) | $2 -N $(PROJECT_NAME)_$3 $4
-$(foreach wt,$(WALLTIMES),\
-	$(eval LSCRIPT_$(wt) = $$(call LSCRIPT_NAMED_$(wt),$$(@F),$$1,$$2,$$3)) \
-	$(eval LSCRIPT_ENV_$(wt) = $$(call LSCRIPT_ENV_NAMED_$(wt),$$1,$$(@F),$$2,$$3,$$4)) \
-	$(eval LSCRIPT_PARALLEL_$(wt) = $$(call LSCRIPT_NAMED_PARALLEL_$(wt),$$(@F),$$1,$$2,$$3,$$4)) \
-	$(eval LSCRIPT_ENV_PARALLEL_$(wt) = $$(call LSCRIPT_ENV_NAMED_PARALLEL_$(wt),$$1,$$(@F),$$2,$$3,$$4,$$5)) \
-	$(eval LSCRIPT_NAMED_PARALLEL_$(wt) = $$(call LS,$$(QSUB),$$1,$$($$(CLUSTER_ENGINE)_QSUB_ARGS) $$(call $$(CLUSTER_ENGINE)_PARALLEL_$(wt),$$2,$$3,$$4),$$5)) \
-	$(eval LSCRIPT_ENV_NAMED_PARALLEL_$(wt) = $$(call LS_ENV,$$1,$$(QSUB),$$2,$$($$(CLUSTER_ENGINE)_QSUB_ARGS) $$(call $$(CLUSTER_ENGINE)_PARALLEL_$(wt),$$3,$$4,$$5),$$6)) \
-	$(eval LSCRIPT_NAMED_$(wt) = $$(call LS,$$(QSUB),$$1,$$($$(CLUSTER_ENGINE)_QSUB_ARGS) $$(call $$(CLUSTER_ENGINE)_$(wt),$$2,$$3),$$4)) \
-	$(eval LSCRIPT_ENV_NAMED_$(wt) = $$(call LS_ENV,$$1,$$(QSUB),$$2,$$($$(CLUSTER_ENGINE)_QSUB_ARGS) $$(call $$(CLUSTER_ENGINE)_$(wt),$$3,$$4),$$5)) \
-	$(eval LSCRIPT_CHECK_PARALLEL_$(wt) = $$(call LSCRIPT_CHECK_NAMED_PARALLEL_$(wt),$$(@F),$$1,$$2,$$3,$$4)) \
-	$(eval LSCRIPT_ENV_CHECK_PARALLEL_$(wt) = $$(call LSCRIPT_ENV_CHECK_NAMED_PARALLEL_$(wt),$$1,$$(@F),$$2,$$3,$$4,$$5)) \
-	$(eval LSCRIPT_CHECK_$(wt) = $$(call LSCRIPT_CHECK_NAMED_$(wt),$$(@F),$$1,$$2,$$3)) \
-	$(eval LSCRIPT_ENV_CHECK_$(wt) = $$(call LSCRIPT_ENV_CHECK_NAMED_$(wt),$$1,$$(@F),$$2,$$3,$$4)) \
-	$(eval LSCRIPT_DOCKER_PARALLEL_$(wt) = $$(call LSCRIPT_DOCKER_NAMED_PARALLEL_$(wt),$$(@F),$$1,$$2,$$3,$$4)) \
-	$(eval LSCRIPT_DOCKER_$(wt) = $$(call LSCRIPT_DOCKER_NAMED_$(wt),$$(@F),$$1,$$2,$$3)) \
-	$(eval LSCRIPT_CHECK_NAMED_PARALLEL_$(wt) = $$(call LS,$$(QSUB_CHECK),$$1,$$($$(CLUSTER_ENGINE)_QSUB_ARGS) $$(call $$(CLUSTER_ENGINE)_PARALLEL_$(wt),$$2,$$3,$$4),$$5)) \
-	$(eval LSCRIPT_ENV_CHECK_NAMED_PARALLEL_$(wt) = $$(call LS_ENV,$$1,$$(QSUB_CHECK),$$3,$$($$(CLUSTER_ENGINE)_QSUB_ARGS) $$(call $$(CLUSTER_ENGINE)_PARALLEL_$(wt),$$3,$$4,$$5),$$6)) \
-	$(eval LSCRIPT_DOCKER_NAMED_PARALLEL_$(wt) = $$(call LS,$$(QSUB),$$1,$$($$(CLUSTER_ENGINE)_QSUB_ARGS) $$(call $$(CLUSTER_ENGINE)_DOCKER_PARALLEL_$(wt),$$2,$$3,$$4),$$5)) \
-	$(eval LSCRIPT_CHECK_NAMED_$(wt) = $$(call LS,$$(QSUB_CHECK),$$1,$$($$(CLUSTER_ENGINE)_QSUB_ARGS) $$(call $$(CLUSTER_ENGINE)_$(wt),$$2,$$3),$$4)) \
-	$(eval LSCRIPT_ENV_CHECK_NAMED_$(wt) = $$(call LS_ENV,$$1,$$(QSUB_CHECK),$$2,$$($$(CLUSTER_ENGINE)_QSUB_ARGS) $$(call $$(CLUSTER_ENGINE)_$(wt),$$3,$$4),$$5)) \
-	$(eval LSCRIPT_DOCKER_NAMED_$(wt) = $$(call LS,$$(QSUB_CHECK),$$1,$$($$(CLUSTER_ENGINE)_QSUB_ARGS) $$(call $$(CLUSTER_ENGINE)_DOCKER_$(wt),$$2,$$3),$$4)))
-
-LSCRIPT = $(call LSCRIPT_NAMED_MEM,$(@F),1G,2G,$1)
-LSCRIPT_ENV = $(call LSCRIPT_ENV_NAMED_MEM,$1,$(@F),1G,2G,$2)
-LSCRIPT_NAMED = $(call LS,$(QSUB),$1,$($(CLUSTER_ENGINE)_QSUB_ARGS) $(call $(CLUSTER_ENGINE)_MEM,1G,2G),$2)
-LSCRIPT_ENV_NAMED = $(call LS_ENV,$1,$(QSUB),$2,$($(CLUSTER_ENGINE)_QSUB_ARGS) $(call $(CLUSTER_ENGINE)_MEM,1G,2G),$3)
-
-LSCRIPT_CHECK = $(call LSCRIPT_CHECK_NAMED,$(@F),$1)
-LSCRIPT_ENV_CHECK = $(call LSCRIPT_ENV_CHECK_NAMED,$1,$(@F),$2)
-
-LSCRIPT_CHECK_NAMED = $(call LS,$(QSUB_CHECK),$1,$($(CLUSTER_ENGINE)_QSUB_ARGS) $(call $(CLUSTER_ENGINE)_MEM,1G,2G),$2)
-LSCRIPT_ENV_CHECK_NAMED = $(call LS_ENV,$1,$(QSUB_CHECK),$2,$($(CLUSTER_ENGINE)_QSUB_ARGS) $(call $(CLUSTER_ENGINE)_MEM,1G,2G),$3)
-endif
+
+RUN_OPTS = $(if $(findstring false,$(USE_CLUSTER)),-l) -o $@ $(if $(CLUSTER_ENGINE),-g $(CLUSTER_ENGINE)) \
+		   --default_env $(JRFLAB_MODULES_ENV) -S $(SHELL) \
+		   --log_file $(LOG) $(if $($(CLUSTER_ENGINE)_NODES),--servers $($(CLUSTER_ENGINE)_NODES)) \
+		   $(if $(PROJECT_NAME),-p $(PROJECT_NAME))
+#usage $(call RUN,args,script)
+RUN = echo $2 | python modules/scripts/run.py $(RUN_OPTS) $1
 
 #MD5 = md5sum $(1:.md5=) > $1
 MD5 = md5sum $(@:.md5=) > $@
@@ -385,10 +249,13 @@ ifeq ($(USE_CLUSTER),true)
 ifeq ($(CLUSTER_ENGINE),SGE)
 export DRMAA_LIBRARY_PATH = /common/sge/lib/lx24-amd64/libdrmaa.so.1.0
 else
+ifeq ($(CLUSTER_ENGINE),LSF)
+export DRMAA_LIBRARY_PATH = /admin/lsf/10.1/linux3.10-glibc2.17-x86_64/lib/libdrmaa.so
+else
 export DRMAA_LIBRARY_PATH = /opt/torque/lib/libdrmaa.so.1
 endif
 endif
-
+endif
 
 endif
 MAKEFILE_INC = true

aligners/align.mk

@@ -9,7 +9,7 @@ $(foreach sample,$(SAMPLES),\
 
 define merged-bam
 $$(ALIGNER)/bam/$1.$$(ALIGNER).sorted.bam : $$(ALIGNER)/sam/$1.header.sam $$(foreach split,$2,$$(ALIGNER)/bam/$$(split).$$(ALIGNER).sorted.bam)
-	$$(call LSCRIPT_MEM,12G,15G,"$$(SAMTOOLS) merge -f -h $$< $$(@) $$(filter %.bam,$$^) && $$(RM) $$^")
+	$$(call RUN,-s 12G -m 15G,"$$(SAMTOOLS) merge -f -h $$< $$(@) $$(filter %.bam,$$^) && $$(RM) $$^")
 endef
 define rename-bam
 $$(ALIGNER)/bam/$1.$$(ALIGNER).bam : $$(ALIGNER)/bam/$2.$$(ALIGNER).bam

aligners/bowtieAligner.mk

@@ -41,22 +41,22 @@ bowtie_bams : $(BAMS) $(addsuffix .bai,$(BAMS))
 
 # memory for human genome: ~3.2G
 bowtie/bam/%.bwt.bam : fastq/%.1.fastq.gz fastq/%.2.fastq.gz
-	$(call LSCRIPT_PARALLEL_MEM,4,1G,1.5G,"LBID=`echo \"$*\" | sed 's/_[A-Za-z0-9]\+//'`; \
+	$(call RUN,-n 4 -s 1G -m 1.5G,"LBID=`echo \"$*\" | sed 's/_[A-Za-z0-9]\+//'`; \
 		$(BOWTIE) $(BOWTIE_OPTS) --rg-id $* --rg \"LB:\$${LBID}\" --rg \"PL:${SEQ_PLATFORM}\" --rg \"SM:\$${LBID}\" -p $(NUM_CORES) \
 		-1 $< -2 $(<<) | $(SAMTOOLS) view -bhS - > $(@)")
 
 bowtie/bam/%.bwt.bam : fastq/%.fastq.gz
-	$(call LSCRIPT_PARALLEL_MEM,4,1G,1.5G,"LBID=`echo \"$*\" | sed 's/_[A-Za-z0-9]\+//'`; \
+	$(call RUN,-n 4 -s 1G -m 1.5G,"LBID=`echo \"$*\" | sed 's/_[A-Za-z0-9]\+//'`; \
 		$(BOWTIE) $(BOWTIE_OPTS) --rg-id $* --rg \"LB:\$${LBID}\" --rg \"PL:${SEQ_PLATFORM}\" --rg \"SM:\$${LBID}\" -p $(NUM_CORES) \
 		-U $(<) | $(SAMTOOLS) view -bhS - > $(@) ")
 
 
 bam/%.bam : bowtie/bam/%.bwt.$(BAM_SUFFIX)
-	$(call LSCRIPT,"ln -f $(<) $(@) ")
+	$(call RUN,,"ln -f $(<) $(@) ")
 
 define align-split-fastq
 bowtie/bam/$2.bwt.bam : $3
-	$$(call LSCRIPT_PARALLEL_MEM,4,1G,1.5G,"$$(BOWTIE) $$(BOWTIE_OPTS) \
+	$$(call RUN,-n 4 -s 1G -m 1.5G,"$$(BOWTIE) $$(BOWTIE_OPTS) \
 		--rg-id $2 --rg \"LB:$1\" --rg \"PL:$${SEQ_PLATFORM}\" --rg \"SM:$1\" \
 		-p $$(NUM_CORES) $$(if $$(<<),-1 $$(<) -2 $$(<<),-U $$<) | $$(SAMTOOLS) view -bhS - > $$(@)")
 endef

aligners/bwaAligner.mk

@@ -34,14 +34,14 @@ BWA_BAMS = $(foreach sample,$(SAMPLES),bam/$(sample).bam)
 bwa : $(addsuffix ,$(BWA_BAMS)) $(addsuffix .bai,$(BWA_BAMS))
 
 bam/%.bam : bwa/bam/%.bwa.$(BAM_SUFFIX)
-	$(call LSCRIPT,"ln -f $(<) $(@) ")
+	$(call RUN,,"ln -f $(<) $(@) ")
 
 ifdef SPLIT_SAMPLES
 
 .SECONDEXPANSION:
 define sai-split-fastq-pair
 bwa/sai/$1.$3.sai : $2
-	$$(call LSCRIPT_PARALLEL_MEM,8,1G,1.2G,"$$(BWA) aln $$(BWA_ALN_OPTS) -t 8 $$(REF_FASTA) $$(<) > $$(@)")
+	$$(call RUN,-n 8 -s 1G -m 1.2G,"$$(BWA) aln $$(BWA_ALN_OPTS) -t 8 $$(REF_FASTA) $$(<) > $$(@)")
 endef
 $(foreach ss,$(SPLIT_SAMPLES), \
 	$(if $(fq.$(ss)),\
@@ -50,22 +50,22 @@ $(foreach ss,$(SPLIT_SAMPLES), \
 
 define align-split-fastq
 bwa/bam/$2.bwa.bam : bwa/sai/$2.1.sai bwa/sai/$2.2.sai $3
-	$$(call LSCRIPT_MEM,4G,10G,"$$(BWA) sampe -P -r \"@RG\tID:$2\tLB:$1\tPL:$${SEQ_PLATFORM}\tSM:$1\" $$(REF_FASTA) $$^ | $$(SAMTOOLS) view -bhS - > $$@")
+	$$(call RUN,-s 4G -m 10G,"$$(BWA) sampe -P -r \"@RG\tID:$2\tLB:$1\tPL:$${SEQ_PLATFORM}\tSM:$1\" $$(REF_FASTA) $$^ | $$(SAMTOOLS) view -bhS - > $$@")
 endef
 $(foreach ss,$(SPLIT_SAMPLES),\
 	$(if $(fq.$(ss)),\
 	$(eval $(call align-split-fastq,$(split.$(ss)),$(ss),$(fq.$(ss))))))
 endif
 
 bwa/sai/%.sai : fastq/%.fastq.gz
-	$(call LSCRIPT_PARALLEL_MEM,8,1G,1.2G,"$(BWA) aln $(BWA_ALN_OPTS) -t 8 $(REF_FASTA) $(<) > $(@) ")
+	$(call RUN,-n 8 -s 1G -m 1.2G,"$(BWA) aln $(BWA_ALN_OPTS) -t 8 $(REF_FASTA) $(<) > $(@) ")
 
 bwa/bam/%.bwa.bam : bwa/sai/%.1.sai bwa/sai/%.2.sai fastq/%.1.fastq.gz fastq/%.2.fastq.gz
 	LBID=`echo "$*" | sed 's/_[A-Za-z0-9]\+//'`; \
-	$(call LSCRIPT_MEM,4G,10G,"$(BWA) sampe -P -r \"@RG\tID:$*\tLB:$${LBID}\tPL:${SEQ_PLATFORM}\tSM:$${LBID}\" $(REF_FASTA) $(^) | $(SAMTOOLS) view -bhS - > $(@) ")
+	$(call RUN,-s 4G -m 10G,"$(BWA) sampe -P -r \"@RG\tID:$*\tLB:$${LBID}\tPL:${SEQ_PLATFORM}\tSM:$${LBID}\" $(REF_FASTA) $(^) | $(SAMTOOLS) view -bhS - > $(@) ")
 
 fastq/%.fastq.gz : fastq/%.fastq
-	$(call LSCRIPT,"gzip -c $< > $(@) && $(RM) $< ")
+	$(call RUN,,"gzip -c $< > $(@) && $(RM) $< ")
 
 
 include modules/bam_tools/processBam.mk

aligners/bwamemAligner.mk

@@ -39,25 +39,25 @@ BWA_BAMS = $(foreach sample,$(SAMPLES),bam/$(sample).bam)
 bwamem : $(BWA_BAMS) $(addsuffix .bai,$(BWA_BAMS))
 
 bam/%.bam : bwamem/bam/%.bwamem.$(BAM_SUFFIX)
-	$(call LSCRIPT,"ln -f $(<) $(@)")
+	$(call RUN,,"ln -f $(<) $(@)")
 
 #$(call align-split-fastq,name,split-name,fastqs)
 define align-split-fastq
 bwamem/bam/$2.bwamem.bam : $3
-	$$(call LSCRIPT_PARALLEL_MEM,$$(BWAMEM_THREADS),1G,$$(BWAMEM_MEM_PER_THREAD),"$$(BWA) mem -t $$(BWAMEM_THREADS) $$(BWA_ALN_OPTS) -R \"@RG\tID:$2\tLB:$1\tPL:$${SEQ_PLATFORM}\tSM:$1\" $$(BWAMEM_REF_FASTA) $$^ | $$(SAMTOOLS) view -bhS - > $$@")
+	$$(call RUN,-n $$(BWAMEM_THREADS) -s 1G -m $$(BWAMEM_MEM_PER_THREAD),"$$(BWA) mem -t $$(BWAMEM_THREADS) $$(BWA_ALN_OPTS) -R \"@RG\tID:$2\tLB:$1\tPL:$${SEQ_PLATFORM}\tSM:$1\" $$(BWAMEM_REF_FASTA) $$^ | $$(SAMTOOLS) view -bhS - > $$@")
 endef
 $(foreach ss,$(SPLIT_SAMPLES),$(if $(fq.$(ss)),$(eval $(call align-split-fastq,$(split.$(ss)),$(ss),$(fq.$(ss))))))
 
 bwamem/bam/%.bwamem.bam : fastq/%.1.fastq.gz fastq/%.2.fastq.gz
 	LBID=`echo "$*" | sed 's/_[A-Za-z0-9]\+//'`; \
-	$(call LSCRIPT_PARALLEL_MEM,$(BWAMEM_THREADS),1G,$(BWAMEM_MEM_PER_THREAD),"$(BWA) mem -t $(BWAMEM_THREADS) $(BWA_ALN_OPTS) -R \"@RG\tID:$*\tLB:$${LBID}\tPL:${SEQ_PLATFORM}\tSM:$${LBID}\" $(BWAMEM_REF_FASTA) $^ | $(SAMTOOLS) view -bhS - > $@")
+	$(call RUN,-n $(BWAMEM_THREADS) -s 1G -m $(BWAMEM_MEM_PER_THREAD),"$(BWA) mem -t $(BWAMEM_THREADS) $(BWA_ALN_OPTS) -R \"@RG\tID:$*\tLB:$${LBID}\tPL:${SEQ_PLATFORM}\tSM:$${LBID}\" $(BWAMEM_REF_FASTA) $^ | $(SAMTOOLS) view -bhS - > $@")
 
 bwamem/bam/%.bwamem.bam : fastq/%.fastq.gz
 	LBID=`echo "$*" | sed 's/_[A-Za-z0-9]\+//'`; \
-	$(call LSCRIPT_PARALLEL_MEM,$(BWAMEM_THREADS),1G,$(BWAMEM_MEM_PER_THREAD),"$(BWA) mem -t $(BWAMEM_THREADS) $(BWA_ALN_OPTS) -R \"@RG\tID:$*\tLB:$${LBID}\tPL:${SEQ_PLATFORM}\tSM:$${LBID}\" $(BWAMEM_REF_FASTA) $^ | $(SAMTOOLS) view -bhS - > $@")
+	$(call RUN,-n $(BWAMEM_THREADS) -s 1G -m $(BWAMEM_MEM_PER_THREAD),"$(BWA) mem -t $(BWAMEM_THREADS) $(BWA_ALN_OPTS) -R \"@RG\tID:$*\tLB:$${LBID}\tPL:${SEQ_PLATFORM}\tSM:$${LBID}\" $(BWAMEM_REF_FASTA) $^ | $(SAMTOOLS) view -bhS - > $@")
 
 fastq/%.fastq.gz : fastq/%.fastq
-	$(call LSCRIPT,"gzip -c $< > $(@) && $(RM) $<")
+	$(call RUN,,"gzip -c $< > $(@) && $(RM) $<")
 
 include modules/bam_tools/processBam.mk
 include modules/fastq_tools/fastq.mk

aligners/hisatAligner.mk

@@ -37,7 +37,7 @@ bam/%.bam : hisat/bam/%.hisat.$(BAM_SUFFIX)
 	$(INIT) ln -f $(<) $(@) 
 
 hisat/bam/%.hisat.bam hisat/unmapped_bam/%.hisat_unmapped.bam : fastq/%.1.fastq.gz fastq/%.2.fastq.gz
-	$(call LSCRIPT_NAMED_PARALLEL_MEM,$*_hisat,8,1G,1.5G,"mkdir -p hisat/bam hisat/unmapped_bam; \
+	$(call RUN,-N $*_hisat -n 8 -s 1G -m 1.5G,"mkdir -p hisat/bam hisat/unmapped_bam; \
 		LBID=`echo \"$*\" | sed 's/_.\+//'`; \
 		$(HISAT) $(HISAT_OPTS) \
 		--rg-id $* --rg \"SM:\$${LBID}\" \
@@ -49,7 +49,7 @@ hisat/bam/%.hisat.bam hisat/unmapped_bam/%.hisat_unmapped.bam : fastq/%.1.fastq.
 # usage $(eval $(call align-merged-split-fastq,merged-sample,split,fastq(s)))
 define align-merged-split-fastq
 hisat/bam/$2.hisat.bam : $3
-	$$(call LSCRIPT_NAMED_PARALLEL_MEM,$2_hisat,8,1G,1.5G,"mkdir -p hisat/bam hisat/unmapped_bam; \
+	$$(call RUN,-N $2_hisat -n 8 -s 1G -m 1.5G,"mkdir -p hisat/bam hisat/unmapped_bam; \
 		$$(HISAT) $$(HISAT_OPTS) \
 		--rg-id $2 --rg \"SM:$1\" \
 		--rg PL:$${SEQ_PLATFORM} --rg \"LB:\$1\" \

aligners/starAligner.mk

@@ -34,7 +34,7 @@ star_chimeric_junctions : $(foreach sample,$(SAMPLES),star/$(sample).Chimeric.ou
 
 define align-split-fastq
 star/$2.star_align_timestamp : $3
-	$$(call LSCRIPT_PARALLEL_MEM,4,6G,10G,"$$(STAR) $$(STAR_OPTS) \
+	$$(call RUN,-n 4 -s 6G -m 10G,"$$(STAR) $$(STAR_OPTS) \
 		--outFileNamePrefix star/$2. --runThreadN 4 \
 		--outSAMattrRGline \"ID:$2\" \"LB:$1\" \"SM:$1\" \"PL:$${SEQ_PLATFORM}\" \
 		--readFilesIn $$^ --readFilesCommand zcat && touch $$@")
@@ -46,7 +46,7 @@ $(foreach ss,$(SPLIT_SAMPLES),\
 	$(eval $(call align-split-fastq,$(split.$(ss)),$(ss),$(fq.$(ss))))))
 
 star/%.Aligned.sortedByCoord.out.bam star/%.Chimeric.out.junction : fastq/%.1.fastq.gz fastq/%.2.fastq.gz
-	$(call LSCRIPT_PARALLEL_MEM,4,6G,10G,"$(STAR) $(STAR_OPTS) \
+	$(call RUN,-n 4 -s 6G -m 10G,"$(STAR) $(STAR_OPTS) \
 		--outFileNamePrefix star/$*. --runThreadN 4 \
 		--outSAMattrRGline \"ID:$*\" \"LB:$*\" \"SM:$*\" \"PL:${SEQ_PLATFORM}\" \
 		--readFilesIn $^ --readFilesCommand zcat")