pull in jrflab updates in the last 1-2 years

Makefile.inc

@@ -99,7 +99,13 @@ CREATE_SEQ_DICT = $(JAVA) -Xmx$(PICARD_MEM) -jar $(PICARD_DIR)/CreateSequenceDic
 CALC_HS_METRICS = $(JAVA) -Xmx$(PICARD_MEM) -jar $(PICARD_DIR)/CalculateHsMetrics.jar $(PICARD_OPTS)
 COLLECT_MULT_METRICS = $(JAVA) -Xmx$(PICARD_MEM) -jar $(PICARD_DIR)/CollectMultipleMetrics.jar $(PICARD_OPTS)
 COLLECT_TARGETED_METRICS = $(JAVA) -Xmx$(PICARD_MEM) -jar $(PICARD_DIR)/CollectTargetedPcrMetrics.jar $(PICARD_OPTS)
-
+COLLECT_ALIGNMENT_METRICS = $(PICARD) -Xmx$(PICARD_MEM) CollectAlignmentSummaryMetrics $(PICARD_OPTS)
+COLLECT_INSERT_METRICS = $(PICARD) -Xmx$(PICARD_MEM) CollectInsertSizeMetrics $(PICARD_OPTS)
+COLLECT_OXOG_METRICS = $(PICARD) -Xmx$(PICARD_MEM) CollectOxoGMetrics $(PICAD_OPTS)
+COLLECT_GC_BIAS = $(PICARD) -Xmx$(PICARD_MEM) CollectGcBiasMetrics $(PICARD_OPTS)
+COLLECT_WGS_METRICS = $(PICARD) -Xmx$(PICARD_MEM) CollectWgsMetrics $(PICARD_OPTS)
+COLLECT_DUP_METRICS = $(PICARD) -Xmx$(PICARD_MEM) CollectDuplicateMetrics $(PICARD_OPTS)
+BAM_INDEX = $(PICARD) -Xmx$(PICARD_MEM) BamIndexStats $(PICARD_OPTS)
 FIX_MATE = $(call FIX_MATE_MEM,$(PICARD_MEM))
 FIX_MATE_MEM = $(JAVA) -Xmx$(1) -jar $(PICARD_DIR)/FixMateInformation.jar $(PICARD_OPTS) TMP_DIR=$(TMPDIR)
 SAM_TO_FASTQ = $(call SAM_TO_FASTQ_MEM,$(PICARD_MEM))
@@ -120,7 +126,7 @@ SNP_EFF_MEM = $(JAVA8) -Xmx$1 -jar $(SNP_EFF_JAR)
 SNP_SIFT_MEM = $(JAVA8) -Xmx$1 -jar $(SNP_SIFT_JAR)
 SNP_SIFT = $(call SNP_SIFT_MEM,$(DEFAULT_JAVA_MEM))
 VCF_EFF_ONE_PER_LINE = $(HOME)/share/usr/snpEff-4.1/scripts/vcfEffOnePerLine.pl
-VCF_JOIN_EFF = modules/scripts/joinEff.pl
+VCF_JOIN_EFF = modules/scripts/join_eff.pl
 
 COUNT_SAMPLES = $(shell expr `sed 's/\..*//; s:.*/::' <<< $1 | grep -o "_" | wc -l` + 1)
 

README.md

@@ -1,61 +1,2 @@
-# jrflab modules
-[![Build Status](https://travis-ci.org/cBioPortal/cbioportal.svg?branch=master)](https://travis-ci.org/jrflab/modules)
+# modules
 
-## Introduction
-This is the implementation of the jrflab pipeline.
-
-## Installation
-The easiest way to download this pipeline is to clone the repository.
-
-```
-git clone https://github.com/jrflab/modules.git
-```
-
-## Dependencies
-- An instance of [anaconda](https://www.anaconda.com) or [miniconda](https://conda.io/en/latest/miniconda.html)
-- IMB's Platform Load Sharing Facility (LSF) or Oracle's Sun Grid Engine (SGE) for resource management
-
-### Following R Packages
-- [xxx](https://)
-
-## Best practices
-
-### Conventions
-- Sample names cannot have "/" or "." in them
-- Fastq files end in ".fastq.gz"
-- Fastq files are stored in DATA_DIR (Set as Environment Variable) 
-
-### Whole genome, whole exome and targeted sequencing
-- QC
-- BWA
-- Broad Standard Practices on bwa bam  
-- Haplotype Caller, Platypus, MuTect, Strelka
-- snpEff, Annovar, SIFT, pph2, vcf2maf, VEP, OncoKB, ClinVar
-- Copy number, tumor purity using Facets
-- Contamination using 
-- HLA Typing
-	* [xxx](http://)
-
-### RNA transcriptome sequencing
-- QC
-- Tophat, STAR
-- Cufflinks (ENS and UCSC)
-- In-house Exon Expression (ENS and UCSC)
-- fusion-catcher, tophat-fusion, deFuse
-- OncoFuse actionable fusion classification
-
-### Patient:
-- Genotyping On Patient. 
-	1000g sites are evaluated for every library and then compared (all vs all)
-	If two libraries come from a patient the match should be pretty good >80%
-- Still to develop:
-	If the match is below a certain threshold, break the pipeline for patient
-
-## Detailed usage
-[wiki](https://github.com/jrflab/modules/wiki)
-
-## Known issues
-
-### Known bugs
-
-### Currently under development

aligners/bwamemAligner.mk

@@ -21,12 +21,6 @@ BWAMEM_REF_FASTA ?= $(REF_FASTA)
 BWAMEM_THREADS = 8
 BWAMEM_MEM_PER_THREAD = $(if $(findstring true,$(PDX)),4G,2G)
 
-..DUMMY := $(shell mkdir -p version; $(BWA) &> version/bwamem.txt; echo "options: $(BWA_ALN_OPTS)" >> version/bwamem.txt )
-.SECONDARY:
-.DELETE_ON_ERROR: 
-.PHONY: bwamem
-
-
 BWA_BAMS = $(foreach sample,$(SAMPLES),bam/$(sample).bam)
 
 bwamem : $(BWA_BAMS) $(addsuffix .bai,$(BWA_BAMS))
@@ -50,6 +44,12 @@ bwamem/bam/%.bwamem.bam : fastq/%.fastq.gz
 
 fastq/%.fastq.gz : fastq/%.fastq
 	$(call RUN,,"gzip -c $< > $(@) && $(RM) $<")
+
+
+..DUMMY := $(shell mkdir -p version; $(BWA) &> version/bwamem.txt; echo "options: $(BWA_ALN_OPTS)" >> version/bwamem.txt )
+.SECONDARY:
+.DELETE_ON_ERROR: 
+.PHONY: bwamem
 
 include modules/bam_tools/processBam.mk
 include modules/fastq_tools/fastq.mk

aligners/tmapAligner.mk

@@ -5,6 +5,7 @@ include modules/aligners/align.inc
 ALIGNER := tmap
 LOGDIR := log/tmap.$(NOW)
 
+
 SAMTOOLS_SORT_MEM = 2000000000
 
 FASTQ_CHUNKS := 10

bam_tools/fix_bam.mk

@@ -0,0 +1,75 @@
+include modules/Makefile.inc
+include modules/genome_inc/b37.inc
+
+LOGDIR = log/fix_bam.$(NOW)
+
+PICARD_JAR = ~/share/usr/picard/bin/picard.jar
+
+fix_bam : $(foreach sample,$(SAMPLES),fixed_bam/$(sample).bam)
+
+define fix-bam
+unprocessed_bam/$1.ubam : unprocessed_bam/$1.bam
+	$$(call RUN,-c -n 1 -s 12G -m 18G -w 72:00:00,"java -Djava.io.tmpdir=$(TMPDIR) -Xmx16G -jar $$(PICARD_JAR) RevertSam \
+						       I=$$(<) \
+						       O=$$(@) \
+						       SANITIZE=true \
+						       MAX_DISCARD_FRACTION=0.005 \
+						       ATTRIBUTE_TO_CLEAR=XT \
+						       ATTRIBUTE_TO_CLEAR=XN \
+						       ATTRIBUTE_TO_CLEAR=AS \
+						       ATTRIBUTE_TO_CLEAR=OC \
+						       ATTRIBUTE_TO_CLEAR=OP \
+						       SORT_ORDER=queryname \
+						       RESTORE_ORIGINAL_QUALITIES=true \
+						       REMOVE_DUPLICATE_INFORMATION=true \
+						       REMOVE_ALIGNMENT_INFORMATION=true \
+						       TMP_DIR=$(TMPDIR)")
+
+unprocessed_bam/$1.fixed.bam : unprocessed_bam/$1.bam unprocessed_bam/$1.ubam
+	$$(call RUN, -c -n 1 -s 12G -m 18G -w 72:00:00,"java -Djava.io.tmpdir=$(TMPDIR) -Xmx16G -jar $$(PICARD_JAR) MergeBamAlignment \
+							R=$$(DMP_FASTA) \
+							ALIGNED_BAM=$$(<) \
+							UNMAPPED_BAM=$$(<<) \
+							O=$$(@) \
+							CREATE_INDEX=true \
+							ADD_MATE_CIGAR=true \
+							CLIP_ADAPTERS=true \
+							CLIP_OVERLAPPING_READS=true \
+							INCLUDE_SECONDARY_ALIGNMENTS=false \
+							MAX_INSERTIONS_OR_DELETIONS=-1 \
+							TMP_DIR=$(TMPDIR)")
+
+unprocessed_bam/$1.dedup.bam : unprocessed_bam/$1.fixed.bam
+	$$(call RUN, -c -n 1 -s 12G -m 18G -w 72:00:00,"java -Djava.io.tmpdir=$$(TMPDIR) -Xmx16G -jar $$(PICARD_JAR) MarkDuplicates \
+							I=$$(<) \
+							O=$$(@) \
+							M=unprocessed_bam/$1.txt \
+							TMP_DIR=$$(TMPDIR)")
+
+fixed_bam/$1.bam : unprocessed_bam/$1.dedup.bam
+	$$(call RUN, -c -n 1 -s 12G -m 18G -w 72:00:00,"java -Djava.io.tmpdir=$(TMPDIR) -Xmx16G -jar $$(PICARD_JAR) AddOrReplaceReadGroups \
+							I=$$(<) \
+							O=$$(@) \
+							RGID=$1 \
+							RGLB=$1 \
+							RGPL=illumina \
+							RGPU=NA \
+							RGSM=$1 \
+							TMP_DIR=$(TMPDIR) && \
+							samtools index $$(@) && \
+							cp fixed_bam/$1.bam.bai fixed_bam/$1.bai && \
+							rm -rf unprocessed_bam/$1.ubam && \
+							rm -rf unprocessed_bam/$1.fixed.bam && \
+							rm -rf unprocessed_bam/$1.dedup.bam && \
+							rm -rf unprocessed_bam/$1.fixed.bai && \
+							rm -rf unprocessed_bam/$1.dedup.bai && \
+							rm -rf unprocessed_bam/$1.txt")
+endef
+ $(foreach sample,$(SAMPLES),\
+		$(eval $(call fix-bam,$(sample))))
+
+..DUMMY := $(shell mkdir -p version; \
+             echo "picard" > version/fix_bam.txt)
+.SECONDARY: 
+.DELETE_ON_ERROR:
+.PHONY: fix_bam

bam_tools/fixRG.mk → bam_tools/fix_rg.mk

@@ -2,13 +2,13 @@ include modules/Makefile.inc
 include modules/variant_callers/gatk.inc
 include modules/aligners/align.inc
 
-LOGDIR ?= log/fixRG.$(NOW)
+LOGDIR ?= log/fix_rg.$(NOW)
 
 BAMS = $(foreach sample,$(SAMPLES),bam/$(sample).bam)
+
 fixed_bams : $(BAMS) $(addsuffix .bai,$(BAMS))
 
 bam/%.bam : unprocessed_bam/%.rg.bam
 	$(INIT) ln -f $(<) $(@)
 
-
 include modules/bam_tools/processBam.mk

bam_tools/get_bam_data_mirror.mk

@@ -0,0 +1,36 @@
+include modules/Makefile.inc
+
+LOGDIR = log/getbam_data_mirror.$(NOW)
+
+get_bam : $(foreach sample,$(SAMPLES),bam/$(sample).bam) \
+	  $(foreach sample,$(SAMPLES),bam/$(sample).bam.bai) \
+	  $(foreach sample,$(SAMPLES),bam/$(sample).bai)
+
+PROJECT_NAME = $(shell basename $(PWD))
+
+define get-bam
+bam/$1.bam :
+	$$(call RUN,-c -n 1 -s 2G -m 4G, "set -o pipefail && \
+					  rsync -aP -e ssh $(USER)@lilac-xfer01.mskcc.org:/oscar/warm/reis-filho/by_user/$(USER)/$(PROJECT_NAME)/$1.bam \
+					  bam/")
+
+bam/$1.bam.bai :
+	$$(call RUN,-c -n 1 -s 2G -m 4G, "set -o pipefail && \
+					  rsync -aP -e ssh $(USER)@lilac-xfer01.mskcc.org:/oscar/warm/reis-filho/by_user/$(USER)/$(PROJECT_NAME)/$1.bam.bai \
+					  bam/")
+
+bam/$1.bai :
+	$$(call RUN,-c -n 1 -s 2G -m 4G, "set -o pipefail && \
+					  rsync -aP -e ssh $(USER)@lilac-xfer01.mskcc.org:/oscar/warm/reis-filho/by_user/$(USER)/$(PROJECT_NAME)/$1.bai \
+					  bam/")
+
+
+endef
+ $(foreach sample,$(SAMPLES),\
+		$(eval $(call get-bam,$(sample))))
+
+..DUMMY := $(shell mkdir -p version; \
+             which scp > version/getbam_data_mirror.txt)
+.SECONDARY: 
+.DELETE_ON_ERROR:
+.PHONY: get_bam

bam_tools/get_bam_irb_mirror.mk

@@ -0,0 +1,32 @@
+include modules/Makefile.inc
+
+LOGDIR = log/getbam_irb_mirror.$(NOW)
+
+get_bam : $(foreach sample,$(SAMPLES),bam/$(sample).bam) \
+	  $(foreach sample,$(SAMPLES),bam/$(sample).bam.bai) \
+	  $(foreach sample,$(SAMPLES),bam/$(sample).bai)
+
+define get-bam
+bam/$1.bam :
+	$$(call RUN,-c -n 1 -s 2G -m 4G, "set -o pipefail && \
+					  scp $(USER)@juno-xfer01.mskcc.org:/juno/dmp/share/irb12_245/`echo $1 | cut -c 1-1`/`echo $1 | cut -c 2-2`/$1.bam \
+					  bam/")
+
+bam/$1.bam.bai : bam/$1.bam
+	$$(call RUN,-c -n 1 -s 2G -m 4G, "set -o pipefail && \
+					  $(SAMTOOLS) index $$(<)")
+
+bam/$1.bai : bam/$1.bam bam/$1.bam.bai
+	$$(call RUN,-c -n 1 -s 2G -m 4G, "set -o pipefail && \
+					  cp $$(<<) $$(@)")
+
+
+endef
+ $(foreach sample,$(SAMPLES),\
+		$(eval $(call get-bam,$(sample))))
+
+..DUMMY := $(shell mkdir -p version; \
+             which scp > version/getbam_irb_mirror.txt)
+.SECONDARY: 
+.DELETE_ON_ERROR:
+.PHONY: get_bam

bam_tools/mergeBam.mk → bam_tools/merge_bam.mk

@@ -2,10 +2,6 @@ include modules/Makefile.inc
 
 LOGDIR = log/merge.$(NOW)
 
-.SECONDARY:
-.DELETE_ON_ERROR: 
-.PHONY : merged_bam
-
 merged_bam : $(foreach sample,$(MERGE_SAMPLES),bam/$(sample).bam bam/$(sample).bam.bai)
 
 define merged-bam
@@ -32,4 +28,8 @@ $(foreach sample,$(MERGE_SAMPLES),\
 bam/%.bam : merged_bam/%.rg.bam
 	$(INIT) ln -f $< $@
 
+.SECONDARY:
+.DELETE_ON_ERROR: 
+.PHONY : merged_bam
+
 include modules/bam_tools/processBam.mk