From 48d82d3936165c5129dd5d4b22bf5b5a77305117 Mon Sep 17 00:00:00 2001 From: Devon Ryan Date: Fri, 7 Apr 2017 17:43:15 +0200 Subject: [PATCH 01/40] Implement #504 --- deeptools/getScaleFactor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deeptools/getScaleFactor.py b/deeptools/getScaleFactor.py index 7779f35804..0afd7cb145 100644 --- a/deeptools/getScaleFactor.py +++ b/deeptools/getScaleFactor.py @@ -168,7 +168,7 @@ def get_scale_factor(args): bam_mapped, bam_mapped_total = get_num_kept_reads(args) if args.normalizeTo1x: # Print output, since normalzation stuff isn't printed to stderr otherwise - sys.stderr.write("normalization: 1x\n") + sys.stderr.write("normalization: 1x (effective genome size {})\n".format(args.normalizeTo1x)) # try to guess fragment length if the bam file contains paired end reads from deeptools.getFragmentAndReadSize import get_read_and_fragment_length From f3914217235c4041129412771f5d0d017e3ce514 Mon Sep 17 00:00:00 2001 From: dpryan79 Date: Thu, 13 Apr 2017 14:05:24 +0200 Subject: [PATCH 02/40] Copy over universal line endings support in the intervals. --- deeptoolsintervals/parse.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/deeptoolsintervals/parse.py b/deeptoolsintervals/parse.py index cafaf3fad9..94f4edb590 100644 --- a/deeptoolsintervals/parse.py +++ b/deeptoolsintervals/parse.py @@ -94,14 +94,19 @@ def openPossiblyCompressed(fname): """ A wrapper to open gzip/bzip/uncompressed files """ + mode = "rU" + modeb = "rbU" + if sys.version_info[0] >= 3: + mode = "r" + modeb = "rb" with open(fname, "rb") as f: first3 = bytes(f.read(3)) if first3 == b"\x1f\x8b\x08": - return gzip.open(fname, "rb") + return gzip.open(fname, modeb) elif first3 == b"\x42\x5a\x68" and supportsBZ2: - return bz2.BZ2File(fname, "rb") + return bz2.BZ2File(fname, modeb) else: - return open(fname) + return open(fname, mode) def getLabel(line): From 437d41db8a28e0a8757687959b549219282ffb47 Mon Sep 17 00:00:00 2001 From: dpryan79 Date: Thu, 13 Apr 2017 14:06:29 +0200 Subject: [PATCH 03/40] Update CHANGES.txt --- CHANGES.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGES.txt b/CHANGES.txt index ae4819ebfc..f26b968d67 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,3 +1,7 @@ +2.5.1 + + * Added universal new line support to deeptoolsintervals (issue #506). + 2.5.0 * Fix a bug where using regions with the same name in multiple BED files in computeMatrix caused downstream problems in plotHeatmap/plotProfile (issue #477). From 0524ce273b5a010e1797582ae2fa7d1943d00804 Mon Sep 17 00:00:00 2001 From: dpryan79 Date: Thu, 13 Apr 2017 14:23:19 +0200 Subject: [PATCH 04/40] Peg to pysam 0.10.0 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 277d192e4c..d086915aa6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,7 +26,7 @@ before_install: # Install packages install: - conda install --yes python=$TRAVIS_PYTHON_VERSION numpy scipy matplotlib=1.5.0 nose flake8 - - conda install --yes -c https://conda.anaconda.org/bioconda pysam pyBigWig py2bit + - conda install --yes -c https://conda.anaconda.org/bioconda pysam==0.10.0 pyBigWig py2bit - python setup.py install # command to run tests From 1ee86f4cecdb1aae6132cc40e4278de885993d86 Mon Sep 17 00:00:00 2001 From: dpryan79 Date: Thu, 13 Apr 2017 15:43:14 +0200 Subject: [PATCH 05/40] Allow image size changes in plotPCA and plotCorrelation (heatmap only). --- deeptools/correlation.py | 8 ++++---- deeptools/plotCorrelation.py | 14 +++++++++++++- deeptools/plotPCA.py | 14 +++++++++++++- galaxy/wrapper/deepTools_macros.xml | 15 +++++++++++++++ galaxy/wrapper/plotCorrelation.xml | 3 +++ galaxy/wrapper/plotPCA.xml | 11 +++++++++++ 6 files changed, 59 insertions(+), 6 deletions(-) diff --git a/deeptools/correlation.py b/deeptools/correlation.py index b087ea40bd..f95783cb43 100644 --- a/deeptools/correlation.py +++ b/deeptools/correlation.py @@ -227,7 +227,7 @@ def compute_correlation(self): def plot_correlation(self, plot_fiilename, plot_title='', vmax=None, vmin=None, colormap='jet', image_format=None, - plot_numbers=False): + plot_numbers=False, plotWidth=11, plotHeight=9.5): """ plots a correlation using a symmetric heatmap """ @@ -248,7 +248,7 @@ def plot_correlation(self, plot_fiilename, plot_title='', vmax=None, vmin = 0 if corr_matrix .min() >= 0 else -1 # Compute and plot dendrogram. - fig = plt.figure(figsize=(11, 9.5)) + fig = plt.figure(figsize=(plotWidth, plotHeight)) plt.suptitle(plot_title) axdendro = fig.add_axes([0.02, 0.12, 0.1, 0.66]) @@ -431,12 +431,12 @@ def plot_scatter(self, plot_fiilename, plot_title='', image_format=None, log1p=F plt.savefig(plot_fiilename, format=image_format) plt.close() - def plot_pca(self, plot_filename, plot_title='', image_format=None, log1p=False): + def plot_pca(self, plot_filename, plot_title='', image_format=None, log1p=False, plotWidth=5, plotHeight=10): """ Plot the PCA of a matrix """ - fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(5, 10)) + fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(plotWidth, plotHeight)) # PCA if self.rowCenter: _ = self.matrix.mean(axis=1) diff --git a/deeptools/plotCorrelation.py b/deeptools/plotCorrelation.py index 5400197135..c9e04790f0 100644 --- a/deeptools/plotCorrelation.py +++ b/deeptools/plotCorrelation.py @@ -134,6 +134,16 @@ def heatmap_options(): parser = argparse.ArgumentParser(add_help=False) heatmap = parser.add_argument_group('Heatmap options') + heatmap.add_argument('--plotHeight', + help='Plot height in cm.', + type=float, + default=9.5) + + heatmap.add_argument('--plotWidth', + help='Plot width in cm. The minimum value is 1 cm.', + type=float, + default=11) + heatmap.add_argument('--zMin', '-min', default=None, help='Minimum value for the heatmap intensities. ' @@ -206,7 +216,9 @@ def main(args=None): colormap=args.colorMap, plot_title=args.plotTitle, image_format=args.plotFileFormat, - plot_numbers=args.plotNumbers) + plot_numbers=args.plotNumbers, + plotWidth=args.plotWidth, + plotHeight=args.plotHeight) if args.outFileCorMatrix: corr.save_corr_matrix(args.outFileCorMatrix) diff --git a/deeptools/plotPCA.py b/deeptools/plotPCA.py index fcb6a9b0b9..0036e03146 100644 --- a/deeptools/plotPCA.py +++ b/deeptools/plotPCA.py @@ -74,6 +74,16 @@ def plotCorrelationArgs(): 'eps, pdf and svg.', choices=['png', 'pdf', 'svg', 'eps']) + optional.add_argument('--plotHeight', + help='Plot height in cm.', + type=float, + default=10) + + optional.add_argument('--plotWidth', + help='Plot width in cm. The minimum value is 1 cm.', + type=float, + default=5) + optional.add_argument('--outFileNameData', help='File name to save the data ' 'underlying data for the average profile, e.g., ' @@ -105,7 +115,9 @@ def main(args=None): corr.plot_pca(args.plotFile.name, plot_title=args.plotTitle, - image_format=args.plotFileFormat) + image_format=args.plotFileFormat, + plotWidth=args.plotWidth, + plotHeight=args.plotHeight) if args.outFileNameData is not None: import matplotlib diff --git a/galaxy/wrapper/deepTools_macros.xml b/galaxy/wrapper/deepTools_macros.xml index bff287ffb2..d8f40f479b 100644 --- a/galaxy/wrapper/deepTools_macros.xml +++ b/galaxy/wrapper/deepTools_macros.xml @@ -25,6 +25,21 @@ + + --plotWidth '$advancedOp.plotWidth' + --plotHeight '$advancedOp.plotHeight' + + + + + + + + #if $advancedOpt.doExtendCustom.doExtend == 'custom': --extendReads $advancedOpt.doExtendCustom.extendReadsValue diff --git a/galaxy/wrapper/plotCorrelation.xml b/galaxy/wrapper/plotCorrelation.xml index 9448b941f9..bf4f90c29c 100644 --- a/galaxy/wrapper/plotCorrelation.xml +++ b/galaxy/wrapper/plotCorrelation.xml @@ -14,6 +14,8 @@ --whatToPlot "$plotting_type.whatToPlot" #if str($plotting_type.whatToPlot) == 'heatmap': @HEATMAP_OPTIONS@ + --plotWidth $plotting_type.plotWidth + --plotHeight $plotting_type.plotHeight #else: --plotTitle '$plotting_type.plotTitle' #end if @@ -37,6 +39,7 @@ + diff --git a/galaxy/wrapper/plotPCA.xml b/galaxy/wrapper/plotPCA.xml index b885faf563..8d5e1bf6d2 100644 --- a/galaxy/wrapper/plotPCA.xml +++ b/galaxy/wrapper/plotPCA.xml @@ -12,6 +12,7 @@ --plotTitle "$plotTitle" --plotFile "$outFileName" --plotFileFormat "$outFileFormat" + @PLOTWIDTHHEIGHT@ $rowCenter #if $outFileNameData --outFileNameData "$output_outFileNameData" @@ -24,6 +25,16 @@ + + + + + + + + + + From 7bd7ba08f8dad49a07a753aa7591e9e44b45cc29 Mon Sep 17 00:00:00 2001 From: dpryan79 Date: Thu, 13 Apr 2017 16:10:54 +0200 Subject: [PATCH 06/40] Forgot to close a tag in the wrapper --- galaxy/wrapper/plotCorrelation.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy/wrapper/plotCorrelation.xml b/galaxy/wrapper/plotCorrelation.xml index bf4f90c29c..a28c400d5c 100644 --- a/galaxy/wrapper/plotCorrelation.xml +++ b/galaxy/wrapper/plotCorrelation.xml @@ -39,7 +39,7 @@ - From 10a0c8d9a8e71f3f5b073029a222ebb9f0ae4649 Mon Sep 17 00:00:00 2001 From: Devon Ryan Date: Sun, 23 Apr 2017 22:28:00 +0200 Subject: [PATCH 07/40] Fix #516 --- deeptools/heatmapper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deeptools/heatmapper.py b/deeptools/heatmapper.py index 45a2a80bc3..5d1d4cd4e5 100644 --- a/deeptools/heatmapper.py +++ b/deeptools/heatmapper.py @@ -555,9 +555,9 @@ def compute_sub_matrix_worker(self, chrom, start, end, score_file_list, paramete if not parameters['missing data as zero']: coverage[:] = np.nan - if parameters['min threshold'] and coverage.min() <= parameters['min threshold']: + if parameters['min threshold'] is not None and coverage.min() <= parameters['min threshold']: continue - if parameters['max threshold'] and coverage.max() >= parameters['max threshold']: + if parameters['max threshold'] is not None and coverage.max() >= parameters['max threshold']: continue if parameters['scale'] != 1: coverage = parameters['scale'] * coverage From c025b2b086004e7bf9c224cc88b27b709d5e378e Mon Sep 17 00:00:00 2001 From: Devon Ryan Date: Sun, 23 Apr 2017 22:39:38 +0200 Subject: [PATCH 08/40] Implement #508 --- deeptools/plotCoverage.py | 17 +++++++++++++---- galaxy/wrapper/deepTools_macros.xml | 15 +++++++++++++++ galaxy/wrapper/plotCoverage.xml | 2 ++ 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/deeptools/plotCoverage.py b/deeptools/plotCoverage.py index ecad83c15e..ba8bcea278 100644 --- a/deeptools/plotCoverage.py +++ b/deeptools/plotCoverage.py @@ -110,6 +110,16 @@ def required_args(): help='Save raw counts (coverages) to file.', metavar='FILE') + optional.add_argument('--plotHeight', + help='Plot height in cm.', + type=float, + default=5.) + + optional.add_argument('--plotWidth', + help='Plot width in cm. The minimum value is 1 cm.', + type=float, + default=15) + optional.add_argument('--plotFileFormat', metavar='FILETYPE', help='Image format type. If given, this option ' @@ -165,7 +175,7 @@ def main(args=None): if args.skipZeros: num_reads_per_bin = countR.remove_row_of_zeros(num_reads_per_bin) - fig, axs = plt.subplots(1, 2, figsize=(15, 5)) + fig, axs = plt.subplots(1, 2, figsize=(args.plotWidth, args.plotHeight)) plt.suptitle(args.plotTitle) # plot up to two std from mean num_reads_per_bin = num_reads_per_bin.astype(int) @@ -212,9 +222,8 @@ def main(args=None): sample_max[idx], )) - # The 'good' x-axis is computed for each sample. The lower value is favored in which - # distributions with a wider x-range can better be seen. - y_max = min(y_max) + # Don't clip plots + y_max = max(y_max) axs[0].set_ylim(0, min(1, y_max + (y_max * 0.10))) axs[0].set_xlim(0, x_max) axs[0].set_xlabel('coverage (#reads per bp)') diff --git a/galaxy/wrapper/deepTools_macros.xml b/galaxy/wrapper/deepTools_macros.xml index bff287ffb2..d8f40f479b 100644 --- a/galaxy/wrapper/deepTools_macros.xml +++ b/galaxy/wrapper/deepTools_macros.xml @@ -25,6 +25,21 @@ + + --plotWidth '$advancedOp.plotWidth' + --plotHeight '$advancedOp.plotHeight' + + + + + + + + #if $advancedOpt.doExtendCustom.doExtend == 'custom': --extendReads $advancedOpt.doExtendCustom.extendReadsValue diff --git a/galaxy/wrapper/plotCoverage.xml b/galaxy/wrapper/plotCoverage.xml index 6cc46b6327..ef17add13f 100644 --- a/galaxy/wrapper/plotCoverage.xml +++ b/galaxy/wrapper/plotCoverage.xml @@ -38,6 +38,7 @@ --plotTitle '$advancedOpt.plotTitle' #end if @ADVANCED_OPTS_READ_PROCESSING@ + @PLOTWIDTHHEIGHT@ @blacklist@ #end if @@ -57,6 +58,7 @@ + From c208db052c2470efabf64d36ec96db114ce46015 Mon Sep 17 00:00:00 2001 From: drakeeee Date: Mon, 24 Apr 2017 00:57:22 -0700 Subject: [PATCH 09/40] fixed debug global bug and shutil bedgraph join bug in correctGCBias (#511) * fixed debug global bug and shutil bedgraph join * PEP8 --- deeptools/correctGCBias.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/deeptools/correctGCBias.py b/deeptools/correctGCBias.py index db19e0e1c4..332b8d623d 100644 --- a/deeptools/correctGCBias.py +++ b/deeptools/correctGCBias.py @@ -237,12 +237,16 @@ def writeCorrected_worker(chrNameBam, chrNameBit, start, end, step): cvg_corr[vectorStart:vectorEnd] += float(1) / R_gc[gc] i += 1 - if debug: - endTime = time.time() - print("{}, processing {} ({:.1f} per sec) ") - "reads @ {}:{}-{}".format(multiprocessing.current_process().name, - i, i / (endTime - startTime), - chrNameBit, start, end) + + try: + if debug: + endTime = time.time() + print("{}, processing {} ({:.1f} per sec) ") + "reads @ {}:{}-{}".format(multiprocessing.current_process().name, + i, i / (endTime - startTime), + chrNameBit, start, end) + except NameError: + pass if i == 0: return None @@ -661,7 +665,7 @@ def main(args=None): res = list(map(writeCorrected_wrapper, mp_args)) # concatenate intermediary bedgraph files - _temp_bg_file = open(_temp_bg_file_name, 'w') + _temp_bg_file = open(_temp_bg_file_name, 'wb') for tempFileName in res: if tempFileName: # concatenate all intermediate tempfiles into one From 70e6dd93af93f0f856f12d5a6c20e24e38a6fffb Mon Sep 17 00:00:00 2001 From: dpryan79 Date: Mon, 24 Apr 2017 10:01:40 +0200 Subject: [PATCH 10/40] Update CHANGES.txt --- CHANGES.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.txt b/CHANGES.txt index f26b968d67..135527ec1f 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,6 +1,7 @@ 2.5.1 * Added universal new line support to deeptoolsintervals (issue #506). + * Fixed a few issues with correctGCBias under python 3.5 (thanks to @drakeeee) 2.5.0 From f1cdcd5decc3729ecb3dfc37870584de5b89fa9c Mon Sep 17 00:00:00 2001 From: dpryan79 Date: Tue, 25 Apr 2017 09:37:08 +0200 Subject: [PATCH 11/40] Unpeg pysam and update a computeMatrix error message. --- .travis.yml | 2 +- deeptools/heatmapper.py | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index d086915aa6..277d192e4c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,7 +26,7 @@ before_install: # Install packages install: - conda install --yes python=$TRAVIS_PYTHON_VERSION numpy scipy matplotlib=1.5.0 nose flake8 - - conda install --yes -c https://conda.anaconda.org/bioconda pysam==0.10.0 pyBigWig py2bit + - conda install --yes -c https://conda.anaconda.org/bioconda pysam pyBigWig py2bit - python setup.py install # command to run tests diff --git a/deeptools/heatmapper.py b/deeptools/heatmapper.py index 5d1d4cd4e5..2249cb9c30 100644 --- a/deeptools/heatmapper.py +++ b/deeptools/heatmapper.py @@ -271,9 +271,7 @@ def computeMatrix(self, score_file_list, regions_file, parameters, blackListFile "matrix length does not match regions length" if len(regions) == 0: - sys.stderr.write( - "\nERROR: BED file does not contain any valid regions. " - "Please check\n") + sys.stderr.write("\nERROR: Either the BED file does not contain any valid regions or there are none remaining after filtering.\n") exit(1) if regions_no_score == len(regions): exit("\nERROR: None of the BED regions could be found in the bigWig" From 7504f1b4caac6c5d87aeb3cf4b62689efbe654bb Mon Sep 17 00:00:00 2001 From: dpryan79 Date: Tue, 25 Apr 2017 10:09:55 +0200 Subject: [PATCH 12/40] Mention fix516 in the change log --- CHANGES.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.txt b/CHANGES.txt index 135527ec1f..92b87fa5ed 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -2,6 +2,7 @@ * Added universal new line support to deeptoolsintervals (issue #506). * Fixed a few issues with correctGCBias under python 3.5 (thanks to @drakeeee) + * Setting `--minThreshold 0.0` or `--maxThreshold 0.0` now works properly. Previously, setting either of these to 0 was ignored. (issue #516) 2.5.0 From 408a8aaa62a622520fd7a28db11a16401140a589 Mon Sep 17 00:00:00 2001 From: dpryan79 Date: Tue, 25 Apr 2017 11:03:06 +0200 Subject: [PATCH 13/40] Get #507 working in galaxy --- galaxy/wrapper/deepTools_macros.xml | 10 +++++----- galaxy/wrapper/plotCorrelation.xml | 2 +- galaxy/wrapper/plotPCA.xml | 6 ++++-- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/galaxy/wrapper/deepTools_macros.xml b/galaxy/wrapper/deepTools_macros.xml index d8f40f479b..3b8ae31fdf 100644 --- a/galaxy/wrapper/deepTools_macros.xml +++ b/galaxy/wrapper/deepTools_macros.xml @@ -26,16 +26,16 @@ - --plotWidth '$advancedOp.plotWidth' - --plotHeight '$advancedOp.plotHeight' + --plotWidth '$advancedOpt.plotWidth' + --plotHeight '$advancedOpt.plotHeight' - - + - diff --git a/galaxy/wrapper/plotCorrelation.xml b/galaxy/wrapper/plotCorrelation.xml index a28c400d5c..f331a270a1 100644 --- a/galaxy/wrapper/plotCorrelation.xml +++ b/galaxy/wrapper/plotCorrelation.xml @@ -39,7 +39,7 @@ - + diff --git a/galaxy/wrapper/plotPCA.xml b/galaxy/wrapper/plotPCA.xml index 8d5e1bf6d2..83fb58bb77 100644 --- a/galaxy/wrapper/plotPCA.xml +++ b/galaxy/wrapper/plotPCA.xml @@ -12,7 +12,9 @@ --plotTitle "$plotTitle" --plotFile "$outFileName" --plotFileFormat "$outFileFormat" - @PLOTWIDTHHEIGHT@ + #if str($advancedOpt.showAdvancedOpt) == 'yes': + @PLOTWIDTHHEIGHT@ + #end if $rowCenter #if $outFileNameData --outFileNameData "$output_outFileNameData" @@ -32,7 +34,7 @@ - + From e90788f8c2bc48486b520959ea8ab927491c2761 Mon Sep 17 00:00:00 2001 From: dpryan79 Date: Tue, 25 Apr 2017 11:30:39 +0200 Subject: [PATCH 14/40] Update change log --- CHANGES.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.txt b/CHANGES.txt index 92b87fa5ed..57bfbe542b 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -3,6 +3,7 @@ * Added universal new line support to deeptoolsintervals (issue #506). * Fixed a few issues with correctGCBias under python 3.5 (thanks to @drakeeee) * Setting `--minThreshold 0.0` or `--maxThreshold 0.0` now works properly. Previously, setting either of these to 0 was ignored. (issue #516) + * You can now specify the plot width and height in `plotPCA` and `plotCorrelation` (heatmap only) with the `--plotWidth` and `--plotHeight` parameters. (issue #507) 2.5.0 From 7c959b6507c86963867e2d84ca2156b7f448b2fd Mon Sep 17 00:00:00 2001 From: dpryan79 Date: Tue, 25 Apr 2017 13:28:16 +0200 Subject: [PATCH 15/40] Update change log and manually modify the freaking font sizes in an SVG file --- CHANGES.txt | 1 + .../plotCoverage_default.svg | 118 +++++++++--------- 2 files changed, 60 insertions(+), 59 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 57bfbe542b..da08a0eec6 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -4,6 +4,7 @@ * Fixed a few issues with correctGCBias under python 3.5 (thanks to @drakeeee) * Setting `--minThreshold 0.0` or `--maxThreshold 0.0` now works properly. Previously, setting either of these to 0 was ignored. (issue #516) * You can now specify the plot width and height in `plotPCA` and `plotCorrelation` (heatmap only) with the `--plotWidth` and `--plotHeight` parameters. (issue #507) + * plotCoverage no longer clips the top off of plots. Further, you can now set the plot width and height with `--plotWidth` and `--plotHeight`. (issue #508) 2.5.0 diff --git a/deeptools/test/test_plotCoverage/plotCoverage_default.svg b/deeptools/test/test_plotCoverage/plotCoverage_default.svg index 2eab67c42d..1f2a6ff865 100644 --- a/deeptools/test/test_plotCoverage/plotCoverage_default.svg +++ b/deeptools/test/test_plotCoverage/plotCoverage_default.svg @@ -129,7 +129,7 @@ L 0 4 - 0 + 0 @@ -144,7 +144,7 @@ L 0 4 - 5 + 5 @@ -159,7 +159,7 @@ L 0 4 - 10 + 10 @@ -174,7 +174,7 @@ L 0 4 - 15 + 15 @@ -189,11 +189,11 @@ L 0 4 - 20 + 20 - coverage (#reads per bp) + coverage (#reads per bp) @@ -219,7 +219,7 @@ L -4 0 - 0.00 + 0.00 @@ -234,7 +234,7 @@ L -4 0 - 0.01 + 0.01 @@ -249,7 +249,7 @@ L -4 0 - 0.02 + 0.02 @@ -264,7 +264,7 @@ L -4 0 - 0.03 + 0.03 @@ -279,7 +279,7 @@ L -4 0 - 0.04 + 0.04 @@ -294,7 +294,7 @@ L -4 0 - 0.05 + 0.05 @@ -309,7 +309,7 @@ L -4 0 - 0.06 + 0.06 @@ -324,7 +324,7 @@ L -4 0 - 0.07 + 0.07 @@ -339,44 +339,44 @@ L -4 0 - 0.08 + 0.08 - fraction of bases sampled + fraction of bases sampled - - - test1.bam, mean=4.9 + test1.bam, mean=4.9 - - test2.bam, mean=6.6 + test2.bam, mean=6.6 @@ -480,7 +480,7 @@ L 591.545455 36 - 0 + 0 @@ -495,7 +495,7 @@ L 591.545455 36 - 5 + 5 @@ -510,7 +510,7 @@ L 591.545455 36 - 10 + 10 @@ -525,7 +525,7 @@ L 591.545455 36 - 15 + 15 @@ -540,11 +540,11 @@ L 591.545455 36 - 20 + 20 - coverage (#reads per bp) + coverage (#reads per bp) @@ -560,7 +560,7 @@ L 591.545455 36 - 0.0 + 0.0 @@ -575,7 +575,7 @@ L 591.545455 36 - 0.2 + 0.2 @@ -590,7 +590,7 @@ L 591.545455 36 - 0.4 + 0.4 @@ -605,7 +605,7 @@ L 591.545455 36 - 0.6 + 0.6 @@ -620,7 +620,7 @@ L 591.545455 36 - 0.8 + 0.8 @@ -635,44 +635,44 @@ L 591.545455 36 - 1.0 + 1.0 - fraction of bases sampled >= coverage + fraction of bases sampled >= coverage - - - test1.bam + test1.bam - - test2.bam + test2.bam From c94ae33a71643b69443829bd98cf700fdb4353e2 Mon Sep 17 00:00:00 2001 From: dpryan79 Date: Tue, 25 Apr 2017 13:34:08 +0200 Subject: [PATCH 16/40] Missed a couple spaces --- deeptools/test/test_plotCoverage/plotCoverage_default.svg | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deeptools/test/test_plotCoverage/plotCoverage_default.svg b/deeptools/test/test_plotCoverage/plotCoverage_default.svg index 1f2a6ff865..962a4538a2 100644 --- a/deeptools/test/test_plotCoverage/plotCoverage_default.svg +++ b/deeptools/test/test_plotCoverage/plotCoverage_default.svg @@ -348,7 +348,7 @@ L -4 0 - - From 3fec94a920a4c17dd4587baee97aac30bdfea5ef Mon Sep 17 00:00:00 2001 From: dpryan79 Date: Wed, 26 Apr 2017 15:24:26 +0200 Subject: [PATCH 17/40] Fix #520, remove the RNA class from bamCoverage and just use the Offset class --- deeptools/bamCoverage.py | 79 ++-------------------------------------- 1 file changed, 4 insertions(+), 75 deletions(-) diff --git a/deeptools/bamCoverage.py b/deeptools/bamCoverage.py index e60c94da45..d2c9b41f41 100644 --- a/deeptools/bamCoverage.py +++ b/deeptools/bamCoverage.py @@ -148,6 +148,10 @@ def main(args=None): func_args = {'scaleFactor': scale_factor} + # This fixes issue #520, where --extendReads wasn't honored if --filterRNAstrand was used + if args.filterRNAstrand and not args.Offset: + args.Offset = [1, -1] + if args.MNase: # check that library is paired end # using getFragmentAndReadSize @@ -210,26 +214,6 @@ def main(args=None): verbose=args.verbose) wr.filter_strand = args.filterRNAstrand wr.Offset = args.Offset - - elif args.filterRNAstrand: - wr = filterRnaStrand([args.bam], - binLength=args.binSize, - stepSize=args.binSize, - region=args.region, - numberOfProcessors=args.numberOfProcessors, - extendReads=args.extendReads, - minMappingQuality=args.minMappingQuality, - ignoreDuplicates=args.ignoreDuplicates, - center_read=args.centerReads, - zerosToNans=args.skipNonCoveredRegions, - samFlag_include=args.samFlagInclude, - samFlag_exclude=args.samFlagExclude, - minFragmentLength=args.minFragmentLength, - maxFragmentLength=args.maxFragmentLength, - verbose=args.verbose, - ) - - wr.filter_strand = args.filterRNAstrand else: wr = writeBedGraph.WriteBedGraph([args.bam], binLength=args.binSize, @@ -399,58 +383,3 @@ def get_fragment_from_read(self, read): fragment_end = fragment_start + 3 return [(fragment_start, fragment_end)] - - -class filterRnaStrand(writeBedGraph.WriteBedGraph): - """ - Class to redefine the get_fragment_from_read for the --filterRNAstrand case - - Only reads either forward or reverse are kept as follows: - - For paired-end - -------------- - reads forward: - - 1. alignments of the second in pair (128) if they map to the forward strand (~16) - 2. alignments of the first in pair (64) if they map to the reverse strand (~32) - - 1. include 128, exclude 16 - or - 2. include 64 exclude 32 - - reads reverse: - 1. alignments of the second in pair (128) if it maps to the reverse strand (16) 128 & 16 = 144 - 2. alignments of the first in pair (64) if their mates map to the reverse strand (32) 64 & 32 = 96 - - 1. include 144 - or - 2. include 96 - - For single-end - -------------- - forward: include 16 (map forward strand) - reverse: exclude 16 - - """ - - def get_fragment_from_read(self, read): - """ - Gets only reads for the given strand - """ - fragment_start = fragment_end = None - - # only paired forward reads are considered - if read.is_paired: - if self.filter_strand == 'forward': - if (read.flag & 128 == 128 and read.flag & 16 == 0) or (read.flag & 64 == 64 and read.flag & 32 == 0): - return read.get_blocks() - else: - if read.flag & 144 == 144 or read.flag & 96 == 96: - return read.get_blocks() - else: - if self.filter_strand == 'forward' and read.flag & 16 == 16: - return read.get_blocks() - elif self.filter_strand == 'reverse' and read.flag & 16 == 0: - return read.get_blocks() - - return [(fragment_start, fragment_end)] From 57aa831d6a7d2ea12c54792e1b94f27a73ff508c Mon Sep 17 00:00:00 2001 From: dpryan79 Date: Wed, 26 Apr 2017 15:52:09 +0200 Subject: [PATCH 18/40] Update change log --- CHANGES.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.txt b/CHANGES.txt index da08a0eec6..bab3a57a78 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -5,6 +5,7 @@ * Setting `--minThreshold 0.0` or `--maxThreshold 0.0` now works properly. Previously, setting either of these to 0 was ignored. (issue #516) * You can now specify the plot width and height in `plotPCA` and `plotCorrelation` (heatmap only) with the `--plotWidth` and `--plotHeight` parameters. (issue #507) * plotCoverage no longer clips the top off of plots. Further, you can now set the plot width and height with `--plotWidth` and `--plotHeight`. (issue #508) + * In bamCoverage, specifying `--filterRNAstrand` no longer results in `--extendReads` being ignored. (issue #520) 2.5.0 From 6d941fb9dae92858ef084a0ffe46d63e95e1c530 Mon Sep 17 00:00:00 2001 From: dpryan79 Date: Wed, 26 Apr 2017 16:07:47 +0200 Subject: [PATCH 19/40] Make the plot part of plotFingerprint optional. Implements #519 --- deeptools/plotFingerprint.py | 46 ++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/deeptools/plotFingerprint.py b/deeptools/plotFingerprint.py index 138ddae696..5b52cc7aa9 100644 --- a/deeptools/plotFingerprint.py +++ b/deeptools/plotFingerprint.py @@ -151,8 +151,7 @@ def get_output_args(): 'format. The available options are typically: "png", ' '"eps", "pdf" and "svg", e.g. : fingerprint.png.', metavar='', - type=argparse.FileType('w'), - required=True) + type=argparse.FileType('w')) group.add_argument('--outRawCounts', help='Output file name to save the read counts per bin.', @@ -349,6 +348,10 @@ def getExpected(mu): def main(args=None): args = process_args(args) + if not args.plotFile and not args.outRawCounts and not args.outQualityMetrics: + sys.stderr.write("\nAt least one of --plotFile, --outRawCounts or --outQualityMetrics is required.\n") + sys.exit(1) + cr = sumR.SumCoveragePerBin( args.bamfiles, args.binSize, @@ -382,25 +385,26 @@ def main(args=None): total = len(num_reads_per_bin[:, 0]) x = np.arange(total).astype('float') / total # normalize from 0 to 1 - i = 0 - # matplotlib won't iterate through line styles by itself - pyplot_line_styles = sum([7 * ["-"], 7 * ["--"], 7 * ["-."], 7 * [":"], 7 * ["."]], []) - for i, reads in enumerate(num_reads_per_bin.T): - count = np.cumsum(np.sort(reads)) - count = count / count[-1] # to normalize y from 0 to 1 - j = i % 35 - plt.plot(x, count, label=args.labels[i], linestyle=pyplot_line_styles[j]) - plt.xlabel('rank') - plt.ylabel('fraction w.r.t. bin with highest coverage') - plt.legend(loc='upper left') - plt.suptitle(args.plotTitle) - # set the plotFileFormat explicitly to None to trigger the - # format from the file-extension - if not args.plotFileFormat: - args.plotFileFormat = None - - plt.savefig(args.plotFile.name, bbox_inches=0, format=args.plotFileFormat) - plt.close() + if args.plotFile: + i = 0 + # matplotlib won't iterate through line styles by itself + pyplot_line_styles = sum([7 * ["-"], 7 * ["--"], 7 * ["-."], 7 * [":"], 7 * ["."]], []) + for i, reads in enumerate(num_reads_per_bin.T): + count = np.cumsum(np.sort(reads)) + count = count / count[-1] # to normalize y from 0 to 1 + j = i % 35 + plt.plot(x, count, label=args.labels[i], linestyle=pyplot_line_styles[j]) + plt.xlabel('rank') + plt.ylabel('fraction w.r.t. bin with highest coverage') + plt.legend(loc='upper left') + plt.suptitle(args.plotTitle) + # set the plotFileFormat explicitly to None to trigger the + # format from the file-extension + if not args.plotFileFormat: + args.plotFileFormat = None + + plt.savefig(args.plotFile.name, bbox_inches=0, format=args.plotFileFormat) + plt.close() if args.outRawCounts: args.outRawCounts.write("'" + "'\t'".join(args.labels) + "'\n") From a778fc0801b63d449c308dd826463cfdf99c92ce Mon Sep 17 00:00:00 2001 From: dpryan79 Date: Wed, 26 Apr 2017 16:35:22 +0200 Subject: [PATCH 20/40] Update change log --- CHANGES.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.txt b/CHANGES.txt index bab3a57a78..0d45a9b615 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -6,6 +6,7 @@ * You can now specify the plot width and height in `plotPCA` and `plotCorrelation` (heatmap only) with the `--plotWidth` and `--plotHeight` parameters. (issue #507) * plotCoverage no longer clips the top off of plots. Further, you can now set the plot width and height with `--plotWidth` and `--plotHeight`. (issue #508) * In bamCoverage, specifying `--filterRNAstrand` no longer results in `--extendReads` being ignored. (issue #520) + * `plotFingerprint` no longers requires producing a plot, which is useful if you only need QC metrics and are using a LOT of samples (such that matplotlib would crash anyway). (issue #519) 2.5.0 From 4677a6a6df0c6c009480beaaf95b55aa57f61313 Mon Sep 17 00:00:00 2001 From: dpryan79 Date: Wed, 26 Apr 2017 16:35:49 +0200 Subject: [PATCH 21/40] Update change log --- CHANGES.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index 0d45a9b615..f5cfce32c9 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -6,7 +6,7 @@ * You can now specify the plot width and height in `plotPCA` and `plotCorrelation` (heatmap only) with the `--plotWidth` and `--plotHeight` parameters. (issue #507) * plotCoverage no longer clips the top off of plots. Further, you can now set the plot width and height with `--plotWidth` and `--plotHeight`. (issue #508) * In bamCoverage, specifying `--filterRNAstrand` no longer results in `--extendReads` being ignored. (issue #520) - * `plotFingerprint` no longers requires producing a plot, which is useful if you only need QC metrics and are using a LOT of samples (such that matplotlib would crash anyway). (issue #519) + * `plotFingerprint` no longers requires producing a plot, which is useful if you only need QC metrics and are using a LOT of samples (such that matplotlib would crash anyway). This hasn't been implemented in Galaxy, but can if people would like it. (issue #519) 2.5.0 From ce9f01950370cc35ea6448d9d7ee10992066fc09 Mon Sep 17 00:00:00 2001 From: Devon Ryan Date: Wed, 26 Apr 2017 22:27:50 +0200 Subject: [PATCH 22/40] Fix #522, allow bamPEFragmentSize --distanceBetweenBins 0 --- deeptools/getFragmentAndReadSize.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/deeptools/getFragmentAndReadSize.py b/deeptools/getFragmentAndReadSize.py index 4583cfe952..53c10cd83d 100644 --- a/deeptools/getFragmentAndReadSize.py +++ b/deeptools/getFragmentAndReadSize.py @@ -84,6 +84,19 @@ def get_read_and_fragment_length(bamFile, return_lengths=False, blackListFileNam distanceBetweenBins *= 2 fl = [] + + # Fix issue #522, allow distanceBetweenBins == 0 + if distanceBetweenBins == 0: + imap_res = mapReduce.mapReduce((bam_handle.filename, distanceBetweenBins), + getFragmentLength_wrapper, + chrom_sizes, + genomeChunkLength=binSize, + blackListFileName=blackListFileName, + numberOfProcessors=numberOfProcessors, + verbose=verbose) + fl = np.concatenate(imap_res) + + # Try to ensure we have at least 1000 regions from which to compute statistics, halving the intra-bin distance as needed while len(fl) < 1000 and distanceBetweenBins > 1: distanceBetweenBins /= 2 stepsize = binSize + distanceBetweenBins From 2588bbe7a2f4b21e12b8e46d44a09149dda38c65 Mon Sep 17 00:00:00 2001 From: Devon Ryan Date: Wed, 26 Apr 2017 23:32:10 +0200 Subject: [PATCH 23/40] Fix the sample labeling stuff in bamPEFragmentSize --- deeptools/bamPEFragmentSize.py | 12 ++++++++---- .../wrapper/test-data/bamPEFragmentSize_result1.txt | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/deeptools/bamPEFragmentSize.py b/deeptools/bamPEFragmentSize.py index 975172fd03..7a855e24f6 100644 --- a/deeptools/bamPEFragmentSize.py +++ b/deeptools/bamPEFragmentSize.py @@ -83,14 +83,18 @@ def parse_arguments(): return parser -def getFragSize(bam, args): +def getFragSize(bam, args, idx): fragment_len_dict, read_len_dict = get_read_and_fragment_length(bam, return_lengths=True, blackListFileName=args.blackListFileName, numberOfProcessors=args.numberOfProcessors, verbose=args.verbose, binSize=args.binSize, distanceBetweenBins=args.distanceBetweenBins) - print("\n\nBAM file : {}".format(bam)) + if args.samplesLabel and idx < len(args.samplesLabel): + print("\n\nSample label: {}".format(args.samplesLabel[idx])) + else: + print("\n\nBAM file : {}".format(bam)) + if fragment_len_dict: if fragment_len_dict['mean'] == 0: print("No pairs were found. Is the data from a paired-end sequencing experiment?") @@ -125,8 +129,8 @@ def main(args=None): args = parse_arguments().parse_args(args) fraglengths = {} - for bam in args.bamfiles: - fraglengths[bam] = getFragSize(bam, args) + for idx, bam in enumerate(args.bamfiles): + fraglengths[bam] = getFragSize(bam, args, idx) if args.histogram: import matplotlib diff --git a/galaxy/wrapper/test-data/bamPEFragmentSize_result1.txt b/galaxy/wrapper/test-data/bamPEFragmentSize_result1.txt index 7934a68776..d02bb237da 100644 --- a/galaxy/wrapper/test-data/bamPEFragmentSize_result1.txt +++ b/galaxy/wrapper/test-data/bamPEFragmentSize_result1.txt @@ -1,6 +1,6 @@ -BAM file : 0.bam +Sample label: bowtie2 test1.bam Sample size: 3 Fragment lengths: From 7115d59ec1ba72e72cd937c72f823a855db5e667 Mon Sep 17 00:00:00 2001 From: dpryan79 Date: Fri, 28 Apr 2017 11:16:52 +0200 Subject: [PATCH 24/40] Start implementing #523. I'm not yet happy with how this is done in Galaxy, where I think it should default to the history item names. --- deeptools/computeMatrix.py | 9 +++++++++ deeptools/heatmapper.py | 5 ++++- galaxy/wrapper/computeMatrix.xml | 12 ++++++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/deeptools/computeMatrix.py b/deeptools/computeMatrix.py index a1e28bbc8f..ec31cd6920 100644 --- a/deeptools/computeMatrix.py +++ b/deeptools/computeMatrix.py @@ -316,6 +316,15 @@ def computeMatrixOptArgs(case=['scale-regions', 'reference-point'][0]): metavar="BED file", required=False) + optional.add_argument('--samplesLabel', + help='Labels for the samples. This will then be passed to plotHeatmap and plotProfile. The ' + 'default is to use the file name of the ' + 'sample. The sample labels should be separated ' + 'by spaces and quoted if a label itself' + 'contains a space E.g. --samplesLabel label-1 "label 2" ', + nargs='+') + + # in contrast to other tools, # computeMatrix by default outputs # messages and the --quiet flag supresses them diff --git a/deeptools/heatmapper.py b/deeptools/heatmapper.py index 2249cb9c30..4655d47831 100644 --- a/deeptools/heatmapper.py +++ b/deeptools/heatmapper.py @@ -296,7 +296,10 @@ def computeMatrix(self, score_file_list, regions_file, parameters, blackListFile numcols = matrix.shape[1] num_ind_cols = self.get_num_individual_matrix_cols() sample_boundaries = list(range(0, numcols + num_ind_cols, num_ind_cols)) - sample_labels = [splitext(basename(x))[0] for x in score_file_list] + if allArgs is not None and allArgs['samplesLabel'] is not None: + sample_labels = allArgs['samplesLabel'] + else: + sample_labels = [splitext(basename(x))[0] for x in score_file_list] # Determine the group boundaries group_boundaries = [] diff --git a/galaxy/wrapper/computeMatrix.xml b/galaxy/wrapper/computeMatrix.xml index 9b627d9050..1fc61d9d86 100644 --- a/galaxy/wrapper/computeMatrix.xml +++ b/galaxy/wrapper/computeMatrix.xml @@ -79,6 +79,9 @@ #if $advancedOpt.scale is not None and str($advancedOpt.scale) != '': --scale $advancedOpt.scale #end if + #if $advancedOpt.samplesLabel is not None and str($advancedOpt.samplesLabel) != '': + --samplesLabel $advancedOpt.samplesLabel + #end if @ADVANCED_OPTS_GTF@ @@ -214,6 +217,15 @@ bias the average values. (--maxThreshold)"/> + + + + + + From d6ccaf49043e4e6b1591accae4929b00716afb45 Mon Sep 17 00:00:00 2001 From: dpryan79 Date: Fri, 28 Apr 2017 11:31:22 +0200 Subject: [PATCH 25/40] Update the Galaxy wrapper. Since it already uses the element identifier we only need to override that upon request. --- galaxy/wrapper/computeMatrix.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/galaxy/wrapper/computeMatrix.xml b/galaxy/wrapper/computeMatrix.xml index 1fc61d9d86..6c908e97ab 100644 --- a/galaxy/wrapper/computeMatrix.xml +++ b/galaxy/wrapper/computeMatrix.xml @@ -218,8 +218,8 @@ From 833222f250247e23c22dce6c8c470cf8c929e10c Mon Sep 17 00:00:00 2001 From: dpryan79 Date: Fri, 28 Apr 2017 13:08:40 +0200 Subject: [PATCH 26/40] Implement #514. This was incorrectly branched from implement523. --- galaxy/wrapper/bamPEFragmentSize.xml | 2 +- galaxy/wrapper/computeMatrix.xml | 22 ++++-------- galaxy/wrapper/deepTools_macros.xml | 48 ++++++++------------------- galaxy/wrapper/multiBamSummary.xml | 2 +- galaxy/wrapper/multiBigwigSummary.xml | 2 +- galaxy/wrapper/plotCoverage.xml | 2 +- galaxy/wrapper/plotEnrichment.xml | 2 +- galaxy/wrapper/plotFingerprint.xml | 2 +- 8 files changed, 26 insertions(+), 56 deletions(-) diff --git a/galaxy/wrapper/bamPEFragmentSize.xml b/galaxy/wrapper/bamPEFragmentSize.xml index 694d4838cd..fe745d60d2 100644 --- a/galaxy/wrapper/bamPEFragmentSize.xml +++ b/galaxy/wrapper/bamPEFragmentSize.xml @@ -31,7 +31,7 @@ ]]> - + diff --git a/galaxy/wrapper/computeMatrix.xml b/galaxy/wrapper/computeMatrix.xml index 6c908e97ab..45f7b24f8f 100644 --- a/galaxy/wrapper/computeMatrix.xml +++ b/galaxy/wrapper/computeMatrix.xml @@ -7,14 +7,7 @@ - + - + @@ -253,7 +243,7 @@ - + @@ -263,7 +253,7 @@ - + diff --git a/galaxy/wrapper/deepTools_macros.xml b/galaxy/wrapper/deepTools_macros.xml index 3b8ae31fdf..75800c9408 100644 --- a/galaxy/wrapper/deepTools_macros.xml +++ b/galaxy/wrapper/deepTools_macros.xml @@ -396,46 +396,26 @@ This tool is developed by the `Bioinformatics and Deep-Sequencing Unit`_ at the - + - - + - - - - - - - - - - - - - - - - - - + @@ -443,13 +423,13 @@ is vital to you, select Yes below."> - + - - + + @@ -494,16 +474,16 @@ is vital to you, select Yes below."> #import re #if $multibigwig_conditional.orderMatters == "No": #for $counter, $bigwig in enumerate($multibigwig_conditional.bigwigfiles): - #set identifier = re.sub('[^\.\s\w\-]', '_', str($bigwig.element_identifier)) - ln -s "${bigwig}" "${counter}.bw" && - #silent $files.append('%s.bw' % $counter) + #set identifier = re.sub('[^\.\s\w\-]', '_', str($bigwig.element_identifier)) + ln -f -s "${bigwig}" "${identifier}_${counter}.bw" && + #silent $files.append('%s_%s.bw' % ($identifier, $counter)) #silent $labels.append("'%s'" % $identifier) #end for #else: #for $counter, $f in enumerate($multibigwig_conditional.multibigwig_repeats): #set identifier = re.sub('[^\.\s\w\-]', '_', str($f.bigwigfiles.element_identifier)) - ln -s "${f.bigwigfiles}" "${counter}.bw" && - #silent $files.append('%s.bw' % $counter) + ln -f -s "${f.bigwigfiles}" "${identifier}_${counter}.bw" && + #silent $files.append('%s_%s.bw' % ($identifier, $counter)) #silent $labels.append("'%s'" % $identifier) #end for #end if diff --git a/galaxy/wrapper/multiBamSummary.xml b/galaxy/wrapper/multiBamSummary.xml index 57efc0c41b..ab47359295 100644 --- a/galaxy/wrapper/multiBamSummary.xml +++ b/galaxy/wrapper/multiBamSummary.xml @@ -43,7 +43,7 @@ - + - + - + diff --git a/galaxy/wrapper/plotEnrichment.xml b/galaxy/wrapper/plotEnrichment.xml index 0f63cf0465..91cc17fb27 100644 --- a/galaxy/wrapper/plotEnrichment.xml +++ b/galaxy/wrapper/plotEnrichment.xml @@ -76,7 +76,7 @@ - + diff --git a/galaxy/wrapper/plotFingerprint.xml b/galaxy/wrapper/plotFingerprint.xml index 72cfae4d54..93418cd353 100644 --- a/galaxy/wrapper/plotFingerprint.xml +++ b/galaxy/wrapper/plotFingerprint.xml @@ -53,7 +53,7 @@ - + From 6149ade4a63c001844b0916aa496b60b58f10bd3 Mon Sep 17 00:00:00 2001 From: dpryan79 Date: Fri, 28 Apr 2017 13:09:02 +0200 Subject: [PATCH 27/40] PEP8 --- deeptools/computeMatrix.py | 1 - 1 file changed, 1 deletion(-) diff --git a/deeptools/computeMatrix.py b/deeptools/computeMatrix.py index ec31cd6920..f3eff5113e 100644 --- a/deeptools/computeMatrix.py +++ b/deeptools/computeMatrix.py @@ -324,7 +324,6 @@ def computeMatrixOptArgs(case=['scale-regions', 'reference-point'][0]): 'contains a space E.g. --samplesLabel label-1 "label 2" ', nargs='+') - # in contrast to other tools, # computeMatrix by default outputs # messages and the --quiet flag supresses them From bca17927f261baec6d7ce32f27dcb9299150f83e Mon Sep 17 00:00:00 2001 From: dpryan79 Date: Fri, 28 Apr 2017 13:35:53 +0200 Subject: [PATCH 28/40] PEP8 --- deeptools/computeMatrix.py | 1 - 1 file changed, 1 deletion(-) diff --git a/deeptools/computeMatrix.py b/deeptools/computeMatrix.py index ec31cd6920..f3eff5113e 100644 --- a/deeptools/computeMatrix.py +++ b/deeptools/computeMatrix.py @@ -324,7 +324,6 @@ def computeMatrixOptArgs(case=['scale-regions', 'reference-point'][0]): 'contains a space E.g. --samplesLabel label-1 "label 2" ', nargs='+') - # in contrast to other tools, # computeMatrix by default outputs # messages and the --quiet flag supresses them From ec3ca93aee8194aab346a2dd6ff4016cf0ac5a3b Mon Sep 17 00:00:00 2001 From: dpryan79 Date: Fri, 5 May 2017 10:12:44 +0200 Subject: [PATCH 29/40] Update change log --- CHANGES.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.txt b/CHANGES.txt index f5cfce32c9..019513520f 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -7,6 +7,7 @@ * plotCoverage no longer clips the top off of plots. Further, you can now set the plot width and height with `--plotWidth` and `--plotHeight`. (issue #508) * In bamCoverage, specifying `--filterRNAstrand` no longer results in `--extendReads` being ignored. (issue #520) * `plotFingerprint` no longers requires producing a plot, which is useful if you only need QC metrics and are using a LOT of samples (such that matplotlib would crash anyway). This hasn't been implemented in Galaxy, but can if people would like it. (issue #519) + * `computeMatrix` now accepts a `--samplesLabel` option, which is useful in those cases when you aren't immediately running `plotHeatmap` and don't have terribly descriptive file names (issue #523) 2.5.0 From 3096cd6b6149ba13cd3e4455a8ab5b515b814694 Mon Sep 17 00:00:00 2001 From: dpryan79 Date: Fri, 5 May 2017 13:37:05 +0200 Subject: [PATCH 30/40] Mention in bamCoverage that filtering results in changes in scaling factors. --- docs/content/tools/bamCoverage.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/content/tools/bamCoverage.rst b/docs/content/tools/bamCoverage.rst index 5ccf691934..bcf095e16e 100644 --- a/docs/content/tools/bamCoverage.rst +++ b/docs/content/tools/bamCoverage.rst @@ -30,6 +30,8 @@ Usage hints .. note:: Like BAM files, bigWig files are compressed, binary files. If you would like to see the coverage values, choose the bedGraph output via ``--outFileFormat``. +.. note:: Note that the value used for normalization by RPKM or to 1x depth will always take filtering into account. In otherwords, it will be adjusted appropriately if you specify `--filterRNAstrand` or `--minMappingQuality` or anything of that sort. + Usage example for ChIP-seq ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ From eed391c7501f17c9f827a9fde3cde75cffb02a35 Mon Sep 17 00:00:00 2001 From: dpryan79 Date: Fri, 5 May 2017 16:36:58 +0200 Subject: [PATCH 31/40] Make the plot in plotEnrichment optional (CLI only) --- deeptools/plotEnrichment.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/deeptools/plotEnrichment.py b/deeptools/plotEnrichment.py index 70f843e02f..a370e77b66 100644 --- a/deeptools/plotEnrichment.py +++ b/deeptools/plotEnrichment.py @@ -78,8 +78,7 @@ def plot_enrichment_args(): 'so heatmap.pdf will save the heatmap in PDF format. ' 'The available formats are: .png, ' '.eps, .pdf and .svg.', - metavar='FILE', - required=True) + metavar='FILE') optional = parser.add_argument_group('Optional arguments') @@ -417,6 +416,9 @@ def main(args=None): args = parse_arguments().parse_args(args) + if not args.outRawCounts and not args.plotFile: + sys.exit("Error: You need to specify at least one of --plotFile or --outRawCounts!\n") + if args.labels is None: args.labels = args.bamfiles if len(args.labels) != len(args.bamfiles): @@ -485,7 +487,8 @@ def main(args=None): featureCounts[i][k] += v # Make a plot - plotEnrichment(args, featureCounts, totalCounts, features) + if args.plotFile: + plotEnrichment(args, featureCounts, totalCounts, features) # Raw counts if args.outRawCounts: From 1f63e1c35dd3510ab03bd1f1be7de1350e7b275b Mon Sep 17 00:00:00 2001 From: dpryan79 Date: Fri, 5 May 2017 17:04:15 +0200 Subject: [PATCH 32/40] Update change log [ci skip] --- CHANGES.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index 019513520f..2a72dcfc88 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -6,7 +6,7 @@ * You can now specify the plot width and height in `plotPCA` and `plotCorrelation` (heatmap only) with the `--plotWidth` and `--plotHeight` parameters. (issue #507) * plotCoverage no longer clips the top off of plots. Further, you can now set the plot width and height with `--plotWidth` and `--plotHeight`. (issue #508) * In bamCoverage, specifying `--filterRNAstrand` no longer results in `--extendReads` being ignored. (issue #520) - * `plotFingerprint` no longers requires producing a plot, which is useful if you only need QC metrics and are using a LOT of samples (such that matplotlib would crash anyway). This hasn't been implemented in Galaxy, but can if people would like it. (issue #519) + * `plotFingerprint` and `plotEnrichment` no longer require producing a plot, which is useful if you only need QC metrics and are using a LOT of samples (such that matplotlib would crash anyway). This hasn't been implemented in Galaxy, but can if people would like it. (issues #519 and #526) * `computeMatrix` now accepts a `--samplesLabel` option, which is useful in those cases when you aren't immediately running `plotHeatmap` and don't have terribly descriptive file names (issue #523) 2.5.0 From 7f9a58e3f3f717f2fcf072981616b99cb73bf6ab Mon Sep 17 00:00:00 2001 From: dpryan79 Date: Fri, 5 May 2017 21:58:21 +0200 Subject: [PATCH 33/40] Implement #527 --- deeptools/plotFingerprint.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/deeptools/plotFingerprint.py b/deeptools/plotFingerprint.py index 5b52cc7aa9..8c63ce7901 100644 --- a/deeptools/plotFingerprint.py +++ b/deeptools/plotFingerprint.py @@ -48,6 +48,11 @@ def process_args(args=None): args = parse_arguments().parse_args(args) + if args.JSDsample is not None and args.JSDsample not in args.bamfiles: + args.bamfiles.append(args.JSDsample) + if args.labels and len(args.bamfiles) == len(args.labels) - 1: + args.labels.append(args.JSDsample) + if args.labels and len(args.bamfiles) != len(args.labels): print("The number of labels does not match the number of BAM files.") exit(0) From e832da840303b23928a429c37954e6ff6ebfa66a Mon Sep 17 00:00:00 2001 From: dpryan79 Date: Fri, 5 May 2017 22:00:05 +0200 Subject: [PATCH 34/40] Update change log [ci skip] --- CHANGES.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.txt b/CHANGES.txt index 2a72dcfc88..1d8208d928 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -8,6 +8,7 @@ * In bamCoverage, specifying `--filterRNAstrand` no longer results in `--extendReads` being ignored. (issue #520) * `plotFingerprint` and `plotEnrichment` no longer require producing a plot, which is useful if you only need QC metrics and are using a LOT of samples (such that matplotlib would crash anyway). This hasn't been implemented in Galaxy, but can if people would like it. (issues #519 and #526) * `computeMatrix` now accepts a `--samplesLabel` option, which is useful in those cases when you aren't immediately running `plotHeatmap` and don't have terribly descriptive file names (issue #523) + * If you use `plotFingerprint` with the `--JSDsample` option and forget to list that file under `--bamfiles` it will be added automatically and the file name added to the labels if needed (issue #527) 2.5.0 From 4b97efc950d281e5a8835f5951d099dd7e120325 Mon Sep 17 00:00:00 2001 From: Devon Ryan Date: Fri, 5 May 2017 22:22:10 +0200 Subject: [PATCH 35/40] Fix the galaxy wrapper --- galaxy/wrapper/plotFingerprint.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy/wrapper/plotFingerprint.xml b/galaxy/wrapper/plotFingerprint.xml index 72cfae4d54..eadb71fcb9 100644 --- a/galaxy/wrapper/plotFingerprint.xml +++ b/galaxy/wrapper/plotFingerprint.xml @@ -24,7 +24,7 @@ #if $output.JSDsample: #if "'{}'".format($output.JSDsample.display_name) in $labels: #set JSDidx = $labels.index("'{}'".format($output.JSDsample.display_name)) - --JSDsample "./${JSDidx}.bam" + --JSDsample "${JSDidx}.bam" #end if #end if #end if From 707e5f39096ae6b8ed7a37130d4b880f736f5f51 Mon Sep 17 00:00:00 2001 From: Devon Ryan Date: Fri, 5 May 2017 22:55:58 +0200 Subject: [PATCH 36/40] fix galaxy tests --- .../wrapper/test-data/plotFingerprint_quality_metrics.tabular | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy/wrapper/test-data/plotFingerprint_quality_metrics.tabular b/galaxy/wrapper/test-data/plotFingerprint_quality_metrics.tabular index bb24fc0dc7..c48cebbf5a 100644 --- a/galaxy/wrapper/test-data/plotFingerprint_quality_metrics.tabular +++ b/galaxy/wrapper/test-data/plotFingerprint_quality_metrics.tabular @@ -1,3 +1,3 @@ Sample AUC Synthetic AUC X-intercept Synthetic X-intercept Elbow Point Synthetic Elbow Point JS Distance Synthetic JS Distance % genome enriched diff. enrichment CHANCE divergence bowtie2 test1.bam 0.00493632029864 0.481650684758 0.984443061605 1.15310443503e-24 0.984940883634 0.523268829811 NA 0.269004498068 NA NA NA -bowtie2 test1.bam 0.00493632029864 0.481650684758 0.984443061605 1.15310443503e-24 0.984940883634 0.523268829811 NA 0.269004498068 NA NA NA +bowtie2 test1.bam 0.00493632029864 0.481650684758 0.984443061605 1.15310443503e-24 0.984940883634 0.523268829811 0.0 0.269004498068 0 0 0 From cec2c46862bf425099ff455ce3b7d92e552dac79 Mon Sep 17 00:00:00 2001 From: dpryan79 Date: Mon, 8 May 2017 15:26:47 +0200 Subject: [PATCH 37/40] The QC metrics in plotFingerprint in Galaxy shouldn't require a JSDsample --- CHANGES.txt | 1 + galaxy/wrapper/plotFingerprint.xml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index 1d8208d928..5e72184b0c 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -9,6 +9,7 @@ * `plotFingerprint` and `plotEnrichment` no longer require producing a plot, which is useful if you only need QC metrics and are using a LOT of samples (such that matplotlib would crash anyway). This hasn't been implemented in Galaxy, but can if people would like it. (issues #519 and #526) * `computeMatrix` now accepts a `--samplesLabel` option, which is useful in those cases when you aren't immediately running `plotHeatmap` and don't have terribly descriptive file names (issue #523) * If you use `plotFingerprint` with the `--JSDsample` option and forget to list that file under `--bamfiles` it will be added automatically and the file name added to the labels if needed (issue #527) + * Various Galaxy wrapper fixes 2.5.0 diff --git a/galaxy/wrapper/plotFingerprint.xml b/galaxy/wrapper/plotFingerprint.xml index eadb71fcb9..b1fe806db8 100644 --- a/galaxy/wrapper/plotFingerprint.xml +++ b/galaxy/wrapper/plotFingerprint.xml @@ -87,7 +87,7 @@ - From 9855729c52c750e322451e1ab51d5d37bbdf3910 Mon Sep 17 00:00:00 2001 From: Devon Ryan Date: Thu, 11 May 2017 21:16:24 +0200 Subject: [PATCH 38/40] Example in plotCoverage should use -o rather than --out --- deeptools/plotCoverage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deeptools/plotCoverage.py b/deeptools/plotCoverage.py index 88958766c8..9c72fa3ba0 100644 --- a/deeptools/plotCoverage.py +++ b/deeptools/plotCoverage.py @@ -40,7 +40,7 @@ def parse_arguments(args=None): """, epilog='example usages:\nplotCoverage ' - '--bamfiles file1.bam file2.bam -out results.png\n\n' + '--bamfiles file1.bam file2.bam -o results.png\n\n' ' \n\n', conflict_handler='resolve') From 81b1fb74854814c611c03cad4ec81ce57f026bf4 Mon Sep 17 00:00:00 2001 From: dpryan79 Date: Fri, 12 May 2017 15:14:58 +0200 Subject: [PATCH 39/40] Bump version to 2.5.1 and convert in Galaxy to conda only --- deeptools/_version.py | 2 +- galaxy/wrapper/deepTools_macros.xml | 15 +-------------- galaxy/wrapper/tool_dependencies.xml | 9 --------- 3 files changed, 2 insertions(+), 24 deletions(-) delete mode 100644 galaxy/wrapper/tool_dependencies.xml diff --git a/deeptools/_version.py b/deeptools/_version.py index dafa7cfba2..8ba53eced4 100644 --- a/deeptools/_version.py +++ b/deeptools/_version.py @@ -2,4 +2,4 @@ # This file is originally generated from Git information by running 'setup.py # version'. Distribution tarballs contain a pre-generated copy of this file. -__version__ = '2.5.0.1' +__version__ = '2.5.1' diff --git a/galaxy/wrapper/deepTools_macros.xml b/galaxy/wrapper/deepTools_macros.xml index 75800c9408..14fc974c9b 100644 --- a/galaxy/wrapper/deepTools_macros.xml +++ b/galaxy/wrapper/deepTools_macros.xml @@ -1,10 +1,9 @@ --numberOfProcessors "\${GALAXY_SLOTS:-4}" - 2.5.0 + 2.5.1.0 - python deeptools @@ -124,18 +123,6 @@ - --numberOfProcessors "\${GALAXY_SLOTS:-4}" - 2.5.0.1 - - - python - deeptools - - - - @BINARY@ --version - - - - - - - - - - From 601d5de7525873082c4447d8104640761d05dc83 Mon Sep 17 00:00:00 2001 From: dpryan79 Date: Fri, 12 May 2017 17:00:57 +0200 Subject: [PATCH 40/40] Fix plotEnrichment --Offset issues in python3 --- deeptoolsintervals/enrichment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deeptoolsintervals/enrichment.py b/deeptoolsintervals/enrichment.py index a434a2197c..25524ca8f3 100644 --- a/deeptoolsintervals/enrichment.py +++ b/deeptoolsintervals/enrichment.py @@ -256,7 +256,7 @@ def findOverlaps(self, chrom, blocks, strand=".", matchType=0, strandType=0): oset = frozenset() for block in blocks: - overlaps = self.tree.findOverlappingFeatures(chrom, block[0], block[1], strand, matchType, strandType) + overlaps = self.tree.findOverlappingFeatures(chrom, int(block[0]), int(block[1]), strand, matchType, strandType) if overlaps is not None: oset = oset.union(frozenset(overlaps))