Merge branch '4.0.0' of github.com:deeptools/deepTools into 4.0.0

deeptools · Jan 22, 2025 · 38a5b6b · 38a5b6b
2 parents 11c6c91 + 485be30
commit 38a5b6b
Show file tree

Hide file tree

Showing 53 changed files with 3,681 additions and 623 deletions.
diff --git a/.github/workflows/cargo_test.yml b/.github/workflows/cargo_test.yml
@@ -0,0 +1,14 @@
+name: Cargo
+on: [push, pull_request]
+
+jobs:
+  cargo_test:
+    name: run cargo test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: rustup update stable && rustup default stable
+      - name: cargo build
+        run: cargo build --verbose
+      - name: cargo test
+        run: cargo test --verbose
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -39,55 +39,27 @@ jobs:
   build-linux:
     name: Test on Linux
     runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v4
-    - uses: actions/setup-python@v5
-      with:
-        python-version: '3.12'
-        cache: 'pip'
-    - name: pip install
-      run: |
-        pip install .[actions]
-    - name: PEP8
-      run: |
-        micromamba activate test_and_build
-        flake8 . --exclude=.venv,.build,build --ignore=E501,F403,E402,F999,F405,E722,W504,W605
-    - name: Test deepTools
-      run: |
-        pytest -v
-    - name: make an artifact
-      run: |
-        rm -f dist/*
-        python -m build
-    - uses: actions/upload-artifact@v4
-      with:
-        name: "distfiles"
-        path: "dist"
-  test-wheels:
-    name: test wheel
-    runs-on: ubuntu-latest
-    needs: build-linux
     strategy:
       matrix:
         python-version: ['3.9','3.10', '3.11', '3.12']
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/download-artifact@v4
-        with:
-          name: "distfiles"
-          path: ~/dist/
       - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
           cache: 'pip'
-      - name: install wheel
+      - name: pip install
         run: |
-          pip install ~/dist/*whl
-          deeptools -h
-      - name: pytest
+          pip install .[actions]
+      - name: Test deepTools
         run: |
-          pip install pytest
           pytest -v
+      - name: Build wheel
+        run: |
+          python -m build
+      - name: Test wheel
+        run: |
+          pip install dist/*whl
   build-osx:
     name: Test on OSX
     runs-on: macOS-latest

diff --git a/.gitignore b/.gitignore
@@ -1,4 +1,5 @@
 *.tar.gz
+*.npz
 *.py[cod]
 
 # C extensions
@@ -60,3 +61,12 @@ env.yml
 # Rust stuff
 Cargo.lock
 target*
+
+# snakemake
+.snakemake
+*.smk
+
+# jupyter notebooks
+.ipynb_checkpoints
+*.ipynb_checkpoints
+*.ipynb
diff --git a/.planemo.sh b/.planemo.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 # Some versions of planemo don't handle symlinks
 unlink galaxy/wrapper/test-data/test.bw
-cp deeptools/test/test_heatmapper/test.bw galaxy/wrapper/test-data/test.bw
+cp pydeeptools/deeptools/test/test_heatmapper/test.bw galaxy/wrapper/test-data/test.bw
 
 if [[ $1 == "1" ]] ; then
     wrappers="galaxy/wrapper/alignmentSieve.xml \

diff --git a/Cargo.toml b/Cargo.toml
@@ -15,4 +15,8 @@ rayon = "1.10.0"
 itertools = "0.12.1"
 bigtools = "0.5.3"
 tokio = "*"
-flate2 = "*"
+flate2 = "*"
+tempfile = "*"
+ndarray = "0.16.1"
+npyz = "*"
+zip = "*"
diff --git a/deeptools4.0.0_changes.md b/deeptools4.0.0_changes.md
@@ -1,18 +1,39 @@
-# Counting
+# Changes
 
-Counting reads has slightly changed.  
-for -bs 1, the counting mechanism remains the same.  
-for -bs > 1, a read is split in multiple contiguous blocks  
-  - multiple blocks in 1 bin only count as 1  
-  - multiple blocks in multiple bins count as 1 per bin  
-  - one block spanning multiple bins counts as 1 in each bin  
+## computeMatrix
 
-# normalization
+ - --sortRegions 'no' option no longer exists
+ - Sorting ascend / descend no longer has subsorting by position.
+ - --quiet / -q option no longer exists.
+ - bed files in computeMatrix no longer support '#' to define groups.
+ - 'chromosome matching' i.e. chr1 <-> 1, chrMT <-> MT is no longer performed.
 
-Exactscaling is no longer an option, it's always performed.
+## normalization
+
+ - Exactscaling is no longer an option, it's always performed.
+
+## alignmentSieve
+
+- options label, smartLabels, genomeChunkLength are removed.
+- ignoreDuplicates is removed, and (if wanted) should be set by the SamFlagExclude setting.
+
+# Testing
+
+## computeMatrix
+ - referencePoint: TSS, center, TES
+ - sortRegions: descend, ascend, keep
+ - sortUsing: mean, median, max, min, sum, region_length
+ - averageTypeBins: mean, median, min, max ,std, sum
+ - skipZeros
+ - duplicate renaming _r1, _r2, ...
+ - GTF, BED3, BED6, BED12, mixedBED (?)
+ - scaleRegions, un5, un3, regionbodylength, metagene
+
+ ## alignmentSieve
+
+ - unmapped reads to unfiltered_out
 
 # Todo
 
- - allow multithreaded bw writing
- - properly divide region work over threads -> region sorting & taking size into account
-
+- AlignmentSieve: Shift, Bed, Optimization.
+- bamCoverage / bamCompare: filtering, extend.
diff --git a/galaxy/wrapper/alignmentSieve.xml b/galaxy/wrapper/alignmentSieve.xml
@@ -19,14 +19,14 @@
         @THREADS@
         -b one.bam
         --label '$label'
-        #if str($filterRNAstrand) != 'no':
+        #if $filterRNAstrand:
             --filterRNAstrand '$filterRNAstrand'
         #end if
 
         $ignoreDuplicates
 
         #if $minMappingQuality:
-            --minMappingQuality '$minMappingQuality'
+            --minMappingQuality $minMappingQuality
         #end if
 
         #if $samFlagInclude:
@@ -98,10 +98,9 @@
                   take strand into consideration for strand-specific
                   protocols. Note that only properly paired reads are considered."/>
         <param argument="--ATACshift" type="boolean" label="Shift fragment ends as appropriate for ATAC-seq"/>
-        <param argument="filterRNAstrand" type="select" label="Only include reads originating from fragments from the forward or reverse strand."
+        <param argument="filterRNAstrand" type="select" optional="true" label="Only include reads originating from fragments from the forward or reverse strand."
             help="By default (the no option), all reads are processed, regardless of the strand they originated from. For RNAseq, it can be useful to separately create bigWig files for the forward or reverse strands.
                   Note that this tools assumes that a dUTP-based method was used, so fragments will be assigned to the reverse strand if the second read in a pair is reverse complemented.">
-            <option value="no" selected="true">no</option>
             <option value="forward">forward</option>
             <option value="reverse">reverse</option>
         </param>
@@ -132,7 +131,7 @@
             <param name="bamfile" value="paired_chr2L.bam" ftype="bam"/>
             <param name="minMappingQuality" value="10"/>
             <param name="filterMetrics" value="True"/>
-            <output name="outFile" file="alignmentSieve.bam" ftype="bam" lines_diff="1"/>
+            <output name="outFile" file="alignmentSieve.bam" ftype="bam" lines_diff="2"/>
             <output name="filterMetricsFile" file="alignmentSieve.txt" ftype="tabular"/>
         </test>
         <test expect_num_outputs="1">
@@ -146,13 +145,13 @@
             <param name="bamfile" value="paired_chr2L.bam" ftype="bam"/>
             <param name="minMappingQuality" value="10"/>
             <param name="shift" value="1 -2 3 -4"/>
-            <output name="outFile" file="alignmentSieve2.bam" ftype="bam" lines_diff="2"/>
+            <output name="outFile" file="alignmentSieve2.bam" ftype="bam" lines_diff="4"/>
         </test>
         <test expect_num_outputs="1">
             <param name="bamfile" value="paired_chr2L.cram" ftype="cram"/>
             <param name="minMappingQuality" value="10"/>
             <param name="shift" value="1 -2 3 -4"/>
-            <output name="outFile" file="alignmentSieve3.bam" ftype="bam" lines_diff="2"/>
+            <output name="outFile" file="alignmentSieve3.bam" ftype="bam" lines_diff="4"/>
         </test>
     </tests>
     <help>

diff --git a/galaxy/wrapper/deepTools_macros.xml b/galaxy/wrapper/deepTools_macros.xml
@@ -49,7 +49,7 @@
         $advancedOpt.ignoreDuplicates
         $advancedOpt.centerReads
         #if $advancedOpt.minMappingQuality:
-            --minMappingQuality '$advancedOpt.minMappingQuality'
+            --minMappingQuality $advancedOpt.minMappingQuality
         #end if
         #if $advancedOpt.samFlagInclude:
             --samFlagInclude $advancedOpt.samFlagInclude
@@ -716,7 +716,7 @@ is vital to you, select Yes below.">
     </xml>
 
     <xml name="output_dpi">
-        <param argument="--dpi" name="dpi" type="integer" value="200" size="3" optional="True"
+        <param argument="--dpi" type="integer" value="200" size="3" optional="True"
             label="Image dpi" help=""/>
     </xml>
 

diff --git a/galaxy/wrapper/multiBamSummary.xml b/galaxy/wrapper/multiBamSummary.xml
@@ -97,13 +97,13 @@
         </data>
     </outputs>
     <tests>
-        <test>
+        <test expect_num_outputs="1">
             <param name="bamfiles" value="bowtie2 test1.bam,bowtie2 test1.bam" ftype="bam" />
             <param name="modeOpt" value="bins" />
             <param name="binSize" value="10" />
             <output name="outFile" file="multiBamSummary_result1.npz" ftype="deeptools_coverage_matrix" compare="sim_size" />
         </test>
-        <test>
+        <test expect_num_outputs="1">
             <param name="bamfiles" value="bowtie2 test1.bam,bowtie2 test1.bam" ftype="bam" />
             <param name="modeOpt" value="BED-file" />
             <param name="region_file" value="multiBamSummary_regions.bed" />

diff --git a/galaxy/wrapper/multiBigwigSummary.xml b/galaxy/wrapper/multiBigwigSummary.xml
@@ -19,7 +19,7 @@
             --outFileName $outFile
 
             --bwfiles #echo ' '.join($files)#
-            
+
             #if $custom_sample_labels_conditional.custom_labels_select == 'Yes'
                 --labels #echo ' '.join($custom_sample_labels_conditional.labels)#
             #end if
@@ -86,13 +86,13 @@
         </data>
     </outputs>
     <tests>
-        <test>
+        <test expect_num_outputs="1">
             <param name="bigwigfiles" value="test.bw,test.bw" ftype="bigwig" />
             <param name="modeOpt" value="bins" />
             <param name="binSize" value="10" />
             <output name="outFile" file="multiBigwigSummary_result1.npz" ftype="deeptools_coverage_matrix" compare="sim_size" />
         </test>
-        <test>
+        <test expect_num_outputs="2">
             <param name="bigwigfiles" value="test.bw,test.bw" ftype="bigwig" />
             <param name="modeOpt" value="bins" />
             <param name="binSize" value="10" />

diff --git a/galaxy/wrapper/plotCorrelation.xml b/galaxy/wrapper/plotCorrelation.xml
@@ -109,14 +109,14 @@
         </data>
     </outputs>
     <tests>
-        <test>
+        <test expect_num_outputs="2">
             <param name="corData" value="multiBamSummary_result1.npz" ftype="deeptools_coverage_matrix" />
             <param name="outFileFormat" value="png" />
             <param name="outFileCorMatrix" value="True" />
             <output name="matrix" file="plotCorrelation_result1.tabular" ftype="tabular" />
             <output name="outFileName" file="plotCorrelation_result1.png" ftype="png" compare="sim_size" delta="400" />
         </test>
-        <test>
+        <test expect_num_outputs="1">
             <param name="corData" value="multiBamSummary_result1.npz" ftype="deeptools_coverage_matrix" />
             <param name="outFileFormat" value="png" />
             <param name="whatToPlot" value="scatterplot" />

diff --git a/galaxy/wrapper/plotCoverage.xml b/galaxy/wrapper/plotCoverage.xml
@@ -114,7 +114,7 @@
         </data>
     </outputs>
     <tests>
-        <test>
+        <test expect_num_outputs="3">
             <param name="bamfiles" value="bowtie2 test1.bam,bowtie2 test1.bam" ftype="bam" />
             <!--param name="outFileFormat" value="png" /-->
             <param name="showAdvancedOpt" value="yes" />
@@ -123,7 +123,7 @@
             <output name="outFileRawCounts" file="plotCoverage_result1.tabular" ftype="tabular" />
             <output name="outFileName" file="plotCoverage_result1.png" ftype="png" compare="sim_size" delta="2400" />
         </test>
-        <test>
+        <test expect_num_outputs="2">
             <param name="bamfiles" value="bowtie2 test1.bam,bowtie2 test1.bam" ftype="bam" />
             <param name="showAdvancedOpt" value="yes" />
             <param name="plotTitle" value="Test Title from Galaxy" />

diff --git a/galaxy/wrapper/plotEnrichment.xml b/galaxy/wrapper/plotEnrichment.xml
@@ -48,7 +48,7 @@
                 $advancedOpt.perSample
 
                 $advancedOpt.variableScales
-                
+
                 $advancedOpt.perSample
 
                 $advancedOpt.variableScales
@@ -175,7 +175,7 @@
         </data>
     </outputs>
     <tests>
-        <test>
+        <test expect_num_outputs="2">
             <param name="bamfiles" value="bowtie2 test1.bam,bowtie2 test1.bam" ftype="bam" />
             <param name="BED" value="multiBamSummary_regions.bed,multiBamSummary_regions.bed" ftype="bed" />
             <param name="outRawCounts" value="true" />

diff --git a/galaxy/wrapper/plotFingerprint.xml b/galaxy/wrapper/plotFingerprint.xml
@@ -114,13 +114,13 @@
         </data>
     </outputs>
     <tests>
-        <test>
+        <test expect_num_outputs="1">
             <param name="bamfiles" value="bowtie2 test1.bam,bowtie2 test1.bam" ftype="bam" />
             <param name="showAdvancedOpt" value="no" />
             <param name="showOutputSettings" value="no" />
             <output name="outFileName" file="plotFingerprint_result1.png" ftype="png" compare="sim_size" />
         </test>
-        <test>
+        <test expect_num_outputs="3">
             <param name="bamfiles" value="bowtie2 test1.bam,bowtie2 test1.bam" ftype="bam" />
             <param name="showAdvancedOpt" value="yes" />
             <param name="showOutputSettings" value="yes" />