Upgrade pyramid

INSTALL.sh

@@ -10,7 +10,6 @@ sudo apt-get install emacs htop
 
 # Download/install pyramid + persona
 sudo easy_install "pyramid==1.4.5" 
-sudo easy_install "pyramid-persona==1.5"
 
 # Download and install SegAnnot and PrunedDP extension modules.
 cd

plotter/__init__.py

@@ -15,6 +15,7 @@ def main(global_config, **settings):
     """
     config = Configurator(settings=settings)
     config.include("seganndb_login")
+    config.include("pyramid_chameleon")
     config.add_static_view('static', 'static', cache_max_age=3600)
     config.add_route('home', '/')
     #config.add_route('delete_profiles', '/delete_profiles/')

plotter/db.py

@@ -1,3 +1,5 @@
+from collections import OrderedDict
+import pdb
 import cPickle as pickle
 import gzip
 import re
@@ -16,7 +18,7 @@
 from gradient_descent import mmir
 import scatterplot
 #import for image splitting
-import Image
+from PIL import Image
 # scatterplot sizes in pixels.
 # DEFAULT_WIDTH = 1500
 DEFAULT_WIDTH = 1250
@@ -31,7 +33,7 @@
     ("name", "[-a-zA-Z0-9]+"),
     ("type", "bedGraph"),
     ("maxSegments", "[0-9]+"),
-    ("db", "hg1[789]"),
+    ("db", "[a-zA-Z0-9]+"),
     ]
 HEADER_PATTERNS = dict(HEADER_TUPS)
 NAME_REGEX = re.compile(HEADER_PATTERNS["name"])
@@ -44,7 +46,7 @@
 for var, regex in TO_COMPILE:
     HEADER_REGEXES[var] = (regex, re.compile(regex))
 LINE_PATTERNS = [
-    "chr(?P<chromosome>[0-9XY]+)",
+    "(?P<chromosome>[0-9a-zA-Z]+)",
     "(?P<chromStart>[0-9]+)",
     "(?P<chromEnd>[0-9]+)",
     # the regexp that we use for validating the logratio column is
@@ -386,11 +388,12 @@ def key_min(self):
 
 
 class ChromLengths(Resource):
-    CHROM_ORDER = [str(x+1) for x in range(22)]+["X"]
-    CHROM_RANK = dict(zip(CHROM_ORDER, enumerate(CHROM_ORDER)))
     keys = ("db", )
     u = "http://hgdownload.soe.ucsc.edu/goldenPath/%s/database/chromInfo.txt.gz"
 
+    def chrom_order(self):
+        return self.get().keys()
+
     def make_details(self):
         s = self.values[0]
         local = os.path.join(CHROMLENGTH_DIR, s+".txt.gz")
@@ -403,15 +406,11 @@ def make_details(self):
         # print "reading %s" % local
         f = gzip.open(local)
         r = csv.reader(f, delimiter="\t")
-        chroms = dict([
-            (ch.replace("chr", ""), int(last))
+        chroms = OrderedDict([
+            (ch, int(last))
             for ch, last, ignore in r
             ])
-        return dict([
-            (ch, chroms[ch])
-            for ch in self.CHROM_ORDER
-            ])
-
+        return chroms
 
 def get_model(probes, break_after):
     """Calculate breaks and segments after PrunedDP."""
@@ -428,10 +427,10 @@ def get_model(probes, break_after):
     # error.
     json = {
         "breakpoints": tuple([
-            {"position": int(p)}
+            {"position": float(p)}
             for p in break_mid
         ]),
-        "segments": segments_json(probes, break_mid, mean),
+        "segments": segments_json(probes, break_mid.tolist(), mean),
         }
     return {
         # for quickly checking model agreement to annotated regions.
@@ -711,7 +710,9 @@ def get_export(self, user):
     def regions(self, user):
         dicts = []
         name = self.values[0]
-        for ch in ChromLengths.CHROM_ORDER:
+        D = Profile(name).get()
+        CHROM_ORDER = ChromLengths(D["db"]).get().keys()
+        for ch in CHROM_ORDER:
             for short, table in REGION_TABLES:
                 for d in table(user, name, ch).json():
                     d["type"] = short
@@ -790,7 +791,7 @@ def process(self):
             meta["intervals"] = get_intervals(sq_err)
             diffs = numpy.diff(probes["logratio"])
             features = [
-                math.log(numpy.median(numpy.abs(diffs))),
+                math.log(numpy.median(numpy.abs(diffs))+0.5),
                 math.log(len(probes["logratio"])),
                 ]
             meta["features"] = numpy.array(features)
@@ -877,17 +878,19 @@ class AnnotationCounts(Resource):
 
     def make_details(self):
         table_count = {}
+        D = Profile(self.info["name"]).get()
+        CHROM_ORDER = ChromLengths(D["db"]).get().keys()
         for short, table in REGION_TABLES:
             table_count[short] = {}
-            for ch in ChromLengths.CHROM_ORDER:
+            for ch in CHROM_ORDER:
                 r = table(self.info["user"], self.info["name"], ch)
                 table_count[short][ch] = r.count()
         counts = {}
-        for ch in ChromLengths.CHROM_ORDER:
+        for ch in CHROM_ORDER:
             counts[ch] = table_count["breakpoints"][ch]
         for short, d in table_count.iteritems():
             counts[short] = 0
-            for ch in ChromLengths.CHROM_ORDER:
+            for ch in CHROM_ORDER:
                 counts[short] += table_count[short][ch]
         return counts
 
@@ -1011,9 +1014,9 @@ def segments_json(probes, breaks, mean):
     seg_begin = [probes["chromStart"][0]-0.5]+breaks
     seg_end = breaks+[probes["chromStart"][-1]+0.5]
     return tuple([
-            {"logratio": float(m), "min": int(b), "max": int(e)}
+            {"logratio": float(m), "min": float(b), "max": float(e)}
             for m, b, e in zip(mean, seg_begin, seg_end)
-        ]),
+        ])
 
 def chrom_model(models, error, regions, profile, ch, user,
                 chrom_meta=None, user_model=None):
@@ -1055,11 +1058,12 @@ def chrom_model(models, error, regions, profile, ch, user,
                                probes["chromStart"],
                                min_array,
                                max_array)
-        break_mid = result["break_mid"].tolist()
+        break_array = (result["break_min"]+result["break_max"])/2 + 0.5
+        break_mid = break_array.tolist()
         model = {
             "segments": segments_json(probes, break_mid, result["mean"]),
             "breakpoints": tuple([
-                {"position": int(p)}
+                {"position": p}
                 for p in break_mid
                 ]),
             "segannot": True,

plotter/scatterplot.py

@@ -1,4 +1,4 @@
-import Image, ImageDraw
+from PIL import Image, ImageDraw
 
 def normalize(x,xmax,m=None,M=None):
     """Scale values to an integer in [0,xmax]."""

plotter/static/chromDisplay.js

@@ -214,14 +214,17 @@ function chromDisplay(svg, meta, plotter) {
             directlabel.remove();
             enable_new();
         }
+        var regionPixelsToBases = function(x_px){
+  	  return Math.round(x.invert(x_px));
+        }
         var saveAnnotation = function() {
             var buttons = svg.selectAll("." + button_class);
             buttons.remove();
             var rect = svg.select("#" + trackType + "NEW");
             var w = parseInt(rect.attr("width"));
             var min_px = parseInt(rect.attr("x"));
-            var min = parseInt(x.invert(min_px));
-            var max = parseInt(x.invert(min_px + w));
+            var min = regionPixelsToBases(min_px);
+            var max = regionPixelsToBases(min_px + w);
             var waiting = svg.append("text")
             .attr("x", min_px + w / 2)
             .attr("y", button_y)
@@ -255,8 +258,8 @@ function chromDisplay(svg, meta, plotter) {
         var doNothing = d3.behavior.drag();
         var newRegion = d3.behavior.drag()
         .on("dragstart", function(d) {
-            drag_origin = d3.mouse(this)[0];
-            var rect = svg.insert("rect", "line")
+          drag_origin = x(regionPixelsToBases(d3.mouse(this)[0]));
+          var rect = svg.insert("rect", "line")
             .attr("id", trackType + "NEW")
             .attr("y", trackY)
             .attr("x", drag_origin)
@@ -267,8 +270,9 @@ function chromDisplay(svg, meta, plotter) {
             ;
         })
         .on("drag", function(d) {
-            var r = getRegion(d3.event.x, drag_origin);
-            svg.select("#" + trackType + "NEW")
+	  var drag_x_px = x(regionPixelsToBases(d3.event.x));
+          var r = getRegion(drag_x_px, drag_origin);
+          svg.select("#" + trackType + "NEW")
             .attr("x", r["x"])
             .attr("width", r["width"])
             ;
@@ -513,25 +517,30 @@ function chromDisplay(svg, meta, plotter) {
         // changed.
         if (response.segments) {
             // draw guide lines first.
+	    var guide_data = [
+		{"logratio": 0}, 
+		{"logratio": 100},
+		{"logratio": 1000},
+		{"logratio": 10000},
+		{"logratio": 10}
+	    ];
+	    var guide_text = svg.selectAll("text.guide")
+		.data(guide_data)
+		.enter().append("text")
+		.attr("x", 0)
+		.attr("y", function(d){
+		    return y(d.logratio);
+		})
+		.classed("guide", 1)
+		.style("text-anchor", "left")
+		.text(function(d){
+		    return d.logratio;
+		})
+	    ;
             var guides = svg.selectAll("line.guide")
-            .data([{
-                "logratio": 0
-            }, {
-                "logratio": -1
-            }, {
-                "logratio": 1
-            }])
-            ;
-            // changing the x2 value to 1250, as each image will only be 1250
-            // pixels long
-            var guideActions = function(selection) {
-                var url = window.location.href;
-                var res = url.indexOf("profile_old");
-
-                // if (res == -1)
-                //     width = 1250;
-
-                selection.attr("x1", 0)
+		.data(guide_data)
+		.enter().append("line")
+		.attr("x1", 0)
                 .attr("x2", width)
                 .attr("y1", function(d) {
                     return y(d.logratio);
@@ -546,10 +555,7 @@ function chromDisplay(svg, meta, plotter) {
                     else
                         return "3px";
                 })
-                ;
-            }
-            guideActions(guides.enter().append("line"));
-            guideActions(guides);
+            ;
             var segmentation = svg.selectAll("line.segmentation")
             .data(response.segments);
             segmentation.enter().append("line");

plotter/static/upload_profiles.py

@@ -10,15 +10,15 @@
     ("name","[-a-zA-Z0-9]+"),
     ("type","bedGraph"),
     ("maxSegments","[0-9]+"),
-    ("db","hg1[789]"),
+    ("db","[a-zA-Z0-9]+"),
     ]
 header_regexes = {}
 for var, pattern in to_check:
     regex = "%s=(%s)"%(var,pattern)
     header_regexes[var] = (regex,re.compile(regex))
 
 line_patterns = [
-    "chr(?P<chromosome>[0-9XY]+)",
+    "(?P<chromosome>[0-9a-zA-Z]+)",
     "(?P<chromStart>[0-9]+)",
     "(?P<chromEnd>[0-9]+)",
     r"(?P<logratio>\S+)",

plotter/templates/base.pt

@@ -37,8 +37,7 @@ It handles basic user login/logout too
             <a href="http://validator.w3.org/check?uri=referer">W3C
             standard HTML5</a>
 
-            web site made by <a href="http://sugiyama-www.cs.titech.ac.jp/~toby/">Toby Dylan
-            Hocking</a> using
+            web site made by <a href="http://members.cbio.mines-paristech.fr/~thocking/">Toby Dylan Hocking</a> using
 
             <a href="http://www.gnu.org/software/emacs/tour/">Emacs</a>,
 
@@ -53,17 +52,6 @@ It handles basic user login/logout too
             <a href="http://d3js.org/">D3</a>.
           </td>
         </tr>
-        <tr>
-          <td>
-            Thanks to <a href="http://www.inria.fr/en/">INRIA</a> for hosting the
-            server, and <a href="https://gforge.inria.fr/">INRIA GForge</a> for
-            hosting
-            the <a href="http://www.gnu.org/licenses/quick-guide-gplv3.html">GPL-3
-            free
-            software</a> <a href="https://gforge.inria.fr/scm/viewvc.php/webapp/pyramid/?root=breakpoints">source
-            code</a> which runs this site.
-          </td>
-        </tr>
       </table>
     </footer>
     <script type="text/javascript" src="/static/auth.js"></script>

plotter/views.py

@@ -21,7 +21,6 @@ def authenticated_userid(request):
     except:
         # in case the cookie is not found it applies, unauthenticated user
         val = None;
-    print val
     return val
 
 def add_userid(fn):
@@ -222,8 +221,12 @@ def read_probes(lines, chrom_lengths):
     # are at least 2 probes.
     chrom_meta = {}
     for ch in chroms.keys():
+        if ch not in chrom_lengths:
+            raise ValueError(
+                ch + " not in possible chroms: " +
+                ",".join(chrom_lengths.keys()))
         probeList = chroms.pop(ch)
-        if len(probeList) > 1 and ch in chrom_lengths:
+        if len(probeList) > 1:
             probeList.sort(key=lambda tup: tup[0])  # position, logratio
             chromStart = numpy.array([
                 pos for pos, lr in probeList], numpy.int32)
@@ -357,7 +360,7 @@ def update_model(models, error, regions, profile, ch, user):
 def profile(request):
     return prof_info(
         request.matchdict["name"],
-        db.ChromLengths.CHROM_ORDER,
+        None,
         "profile")
 
 
@@ -374,10 +377,18 @@ def prof_info(name_str, chroms, size):
     out = {"names": name_str}
     if "," in name_str:
         namelist = name_str.split(",")
+        p = None
         out["p"] = None
     else:
         namelist = [name_str]
-        out["p"] = db.Profile(name_str).get()
+        p = db.Profile(name_str).get()
+        out["p"] = p
+    if chroms == None:
+        if p is None:
+            p = db.Profile(namelist[0]).get()
+        cl = db.ChromLengths(p["db"])
+        cl_info = cl.get()
+        chroms = cl_info.keys()
     out["plot"] = plotJS(namelist, chroms, size)
     return out
 

process_daemon.py

@@ -1,5 +1,4 @@
 from plotter.db import ProfileQueue
-
 while 1:
     ProfileQueue.process_one()
 

production.ini

@@ -7,8 +7,8 @@
 use = egg:plotter
 
 #pyramid.includes = pyramid_google_login
-security.google_login.client_id =  532435863603-7s950m1gt3h2ls8g3jm8kiqp419dpppp.apps.googleusercontent.com 
-security.google_login.client_secret =  kU9VH0ALE7nZmXV_QWi3H3d9 
+security.google_login.client_id =   532435863603-gba5ip9cmc8tbijpcbfe32iq58l1go6e.apps.googleusercontent.com
+security.google_login.client_secret =  iUEUTxRu49aiIey80MSzMh18
 
 pyramid.reload_templates = false
 pyramid.debug_authorization = false

recover-restart.sh

@@ -4,4 +4,5 @@ pkill -9 python
 db_recover -h db
 python process_daemon.py &
 python learn_daemon.py &
+##python ~/lib/python2.7/old-packages/pyramid/scripts/pserve.py --reload development.ini
 pserve --reload development.ini