diff --git a/INSTALL.sh b/INSTALL.sh index 1652bf2..2c24cfb 100755 --- a/INSTALL.sh +++ b/INSTALL.sh @@ -10,7 +10,6 @@ sudo apt-get install emacs htop # Download/install pyramid + persona sudo easy_install "pyramid==1.4.5" -sudo easy_install "pyramid-persona==1.5" # Download and install SegAnnot and PrunedDP extension modules. cd diff --git a/plotter/__init__.py b/plotter/__init__.py index 8c0d03f..f750e5b 100644 --- a/plotter/__init__.py +++ b/plotter/__init__.py @@ -15,6 +15,7 @@ def main(global_config, **settings): """ config = Configurator(settings=settings) config.include("seganndb_login") + config.include("pyramid_chameleon") config.add_static_view('static', 'static', cache_max_age=3600) config.add_route('home', '/') #config.add_route('delete_profiles', '/delete_profiles/') diff --git a/plotter/db.py b/plotter/db.py index 9ce74e1..0573f11 100644 --- a/plotter/db.py +++ b/plotter/db.py @@ -1,3 +1,5 @@ +from collections import OrderedDict +import pdb import cPickle as pickle import gzip import re @@ -16,7 +18,7 @@ from gradient_descent import mmir import scatterplot #import for image splitting -import Image +from PIL import Image # scatterplot sizes in pixels. # DEFAULT_WIDTH = 1500 DEFAULT_WIDTH = 1250 @@ -31,7 +33,7 @@ ("name", "[-a-zA-Z0-9]+"), ("type", "bedGraph"), ("maxSegments", "[0-9]+"), - ("db", "hg1[789]"), + ("db", "[a-zA-Z0-9]+"), ] HEADER_PATTERNS = dict(HEADER_TUPS) NAME_REGEX = re.compile(HEADER_PATTERNS["name"]) @@ -44,7 +46,7 @@ for var, regex in TO_COMPILE: HEADER_REGEXES[var] = (regex, re.compile(regex)) LINE_PATTERNS = [ - "chr(?P[0-9XY]+)", + "(?P[0-9a-zA-Z]+)", "(?P[0-9]+)", "(?P[0-9]+)", # the regexp that we use for validating the logratio column is @@ -386,11 +388,12 @@ def key_min(self): class ChromLengths(Resource): - CHROM_ORDER = [str(x+1) for x in range(22)]+["X"] - CHROM_RANK = dict(zip(CHROM_ORDER, enumerate(CHROM_ORDER))) keys = ("db", ) u = "http://hgdownload.soe.ucsc.edu/goldenPath/%s/database/chromInfo.txt.gz" + def chrom_order(self): + return self.get().keys() + def make_details(self): s = self.values[0] local = os.path.join(CHROMLENGTH_DIR, s+".txt.gz") @@ -403,15 +406,11 @@ def make_details(self): # print "reading %s" % local f = gzip.open(local) r = csv.reader(f, delimiter="\t") - chroms = dict([ - (ch.replace("chr", ""), int(last)) + chroms = OrderedDict([ + (ch, int(last)) for ch, last, ignore in r ]) - return dict([ - (ch, chroms[ch]) - for ch in self.CHROM_ORDER - ]) - + return chroms def get_model(probes, break_after): """Calculate breaks and segments after PrunedDP.""" @@ -428,10 +427,10 @@ def get_model(probes, break_after): # error. json = { "breakpoints": tuple([ - {"position": int(p)} + {"position": float(p)} for p in break_mid ]), - "segments": segments_json(probes, break_mid, mean), + "segments": segments_json(probes, break_mid.tolist(), mean), } return { # for quickly checking model agreement to annotated regions. @@ -711,7 +710,9 @@ def get_export(self, user): def regions(self, user): dicts = [] name = self.values[0] - for ch in ChromLengths.CHROM_ORDER: + D = Profile(name).get() + CHROM_ORDER = ChromLengths(D["db"]).get().keys() + for ch in CHROM_ORDER: for short, table in REGION_TABLES: for d in table(user, name, ch).json(): d["type"] = short @@ -790,7 +791,7 @@ def process(self): meta["intervals"] = get_intervals(sq_err) diffs = numpy.diff(probes["logratio"]) features = [ - math.log(numpy.median(numpy.abs(diffs))), + math.log(numpy.median(numpy.abs(diffs))+0.5), math.log(len(probes["logratio"])), ] meta["features"] = numpy.array(features) @@ -877,17 +878,19 @@ class AnnotationCounts(Resource): def make_details(self): table_count = {} + D = Profile(self.info["name"]).get() + CHROM_ORDER = ChromLengths(D["db"]).get().keys() for short, table in REGION_TABLES: table_count[short] = {} - for ch in ChromLengths.CHROM_ORDER: + for ch in CHROM_ORDER: r = table(self.info["user"], self.info["name"], ch) table_count[short][ch] = r.count() counts = {} - for ch in ChromLengths.CHROM_ORDER: + for ch in CHROM_ORDER: counts[ch] = table_count["breakpoints"][ch] for short, d in table_count.iteritems(): counts[short] = 0 - for ch in ChromLengths.CHROM_ORDER: + for ch in CHROM_ORDER: counts[short] += table_count[short][ch] return counts @@ -1011,9 +1014,9 @@ def segments_json(probes, breaks, mean): seg_begin = [probes["chromStart"][0]-0.5]+breaks seg_end = breaks+[probes["chromStart"][-1]+0.5] return tuple([ - {"logratio": float(m), "min": int(b), "max": int(e)} + {"logratio": float(m), "min": float(b), "max": float(e)} for m, b, e in zip(mean, seg_begin, seg_end) - ]), + ]) def chrom_model(models, error, regions, profile, ch, user, chrom_meta=None, user_model=None): @@ -1055,11 +1058,12 @@ def chrom_model(models, error, regions, profile, ch, user, probes["chromStart"], min_array, max_array) - break_mid = result["break_mid"].tolist() + break_array = (result["break_min"]+result["break_max"])/2 + 0.5 + break_mid = break_array.tolist() model = { "segments": segments_json(probes, break_mid, result["mean"]), "breakpoints": tuple([ - {"position": int(p)} + {"position": p} for p in break_mid ]), "segannot": True, diff --git a/plotter/scatterplot.py b/plotter/scatterplot.py index e806788..7f554f4 100644 --- a/plotter/scatterplot.py +++ b/plotter/scatterplot.py @@ -1,4 +1,4 @@ -import Image, ImageDraw +from PIL import Image, ImageDraw def normalize(x,xmax,m=None,M=None): """Scale values to an integer in [0,xmax].""" diff --git a/plotter/static/chromDisplay.js b/plotter/static/chromDisplay.js index 7b62057..c428794 100644 --- a/plotter/static/chromDisplay.js +++ b/plotter/static/chromDisplay.js @@ -214,14 +214,17 @@ function chromDisplay(svg, meta, plotter) { directlabel.remove(); enable_new(); } + var regionPixelsToBases = function(x_px){ + return Math.round(x.invert(x_px)); + } var saveAnnotation = function() { var buttons = svg.selectAll("." + button_class); buttons.remove(); var rect = svg.select("#" + trackType + "NEW"); var w = parseInt(rect.attr("width")); var min_px = parseInt(rect.attr("x")); - var min = parseInt(x.invert(min_px)); - var max = parseInt(x.invert(min_px + w)); + var min = regionPixelsToBases(min_px); + var max = regionPixelsToBases(min_px + w); var waiting = svg.append("text") .attr("x", min_px + w / 2) .attr("y", button_y) @@ -255,8 +258,8 @@ function chromDisplay(svg, meta, plotter) { var doNothing = d3.behavior.drag(); var newRegion = d3.behavior.drag() .on("dragstart", function(d) { - drag_origin = d3.mouse(this)[0]; - var rect = svg.insert("rect", "line") + drag_origin = x(regionPixelsToBases(d3.mouse(this)[0])); + var rect = svg.insert("rect", "line") .attr("id", trackType + "NEW") .attr("y", trackY) .attr("x", drag_origin) @@ -267,8 +270,9 @@ function chromDisplay(svg, meta, plotter) { ; }) .on("drag", function(d) { - var r = getRegion(d3.event.x, drag_origin); - svg.select("#" + trackType + "NEW") + var drag_x_px = x(regionPixelsToBases(d3.event.x)); + var r = getRegion(drag_x_px, drag_origin); + svg.select("#" + trackType + "NEW") .attr("x", r["x"]) .attr("width", r["width"]) ; @@ -513,25 +517,30 @@ function chromDisplay(svg, meta, plotter) { // changed. if (response.segments) { // draw guide lines first. + var guide_data = [ + {"logratio": 0}, + {"logratio": 100}, + {"logratio": 1000}, + {"logratio": 10000}, + {"logratio": 10} + ]; + var guide_text = svg.selectAll("text.guide") + .data(guide_data) + .enter().append("text") + .attr("x", 0) + .attr("y", function(d){ + return y(d.logratio); + }) + .classed("guide", 1) + .style("text-anchor", "left") + .text(function(d){ + return d.logratio; + }) + ; var guides = svg.selectAll("line.guide") - .data([{ - "logratio": 0 - }, { - "logratio": -1 - }, { - "logratio": 1 - }]) - ; - // changing the x2 value to 1250, as each image will only be 1250 - // pixels long - var guideActions = function(selection) { - var url = window.location.href; - var res = url.indexOf("profile_old"); - - // if (res == -1) - // width = 1250; - - selection.attr("x1", 0) + .data(guide_data) + .enter().append("line") + .attr("x1", 0) .attr("x2", width) .attr("y1", function(d) { return y(d.logratio); @@ -546,10 +555,7 @@ function chromDisplay(svg, meta, plotter) { else return "3px"; }) - ; - } - guideActions(guides.enter().append("line")); - guideActions(guides); + ; var segmentation = svg.selectAll("line.segmentation") .data(response.segments); segmentation.enter().append("line"); diff --git a/plotter/static/upload_profiles.py b/plotter/static/upload_profiles.py index 11ef57a..e13f25e 100644 --- a/plotter/static/upload_profiles.py +++ b/plotter/static/upload_profiles.py @@ -10,7 +10,7 @@ ("name","[-a-zA-Z0-9]+"), ("type","bedGraph"), ("maxSegments","[0-9]+"), - ("db","hg1[789]"), + ("db","[a-zA-Z0-9]+"), ] header_regexes = {} for var, pattern in to_check: @@ -18,7 +18,7 @@ header_regexes[var] = (regex,re.compile(regex)) line_patterns = [ - "chr(?P[0-9XY]+)", + "(?P[0-9a-zA-Z]+)", "(?P[0-9]+)", "(?P[0-9]+)", r"(?P\S+)", diff --git a/plotter/templates/base.pt b/plotter/templates/base.pt index 388aca9..7e0d8d0 100644 --- a/plotter/templates/base.pt +++ b/plotter/templates/base.pt @@ -37,8 +37,7 @@ It handles basic user login/logout too W3C standard HTML5 - web site made by Toby Dylan - Hocking using + web site made by Toby Dylan Hocking using Emacs, @@ -53,17 +52,6 @@ It handles basic user login/logout too D3. - - - Thanks to INRIA for hosting the - server, and INRIA GForge for - hosting - the GPL-3 - free - software source - code which runs this site. - - diff --git a/plotter/views.py b/plotter/views.py index e1d1f4c..003c20e 100644 --- a/plotter/views.py +++ b/plotter/views.py @@ -21,7 +21,6 @@ def authenticated_userid(request): except: # in case the cookie is not found it applies, unauthenticated user val = None; - print val return val def add_userid(fn): @@ -222,8 +221,12 @@ def read_probes(lines, chrom_lengths): # are at least 2 probes. chrom_meta = {} for ch in chroms.keys(): + if ch not in chrom_lengths: + raise ValueError( + ch + " not in possible chroms: " + + ",".join(chrom_lengths.keys())) probeList = chroms.pop(ch) - if len(probeList) > 1 and ch in chrom_lengths: + if len(probeList) > 1: probeList.sort(key=lambda tup: tup[0]) # position, logratio chromStart = numpy.array([ pos for pos, lr in probeList], numpy.int32) @@ -357,7 +360,7 @@ def update_model(models, error, regions, profile, ch, user): def profile(request): return prof_info( request.matchdict["name"], - db.ChromLengths.CHROM_ORDER, + None, "profile") @@ -374,10 +377,18 @@ def prof_info(name_str, chroms, size): out = {"names": name_str} if "," in name_str: namelist = name_str.split(",") + p = None out["p"] = None else: namelist = [name_str] - out["p"] = db.Profile(name_str).get() + p = db.Profile(name_str).get() + out["p"] = p + if chroms == None: + if p is None: + p = db.Profile(namelist[0]).get() + cl = db.ChromLengths(p["db"]) + cl_info = cl.get() + chroms = cl_info.keys() out["plot"] = plotJS(namelist, chroms, size) return out diff --git a/process_daemon.py b/process_daemon.py index b5bf5fc..94cd63f 100644 --- a/process_daemon.py +++ b/process_daemon.py @@ -1,5 +1,4 @@ from plotter.db import ProfileQueue - while 1: ProfileQueue.process_one() diff --git a/production.ini b/production.ini index 636e607..4a31521 100644 --- a/production.ini +++ b/production.ini @@ -7,8 +7,8 @@ use = egg:plotter #pyramid.includes = pyramid_google_login -security.google_login.client_id = 532435863603-7s950m1gt3h2ls8g3jm8kiqp419dpppp.apps.googleusercontent.com -security.google_login.client_secret = kU9VH0ALE7nZmXV_QWi3H3d9 +security.google_login.client_id = 532435863603-gba5ip9cmc8tbijpcbfe32iq58l1go6e.apps.googleusercontent.com +security.google_login.client_secret = iUEUTxRu49aiIey80MSzMh18 pyramid.reload_templates = false pyramid.debug_authorization = false diff --git a/recover-restart.sh b/recover-restart.sh index f28a8d0..e35f29e 100755 --- a/recover-restart.sh +++ b/recover-restart.sh @@ -4,4 +4,5 @@ pkill -9 python db_recover -h db python process_daemon.py & python learn_daemon.py & +##python ~/lib/python2.7/old-packages/pyramid/scripts/pserve.py --reload development.ini pserve --reload development.ini diff --git a/server-backup.sh b/server-backup.sh index 03ae9c4..e59915b 100755 --- a/server-backup.sh +++ b/server-backup.sh @@ -1,5 +1,10 @@ +#!/bin/bash +set -o errexit bash server-stop.sh -pushd /var/www -sudo -u www-data cp -r db secret /home/www-data/backup +PREFIX=/var/www +BACKUP=$PREFIX/backup +pushd $PREFIX +sudo -u apache mkdir -p $BACKUP +sudo -u apache cp -r db secret $BACKUP popd bash server-start.sh diff --git a/server-recover-restart.sh b/server-recover-restart.sh index 2e9d3dc..8852dc7 100755 --- a/server-recover-restart.sh +++ b/server-recover-restart.sh @@ -1,3 +1,4 @@ #!/bin/bash +set -o errexit bash server-stop.sh -bash server-start.sh \ No newline at end of file +bash server-start.sh diff --git a/server-reinitialize.sh b/server-reinitialize.sh index 235b2f5..a5067ae 100755 --- a/server-reinitialize.sh +++ b/server-reinitialize.sh @@ -1,7 +1,4 @@ #!/bin/bash -sudo /etc/init.d/apache2 stop -sudo -u www-data pkill -9 python -sudo -u www-data rm -rf /var/www/db/* -sudo -u www-data python process_daemon.py & -sudo -u www-data python learn_daemon.py & -sudo /etc/init.d/apache2 start \ No newline at end of file +bash server-stop.sh +sudo -u apache rm -rf /var/www/db/* +bash server-start.sh diff --git a/server-start.sh b/server-start.sh index 1fe2c08..bd3b6ba 100755 --- a/server-start.sh +++ b/server-start.sh @@ -1,4 +1,6 @@ -sudo -u www-data db_recover -h /var/www/db -sudo -u www-data python process_daemon.py & -sudo -u www-data python learn_daemon.py & -sudo /etc/init.d/apache2 start +#!/bin/bash +set -o errexit +sudo -u apache db_recover -h /var/www/db +sudo -u apache python process_daemon.py & +sudo -u apache python learn_daemon.py & +sudo /sbin/httpd -k start diff --git a/server-stop.sh b/server-stop.sh index dc20ee3..6b62abe 100755 --- a/server-stop.sh +++ b/server-stop.sh @@ -1,2 +1,4 @@ -sudo /etc/init.d/apache2 stop -sudo -u www-data pkill -9 python +#!/bin/bash +set -o errexit +sudo /sbin/httpd -k stop +sudo -u apache pkill -9 python diff --git a/server-update.sh b/server-update.sh index 562bb91..dc708c1 100755 --- a/server-update.sh +++ b/server-update.sh @@ -1,5 +1,6 @@ #!/bin/bash +set -o errexit bash server-stop.sh -git pull +##git pull sudo python setup.py install bash server-start.sh diff --git a/setup.py b/setup.py index 55dd709..1691ccc 100644 --- a/setup.py +++ b/setup.py @@ -9,8 +9,9 @@ requires = [ 'pyramid', 'seganndb_login', - 'pyramid_debugtoolbar', - 'waitress', + #'pyramid_debugtoolbar', + 'pyramid_chameleon', + #'waitress', ] setup(name='plotter', diff --git a/wsgi.py b/wsgi.py index 6d554c9..a865388 100644 --- a/wsgi.py +++ b/wsgi.py @@ -1,3 +1,6 @@ +import sys +sys.path.insert(0, "/home/th798/lib/python2.7/site-packages") +print sys.path from pyramid.paster import get_app -application = get_app('/home/ubuntu/SegAnnDB/production.ini', 'main') +application = get_app('/build/SegAnnDB/production.ini', 'main')