diff --git a/INSTALL.sh b/INSTALL.sh index 46f1088..f3ee7f6 100755 --- a/INSTALL.sh +++ b/INSTALL.sh @@ -30,6 +30,23 @@ fi sed -i 's#^FILE_PREFIX.*$#FILE_PREFIX = "/var/www"#' plotter/db.py sudo python setup.py install +if [ "$TRAVIS" = "true" ]; then + cd $TRAVIS_BUILD_DIR +else + cd +fi +mkdir kent +cd kent +if [ uname != "Darwin" ]; then + rsync -a -P rsync://hgdownload.soe.ucsc.edu/genome/admin/exe/linux.x86_64/bedToBigBed ./ + rsync -a -P rsync://hgdownload.soe.ucsc.edu/genome/admin/exe/linux.x86_64/bedGraphToBigWig ./ +else + rsync -a -P rsync://hgdownload.soe.ucsc.edu/genome/admin/exe/macOSX.x86_64/bedToBigBed ./ + rsync -a -P rsync://hgdownload.soe.ucsc.edu/genome/admin/exe/macOSX.x86_64/bedGraphToBigWig ./ +fi +sudo cp bedToBigBed /usr/local/bin/ +sudo cp bedGraphToBigWig /usr/local/bin/ + # for an apache web server sudo apt-get install apache2 libapache2-mod-wsgi cd /var/www diff --git a/plotter/__init__.py b/plotter/__init__.py index 8c0d03f..fd43e9d 100644 --- a/plotter/__init__.py +++ b/plotter/__init__.py @@ -29,6 +29,10 @@ def main(global_config, **settings): config.add_route('add_region', '/add_region/{name}/{chr}/{trackType}/{annotation}/{min}/{max}/') config.add_route("export","/export/{user}/{name}/{what}/{format}/") + config.add_route('trackhub_export', '/trackhub_init/') + config.add_route('trackhub_details', '/trackhub_details/{short_name}/') + config.add_route('trackhub_file', '/trackhub/{short_name}/*file') + config.add_route('trackhub_list', '/trackhub_list/') name_regex = db.HEADER_PATTERNS["name"] # config.add_route("secret","/secret/{name:%s}{suffix}"%name_regex) config.add_route("secret","/secret/{profile_name}/{name:%s}{suffix}"%name_regex) diff --git a/plotter/db.py b/plotter/db.py index b6703f7..c65b162 100644 --- a/plotter/db.py +++ b/plotter/db.py @@ -855,6 +855,20 @@ def process(self): # print time.time()-before, "seconds elapsed" +class Trackhub(Resource): + keys = ("short_name",) + + def make_details(self): + return {} + + +class UserTrackhubs(Resource): + keys = ("trackhubs",) + + def make_details(self): + return [] + + class Models(Resource): keys = ("name", "chr") diff --git a/plotter/templates/export_table.pt b/plotter/templates/export_table.pt new file mode 100644 index 0000000..cc7b6ab --- /dev/null +++ b/plotter/templates/export_table.pt @@ -0,0 +1,42 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + +
dbUploaded byonProbesCopy#BkptAnnotationsSegmentationBreakpointsLinks
+ + ${p.name} + + + ${p.name} + ${p.db} + you [Delete] + + ${p.uploader} + ${p.uploaded_on.strftime("%d-%b-%Y")}${p.share}${p.probes} +
${p.copies}
+ 		+
${p.breakpoints}
+
+ diff --git a/plotter/templates/profile_table.pt b/plotter/templates/profile_table.pt index e034d30..4db86d5 100644 --- a/plotter/templates/profile_table.pt +++ b/plotter/templates/profile_table.pt @@ -3,9 +3,14 @@ If the profile name link is not active for a recently-uploaded profile, you need to wait a few minutes for the server to process the data.

+ + + +
+ diff --git a/plotter/templates/trackhub_details.pt b/plotter/templates/trackhub_details.pt new file mode 100644 index 0000000..2ee82a1 --- /dev/null +++ b/plotter/templates/trackhub_details.pt @@ -0,0 +1,35 @@ + + + + SegAnnDB: Trackhub Details + + +
+ +

+
+ Trackhub data for ${short_name} +

+
+

+ Description: ${long_name} +

+
+

+ URL: ${url} +
+ Paste this into UCSC's My Hub page to view your trackhub. +

+
+

+ Profiles: +
+

+
+ +
+

Return to home.

+ + \ No newline at end of file diff --git a/plotter/templates/trackhub_export.pt b/plotter/templates/trackhub_export.pt new file mode 100644 index 0000000..9d5a753 --- /dev/null +++ b/plotter/templates/trackhub_export.pt @@ -0,0 +1,26 @@ + + + > + SegAnnDB: Export Trackhub + + +
+

Please fill out the profile data.

+ +

+ + + + + + +

+ + + + +

Return to home.

+ +
+ \ No newline at end of file diff --git a/plotter/templates/trackhub_list.pt b/plotter/templates/trackhub_list.pt new file mode 100644 index 0000000..3f33258 --- /dev/null +++ b/plotter/templates/trackhub_list.pt @@ -0,0 +1,17 @@ + + + + SegAnnDB: Trackhub List for ${user} + + + +

+ Trackhubs: +

+

+

+
+ +

diff --git a/plotter/views.py b/plotter/views.py index e1d1f4c..0e4dcdc 100644 --- a/plotter/views.py +++ b/plotter/views.py @@ -1,3 +1,4 @@ +from pyramid.httpexceptions import HTTPFound from pyramid.view import view_config from pyramid.exceptions import Forbidden from pyramid.httpexceptions import HTTPFound @@ -11,6 +12,10 @@ import os from datetime import datetime import json +import subprocess +from shutil import copy2 +import gzip +import operator # I am trying to override the authenticated_userid function here # retrieve the cookie and return to the user @@ -626,8 +631,31 @@ def upload_profile_user_file(userid, upload): f.seek(0) out_name = db.secret_file("%(name)s.bedGraph.gz" % info) saved = gzip.open(out_name, "w") + saved.write(header) + # https://stackoverflow.com/questions/9835762/how-do-i-find-the-duplicates-in-a-list-and-create-another-list-with-them/9835819#9835819 + probelist = set() + deduplist = [] + finallist = [] for data in f: - saved.write(data) + if data != header: + linelist = data.split('\t') # seperates line by tabs + if linelist == ['']: # if the line is blank, skip it + continue + if len(linelist) >= 2: # ignores first line + if linelist[0][3:] not in cl.keys(): # removes invalid chromosomes + continue + if linelist[0] + linelist[1] in probelist: # removes duplicates + continue + probelist.add(linelist[0] + linelist[1]) + linelist[1] = int(linelist[1]) + deduplist.append(linelist) + # fancy sorting by: https://stackoverflow.com/questions/5212870/sorting-a-python-list-by-two-fields + sortedlist = sorted(deduplist, key=operator.itemgetter(0, 1)) + for sublist in sortedlist: + sublist[1] = str(sublist[1]) + finalline = '\t'.join(sublist) + finallist.append(finalline) + saved.write('\n'.join(finallist)) saved.close() # add profile to user lists. uprofs = [db.UserProfiles(u) for u in db_users] @@ -818,14 +846,222 @@ def export(request): return response -def respond_bed_csv(table, fmt, hinfo, dicts): +@view_config(route_name="trackhub_export", + request_method="POST", + renderer="templates/trackhub_export.pt") +def export_trackhub(request): + # get filename and db version + os.makedirs(os.getcwd()+'/trackhubs/' + request.POST['short_label'] + '/') + olddir = os.getcwd() + os.chdir('trackhubs/' + request.POST['short_label']) + hubtxt = open('hub.txt', 'w') + hubtxt.write('hub ' + request.POST['short_label'] + '\nshortLabel ' + request.POST['short_label'] + '\nlongLabel ' + + request.POST['long_label'] + '\ngenomesFile genomes.txt\nemail ' + authenticated_userid(request)) + hubtxt.close() + genomestxt = open('genomes.txt', 'w') + genomestxt.write('genome ' + request.POST['db'] + '\ntrackDb ' + request.POST['db'] + '/trackDb.txt') + genomestxt.close() + os.mkdir(request.POST['db']) + os.chdir(request.POST['db']) + chromsizes = open('chrom.sizes', 'w') + copy2(db.CHROMLENGTH_DIR + '/' + request.POST['db'] + '.txt.gz', os.getcwd() + '/chrom.sizes.gz') + gzipsizefile = gzip.open(os.getcwd() + '/chrom.sizes.gz') + chromsizes.write(gzipsizefile.read()) + chromsizes.close() + os.remove("chrom.sizes.gz") + usedlist = [] + for x in request.POST.getall('profile'): + pro = db.Profile(x) + pinfo = pro.get() + for datatype in ["breaks", "regions", "copies", "segments"]: + fun = getattr(pro, datatype) + # need to + dicts = fun(authenticated_userid(request)) + pinfo["table"] = datatype + pinfo["visibility"] = EXPORT_VISIBILITY[datatype] + for d in dicts: + d["user_id"] = authenticated_userid(request) + d["profile_id"] = x + if datatype != "segments": + tosplitbed = respond_bed_csv(datatype, "bed", pinfo, dicts, False).text + else: + tosplitbed = respond_bed_csv(datatype, "bedGraph", pinfo, dicts, False).text + if tosplitbed == '': + continue + tosplitbedlist = tosplitbed.split('\n') + tosortbedlist = [] + for line in tosplitbedlist: + if line != '': + tosortbedlist.append(line.split(' ')) + tounsplitbedlist = sorted(tosortbedlist, key=lambda lambdaline: (unicode(lambdaline[0]), int(lambdaline[1]))) + unsplitbedlist = [] + for line in tounsplitbedlist: + unsplitbedlist.append('\t'.join(line)) + tosavebed = '\n'.join(unsplitbedlist) + if datatype != "segments": + bedfile = open(x+"_"+datatype+'.bed', 'w') + else: + bedfile = open(x+"_"+datatype+'.bedGraph', 'w') + bedfile.write(tosavebed) + bedfile.close() + usedlist.append(datatype) + if datatype != "segments": + subprocess.call(['bedToBigBed', x+"_"+datatype+'.bed', 'chrom.sizes', x+"_"+datatype+'.bigbed']) + else: + subprocess.call(['bedGraphToBigWig', x + "_segments.bedGraph", 'chrom.sizes', x + "_segments.bigWig"]) + if x != "ES0004" and datatype != "segments": + os.remove(x+"_"+datatype+'.bed') + elif x != "ES0004": + os.remove(x+"_segments.bedGraph") + bedgraphsecretfilelocation = db.secret_file(x+'.bedGraph.gz') + copy2(bedgraphsecretfilelocation, os.getcwd()+'/'+x+'.bedGraph.gz') + gzipfile = gzip.open(os.getcwd()+'/'+x+'.bedGraph.gz') + gzipfile.next() + bedgraphdata = gzipfile.read() + gzipfile.close() + os.remove(os.getcwd()+'/'+x+'.bedGraph.gz') + bedgraph = open(x+'.bedGraph', 'w') + bedgraph.write(bedgraphdata) + bedgraph.close() + subprocess.call(['bedGraphToBigWig', x+'.bedGraph', 'chrom.sizes', x+'.bigWig']) + os.remove(x+'.bedGraph') + trackdbtxt = open('trackDb.txt', 'w') + for x in request.POST.getall('profile'): + trackdbtxt.write("""track %smultiWig +type bigWig +container multiWig +aggregate transparentOverlay +shortLabel %smultiWig +longLabel %smultiWig +autoScale on +negativeValues on +alwaysZero on + +""" % (x, request.POST['short_label'], request.POST['long_label'])) + for datatype in usedlist: + if datatype != "segments": + trackdbtxt.write("""track %s_%s +bigDataUrl %s_%s.bigbed +shortLabel %s%s +longLabel %s%s +type bigBed +color 0,253,0 +alwaysZero on + +""" % (x, datatype, x, datatype, request.POST['short_label'], datatype, request.POST['long_label'], datatype)) + + trackdbtxt.write("""track %s +bigDataUrl %s.bigWig +shortLabel %sbigwig +longLabel %sbigwig +type bigWig +color 0,0,0 +parent %smultiWig +autoScale on +negativeValues on +graphTypeDefault points +alwaysZero on + +""" % (x, x, request.POST['short_label'], request.POST['long_label'], x)) + + trackdbtxt.write("""track %s_segments +bigDataUrl %s_segments.bigWig +shortLabel %ssegments +longLabel %ssegments +type bigWig +color 0,253,0 +parent %smultiWig +autoscale on +negativeValues on +alwaysZero on + +""" % (x, x, request.POST['short_label'], request.POST['long_label'], x)) + trackdbtxt.close() + os.chdir(olddir) + trackhub = db.Trackhub(request.POST['short_label']) + trackhubdata = trackhub.get() + trackhubdata['long_label'] = request.POST['long_label'] + trackhubdata['profiles'] = request.POST.getall('profile') + trackhub.put(trackhubdata) + usertrackhubs = db.UserTrackhubs(authenticated_userid(request)) + usertrackhubsdata = usertrackhubs.get() + usertrackhubsdata.append(request.POST['short_label']) + usertrackhubs.put(usertrackhubsdata) + return HTTPFound(location=request.host_url+'/trackhub_details/'+request.POST['short_label']+'/') + + +@view_config(route_name="trackhub_export", + request_method="GET", + renderer="templates/trackhub_export.pt") +def trackhub_export_show(request): + userid = authenticated_userid(request) + profile_names = db.UserProfiles(userid).get() + # show only a few of the most recently uploaded profiles. + profiles = table_profiles(profile_names, userid) + info = { + 'profile_count': len(profile_names), + 'profiles': profiles, + 'user': userid, + } + return info + + +@view_config(route_name="trackhub_details", + request_method="GET", + renderer="templates/trackhub_details.pt") +def trackhub_details(request): + userid = authenticated_userid(request) + md = request.matchdict + trackhub = db.Trackhub(md["short_name"]).get() + info = { + 'url': request.route_url('trackhub_file', short_name=md["short_name"], file="hub.txt"), + 'profiles': trackhub["profiles"], + 'short_name': md["short_name"], + 'long_name': trackhub["long_label"], + 'user': userid + } + return info + + +@view_config(route_name="trackhub_file", + request_method="GET") +def trackhub_file(request): + md = request.matchdict + p = '' + for level in md["file"]: + p = p + "/" + level + return FileResponse(os.getcwd() + '/trackhubs/' + md["short_name"] + p) + + +@view_config(route_name="trackhub_list", + request_method="GET", + renderer="templates/trackhub_list.pt") +def trackhub_list(request): + md = request.matchdict + userid = authenticated_userid(request) + trackhubs = db.UserTrackhubs(userid) + trackhubslist = trackhubs.get() + info = { + 'url': request.host_url, + 'trackhubs': trackhubslist, + 'user': userid + } + return info + + +def respond_bed_csv(table, fmt, hinfo, dicts, header=True): response = Response(content_type="text/plain") tup = (table, fmt) - header_tmp = EXPORT_HEADERS[tup] - header = header_tmp % hinfo + '\n' - response.write(header) + if header: + header_tmp = EXPORT_HEADERS[tup] + header = header_tmp % hinfo + '\n' + response.write(header) fmt = EXPORT_FORMATS[tup] + text = [] for d in dicts: line = fmt % d + "\n" - response.write(line) + text.append(line) + text.sort() + textout = ''.join(text) + response.write(textout) return response diff --git a/tests/test_Normal02.bedGraph.gz b/tests/test_Normal02.bedGraph.gz new file mode 100644 index 0000000..1b428bf Binary files /dev/null and b/tests/test_Normal02.bedGraph.gz differ diff --git a/tests/tests.py b/tests/tests.py index d1d8234..172f211 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -20,13 +20,14 @@ def setUp(self): self.driver = webdriver.Firefox() # enter the test_profile_path here self.test_profile_path = os.path.join(os.getcwd(),"test_profile.bedGraph.gz") + self.test_second_profile_path = os.path.join(os.getcwd(),"test_Normal02.bedGraph.gz") def test010_isSegAnnUp(self): """ Test#1 This test is for checking whether the page is loading or not """ - print "Test#1 isSegAnnUp ?" + print "Test#1 isSegAnnUp?" driver = self.driver driver.get("http://127.0.0.1:8080") assert "SegAnnDB" in driver.title @@ -89,7 +90,7 @@ def test040_annotate(self): This all happens on the test uploaded profile """ - print "Test #4 for testing annotation" + print "Test#4 for testing annotation" # we need to give some time for profile processing before we can make # annotations @@ -123,11 +124,118 @@ def test040_annotate(self): del_resp.close() - def test050_delete(self): + def test050_trackhub(self): + print "Test#5 Trackhub Creation Test" + driver = self.driver + driver.get("http://127.0.0.1:8080/") + self.login(driver) + + # make sure that we are logged in + wait = WebDriverWait(driver, 60) + assert wait.until( + EC.element_to_be_clickable((By.ID, "signout"))).is_displayed() + driver.get("http://127.0.0.1:8080/trackhub_init/") + shortname_field = driver.find_element_by_id("id_short_label") + shortname_field.send_keys("test1") + longname_field = driver.find_element_by_id("id_long_label") + longname_field.send_keys("test1long") + db_field = driver.find_element_by_id("id_db") + db_field.send_keys("hg19") + profile_box = driver.find_element_by_xpath("//input[@value='ES0004']") + profile_box.click() + submit_button = driver.find_element_by_id("submit_button") + submit_button.click() + sleep(10) + if ("Trackhub data for test1" in driver.page_source) and ("ES0004" in driver.page_source): + assert True + else: + assert False + + def test060_multi_profile_trackhub(self): + print "Test#6 Multi Profile Trackhub Creation Test" + driver = self.driver + driver.get("http://127.0.0.1:8080/") + self.login(driver) + + # make sure that we are logged in + wait = WebDriverWait(driver, 60) + assert wait.until( + EC.element_to_be_clickable((By.ID, "signout"))).is_displayed() + + # go to the upload page + driver.get("http://127.0.0.1:8080/upload") + + # in the upload file field, upload the file + upload_field = driver.find_element_by_id("id_file") + test_second_profile_path = self.test_second_profile_path + upload_field.send_keys(test_second_profile_path) + + # we need to use the submit() button + # because when we submit forms via click + # the whole thing has been found to freeze. + elem = driver.find_element_by_id("submit_button") + elem.submit() + + assert wait.until( + EC.presence_of_element_located((By.ID, "success"))) + + time.sleep(60) + + driver.get("http://127.0.0.1:8080/trackhub_init/") + shortname_field = driver.find_element_by_id("id_short_label") + shortname_field.send_keys("test2") + longname_field = driver.find_element_by_id("id_long_label") + longname_field.send_keys("test2long") + db_field = driver.find_element_by_id("id_db") + db_field.send_keys("hg19") + profile_box = driver.find_element_by_xpath("//input[@value='ES0004']") + profile_box.click() + profile_box2 = driver.find_element_by_xpath("//input[@value='Normal02']") + profile_box2.click() + submit_button = driver.find_element_by_id("submit_button") + submit_button.click() + sleep(10) + if ("Trackhub data for test2" in driver.page_source) and ("ES0004" in driver.page_source) \ + and ("Normal02" in driver.page_source): + assert True + else: + assert False + + def test070_no_header(self): + """ + This test checks if header non-creation is properly done. + It also performs no login to simulate UCSC access. + """ + print "Test#7 Non-Header export test" + bedfile = urllib2.urlopen('http://127.0.0.1:8080/trackhub/test1/hg19/ES0004_breaks.bed') + if ("track" in bedfile.read()): + assert False + else: + assert True + + def test080_header(self): + """ + This checks that file header creation occurs by default + """ + print "Test#8 Header export test" + driver = self.driver + driver.get("http://127.0.0.1:8080/") + self.login(driver) + # make sure that we are logged in + wait = WebDriverWait(driver, 60) + assert wait.until( + EC.element_to_be_clickable((By.ID, "signout"))).is_displayed() + driver.get("http://127.0.0.1:8080/export/seganntest2@gmail.com/ES0004/breaks/bed/") + if ("track" in driver.page_source): + assert True + else: + assert False + + def test090_delete(self): """ This test is for checking if we are able to delete the uploaded profile """ - print "Test#5 Profile Deleting test." + print "Test#9 Profile Deleting test." driver = self.driver driver.get("http://127.0.0.1:8080/") self.login(driver) @@ -156,21 +264,21 @@ def login(self, driver): Parameters: driver - reference to driver being used """ - #check if cookie file exists and is up to date - import os.path - if os.path.isfile('cookies.txt'): - fr = open('cookies.txt') - cookies = eval(fr.read()) - for cookie in cookies: - if cookie["domain"] == "localhost" or cookie["domain"] == "127.0.0.1": - driver.get("http://127.0.0.1:8080/") - cookie["domain"] = "127.0.0.1" - driver.add_cookie(cookie) - driver.get("http://127.0.0.1:8080/") - return - # no usable cookie/cookie.txt, therefore let's switch back to localhost to get a cookie first - driver.get("http://localhost:8080/") - # this is the tricky part + # check if cookie file exists and is up to date + import os.path + if os.path.isfile('cookies.txt'): + fr = open('cookies.txt') + cookies = eval(fr.read()) + for cookie in cookies: + if cookie["domain"] == "localhost" or cookie["domain"] == "127.0.0.1": + driver.get("http://127.0.0.1:8080/") + cookie["domain"] = "127.0.0.1" + driver.add_cookie(cookie) + driver.get("http://127.0.0.1:8080/") + return + # no usable cookie/cookie.txt, therefore let's switch back to localhost to get a cookie first + driver.get("http://localhost:8080/") + # this is the tricky part # We have to get the right handle for the correct popup login window main_window_handle = driver.current_window_handle @@ -207,19 +315,19 @@ def login(self, driver): # enter the email of test user email_field.send_keys("seganntest2@gmail.com") - sleep(10) + sleep(10) # click next driver.find_element_by_id('identifierNext').click() - sleep(10) + sleep(10) # enter password - wait.until(EC.presence_of_element_located((By.NAME, "password"))).send_keys('segann@test',Keys.RETURN) - wait = WebDriverWait(driver, 60) + wait.until(EC.presence_of_element_located((By.NAME, "password"))).send_keys('segann@test', Keys.RETURN) + wait = WebDriverWait(driver, 60) assert wait.until( EC.element_to_be_clickable((By.ID, "signout"))).is_displayed() # This check is important so we get the localhost cookie cookies = driver.get_cookies() - fw = open('cookies.txt','w') - fw.write(str(cookies)) - fw.close() + fw = open('cookies.txt', 'w') + fw.write(str(cookies)) + fw.close() def tearDown(self): self.driver.close()
+ ${p} + + ${t} +