diff --git a/supernova.sql b/supernova.sql index 813ae073..36451697 100644 --- a/supernova.sql +++ b/supernova.sql @@ -494,6 +494,35 @@ CREATE TABLE `notes` ( +# Dump of table o4_galaxies +# ------------------------------------------------------------ + +DROP TABLE IF EXISTS `o4_galaxies`; + +CREATE TABLE `o4_galaxies` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT, + `targetid` int(11) NOT NULL, + `event_id` varchar(255) NOT NULL, + `ra0` double NOT NULL, + `dec0` double NOT NULL, + `SDSS_g` varchar(255) DEFAULT NULL, + `SDSS_r` varchar(255) DEFAULT NULL, + `SDSS_i` varchar(255) DEFAULT NULL, + `PS1_g` varchar(255) DEFAULT NULL, + `PS1_r` varchar(255) DEFAULT NULL, + `PS1_i` varchar(255) DEFAULT NULL, + `DECam_g` varchar(255) DEFAULT NULL, + `DECam_r` varchar(255) DEFAULT NULL, + `DECam_i` varchar(255) DEFAULT NULL, + `LCO_g` varchar(255) DEFAULT NULL, + `LCO_r` varchar(255) DEFAULT NULL, + `LCO_i` varchar(255) DEFAULT NULL, + `ingested` tinyint(1) NOT NULL DEFAULT 0, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + + + # Dump of table obslog # ------------------------------------------------------------ diff --git a/trunk/bin/ingesttemplates.py b/trunk/bin/ingesttemplates.py new file mode 100644 index 00000000..97b199de --- /dev/null +++ b/trunk/bin/ingesttemplates.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python + +from runlsc import run_cmd +from argparse import ArgumentParser + + +if __name__ == "__main__": + basecmd = 'lscloop.py' + logfile = open('ingesttemplates.log', 'w+') + + parser = ArgumentParser() + parser.add_argument("-f", "--filter", default='') + parser.add_argument("--targetid", default=None, type=int) + parser.add_argument("-t", "--timeout", type=float, default=86400.) + args = parser.parse_args() + + if args.filter: + basecmd += ' -f ' + args.filter + if args.targetid is not None: + basecmd += ' --targetid ' + str(args.targetid) + + run_cmd(basecmd + ' -s ingestsloan', logfile, args.timeout) + #run_cmd(basecmd + ' -s ingestps1', logfile, args.timeout) + #run_cmd(basecmd + ' -s ingestdecam', logfile, args.timeout) + + ### Generate PSFs and run cosmic ray removal + + run_cmd(basecmd + ' -e 19990101-20191231 -s psf --filetype 4 --use-sextractor --fwhm 5 --max_apercorr 2', logfile, args.timeout) + run_cmd(basecmd + ' -e 19990101-20191231 -s cosmic --filetype 4', logfile, args.timeout) + diff --git a/trunk/bin/lscingestsloan.py b/trunk/bin/lscingestsloan.py index 2ecc7173..480cedf9 100755 --- a/trunk/bin/lscingestsloan.py +++ b/trunk/bin/lscingestsloan.py @@ -10,19 +10,28 @@ from LCOGTingest import db_ingest parser = argparse.ArgumentParser(description='Download and ingest SDSS or PS1 templates') - parser.add_argument('images', nargs='+', help='LCOGT images to use for field of view and filter') + parser.add_argument('images', nargs='*', help='LCOGT images to use for field of view and filter') parser.add_argument('--type', default='sloan', help='Survey to download from', choices=['sloan', 'ps1']) parser.add_argument('--ps1frames', default='') parser.add_argument('--show', action='store_true', help='show the assembled, swarped SDSS template in DS9') parser.add_argument('-F', '--force', action='store_true', help='redownload SDSS images if already present in CWD') + parser.add_argument('--targetid', default=None, type=int) + parser.add_argument('-T', '--tel', default='') + parser.add_argument('-f', '--filter', default='') args = parser.parse_args() imglist = args.images imgtype = args.type ps1frames = args.ps1frames if imgtype =='sloan': - for img in imglist: - image0, varimg = lsc.sloanimage(img,'sloan','', args.show, args.force) + if not imglist and args.targetid and args.tel and args.filter: + image0, varimg = lsc.sloanimage('', 'sloan', '', args.show, args.force, args.targetid, args.tel, args.filter) + elif not imglist: + image0='' + print 'no data selected' + else: + for img in imglist: + image0, varimg = lsc.sloanimage(img,'sloan','', args.show, args.force) elif imgtype =='ps1': print "WARNING: PS1 ingestion works at the moment with single object and filter\n " print "please, do not provide multiple objects and filter in the same query" diff --git a/trunk/bin/lscloop.py b/trunk/bin/lscloop.py index 6281a765..9b7c131a 100644 --- a/trunk/bin/lscloop.py +++ b/trunk/bin/lscloop.py @@ -260,9 +260,7 @@ def run_absphot(img): fake_temptel = 'sbig' elif args.telescope == 'fs': fake_temptel = 'spectral' - elif args.telescope == 'fl': - fake_temptel = 'sinistro' - elif args.telescope == 'fa': + elif args.telescope in ['fl', 'fa', '1m0']: fake_temptel = 'sinistro' elif args.telescope == 'ep': fake_temptel = 'muscat' @@ -330,5 +328,9 @@ def run_absphot(img): str(ll0['psfmag'][i]), str(ll0['zcat'][i]), str(ll0['mag'][i]), str(ll0['abscat'][i])) print '\n### total number = ' + str(len(ll0['filename'])) lsc.myloopdef.run_merge(listfile[ll['dayobs'] == epo], args.force) # merge images using lacos and swarp + elif args.stage == 'ingestsloan': + lsc.myloopdef.run_ingestsloan('', 'sloan', show=args.show, force=args.force, targetid=args.targetid, tel='fa', filt=args.filter) + elif args.stage == 'ingestps1': + lsc.myloopdef.run_ingestsloan('', 'ps1', args.ps1frames, show=args.show, force=args.force, targetid=args.targetid, tel='fa', filt=args.filter) else: print '\n### no data selected' diff --git a/trunk/bin/rungwgalaxydiff.py b/trunk/bin/rungwgalaxydiff.py new file mode 100644 index 00000000..1ff05525 --- /dev/null +++ b/trunk/bin/rungwgalaxydiff.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python + +from runlsc import run_cmd +from lsc.mysqldef import query +from lsc import conn + +if __name__ == "__main__": + logfile = open('gwgalaxydiff.log', 'w+') + galaxies_to_subtract = query(['select targetid, SDSS_g, SDSS_r, SDSS_i, PS1_g, PS1_r, PS1_i, DECam_g, DECam_r, DECam_i, LCO_g, LCO_r, LCO_i from o4_galaxies'], conn) + if galaxies_to_subtract: + for galaxy in galaxies_to_subtract: + targetid = int(galaxy['targetid']) + images_to_subtract = query(['select distinct filter from photlco where targetid={} and filetype=1'.format(targetid)], conn) + for filt_dict in images_to_subtract: + filt = filt_dict['filter'][0] + run_cmd('lscloop.py --targetid {} -f {} -e 20200101-20300101 --filetype 1 -s cosmic'.format(targetid, filt), logfile, 86400.0) + if galaxy['SDSS_'+filt]: + run_cmd('lscloop.py --targetid {} -f {} -e 20200101-20300101 -s diff --temptel SDSS --tempdate 19990101-20191231 --normalize t -T 1m0'.format(targetid, filt), logfile, 86400.0) + elif galaxy['PS1_'+filt]: + run_cmd('lscloop.py --targetid {} -f {} -e 20200101-20300101 -s diff --temptel PS1 --tempdate 19990101-20191231 --normalize t -T 1m0'.format(targetid, filt), logfile, 86400.0) + elif galaxy['DECam_'+filt]: + run_cmd('lscloop.py --targetid {} -f {} -e 20200101-20300101 -s diff --temptel decam --tempdate 19990101-20191231 --normalize t -T 1m0'.format(targetid, filt), logfile, 86400.0) + elif galaxy['LCO_'+filt]: + run_cmd('lscloop.py --targetid {} -f {} -e 20200101-20300101 -s diff --tempdate 19990101-20191231 --normalize t -T 1m0'.format(targetid, filt), logfile, 86400.0) + + + diff --git a/trunk/bin/runo4templateingest.py b/trunk/bin/runo4templateingest.py new file mode 100644 index 00000000..18453385 --- /dev/null +++ b/trunk/bin/runo4templateingest.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python + +from lsc.mysqldef import query +from lsc import conn +from runlsc import run_cmd + + +if __name__ == "__main__": + ### Get list of galaxies with no templates ingested + logfile = open('runo4templateingest.log', 'w+') + undownloaded_galaxies = query(['select id, targetid from o4_galaxies where ingested = 0'], conn) + if undownloaded_galaxies: + for galaxy in undownloaded_galaxies: + ### Run the template ingestion + for filt in ['g', 'r', 'i']: + run_cmd('ingesttemplates.py -f {} --targetid {}'.format(filt, galaxy['targetid']), logfile) + + ### Look for templates for this filter in each survey + template_query = query(["select filepath, filename from photlco where targetid={} and filetype=4 and filter='{}p'".format(galaxy['targetid'], filt)], conn) + if template_query: + ### Update the filepath and ingested columns in the o4_galaxies table + for temp in template_query: + if 'SDSS' in temp['filename']: + update_row = query(["update o4_galaxies set SDSS_{} = '{}', ingested = 1 where id = {}".format(filt, temp['filepath']+temp['filename'], int(galaxy['id']))], conn) + elif 'PS' in temp['filename']: + update_row = query(["update o4_galaxies set PS1_{} = '{}', ingested = 1 where id = {}".format(filt, temp['filepath']+temp['filename'], int(galaxy['id']))], conn) + elif 'dec' in temp['filename'].lower(): + update_row = query(["update o4_galaxies set DECam_{} = '{}', ingested = 1 where id = {}".format(filt, temp['filepath']+temp['filename'], int(galaxy['id']))], conn) + else: + update_row = query(["update o4_galaxies set LCO_{} = '{}', ingested = 1 where id = {}".format(filt, temp['filepath']+temp['filename'], int(galaxy['id']))], conn) + + diff --git a/trunk/src/lsc/externaldata.py b/trunk/src/lsc/externaldata.py index 820f10ea..e352467e 100755 --- a/trunk/src/lsc/externaldata.py +++ b/trunk/src/lsc/externaldata.py @@ -482,17 +482,27 @@ def northupeastleft(filename='', data=None, header=None): return data, header ################################################################################## -def sloanimage(img,survey='sloan',frames=[], show=False, force=False): +def sloanimage(img,survey='sloan',frames=[], show=False, force=False, targetid=None, tel='', filt=''): import sys - from lsc import readhdr, readkey3,deg2HMS,display_image - if show: - display_image(img,1,True,'','') - hdr = readhdr(img) - _ra = readkey3(hdr,'RA') - _dec = readkey3(hdr,'DEC') - _object = readkey3(hdr,'object') - _instrume = readkey3(hdr,'instrume') - _filter = readkey3(hdr,'filter') + from lsc import readhdr, readkey3,deg2HMS,display_image, conn + from lsc.mysqldef import query + if img: + if show: + display_image(img,1,True,'','') + hdr = readhdr(img) + _ra = readkey3(hdr,'RA') + _dec = readkey3(hdr,'DEC') + _object = readkey3(hdr,'object') + _instrume = readkey3(hdr,'instrume') + _filter = readkey3(hdr,'filter') + elif targetid: + targetquery = query(['select ra0, dec0 from targets where id={}'.format(targetid)], conn) + namequery = query(['select name from targetnames where targetid={}'.format(targetid)], conn) + _ra = float(targetquery[0]['ra0']) + _dec = float(targetquery[0]['dec0']) + _object = namequery[0]['name'] + _instrume = tel + _filter = filt _radius = 1100 filt={'up':'u','gp':'g','rp':'r','ip':'i','zs':'z'} diff --git a/trunk/src/lsc/myloopdef.py b/trunk/src/lsc/myloopdef.py index 69d26672..d0682e24 100755 --- a/trunk/src/lsc/myloopdef.py +++ b/trunk/src/lsc/myloopdef.py @@ -1964,8 +1964,10 @@ def run_merge(imglist, _redu=False): ######################################################################################## -def run_ingestsloan(imglist,imgtype = 'sloan', ps1frames='', show=False, force=False): - command = 'lscingestsloan.py ' + ' '.join(imglist) +def run_ingestsloan(imglist,imgtype = 'sloan', ps1frames='', show=False, force=False, targetid=None, tel='fa', filt=''): + command = 'lscingestsloan.py' + if imglist: + command += ' '.join(imglist) if imgtype != 'sloan': command += ' --type ' + imgtype if ps1frames: @@ -1974,6 +1976,8 @@ def run_ingestsloan(imglist,imgtype = 'sloan', ps1frames='', show=False, force=F command += ' --show' if force: command += ' -F' + if targetid and filt: + command += ' --targetid ' + str(targetid) + ' --tel ' + tel + ' --filter ' + filt[0] print command os.system(command)