diff --git a/README.md b/README.md index da75d074..8baad97e 100644 --- a/README.md +++ b/README.md @@ -253,6 +253,10 @@ You can access the database from your host machine, e.g. with sequelpro by setti ``` conda install MySQL-python ``` + 8. Install mysql-connector-python-rf + ``` + pip install mysql-connector-python-rf + ``` 5. Install the pipeline 1. Install git if you do not already have it (https://ucdavis.github.io/DS4S/#setup) 2. Download the pipeline: diff --git a/trunk/bin/automated_subtraction.py b/trunk/bin/automated_subtraction.py new file mode 100644 index 00000000..2c6ced09 --- /dev/null +++ b/trunk/bin/automated_subtraction.py @@ -0,0 +1,322 @@ +#!/usr/bin/env python + + +""" +==================================== + +* **Filename**: run_subtraction.py +* **Author**: Joseph Farah +* **Description**: Automates subtraction on a schedule. + +==================================== + +**Notes** +* Observation created on target with ID at MJD. Cronjob runs. +* Two things to check: +* - does template exist for object in filter? +* - has observation in filter been conducted after MJD? +* If yes: get filter, mjd, filetag. run subtraction with that info + + +Steps: +- + + +""" + +#------------- IMPORTS -------------# +import lsc +import datetime +from subprocess import call +from mysql.connector import connect + + +#------------- SETTINGS -------------# +template_order_labels = ['PS1', 'LCO', 'DECam', 'Skymapper']#, 'SDSS'] +template_order = [] +for t in template_order_labels: + for f in ['g', 'r', 'i']: + template_order.append(t+'_'+f) + +# INGESTION_MODE = True will download a new set of templates +# INGESTION_MODE = False will try to use Giacomo's templates +INGESTION_MODE = True +DEFAULT = 'PS1' + + +#------------- CLASSES -------------# +class PotentialTarget(object): + ''' + Object to organize data from table. Will be used to check if template + exists for unsubtracted observation. Bundles information and passes to + subtraction. + ''' + + def __init__(self, unpacked_attr): + self.targetid = unpacked_attr[0] + self.objname = unpacked_attr[1] + self.filepath = unpacked_attr[2] + self.filename = unpacked_attr[3] + self.filetype = unpacked_attr[4] + self.dayobs = unpacked_attr[5] + self.filter = unpacked_attr[6] + self.PS1_g = unpacked_attr[7] + self.PS1_r = unpacked_attr[8] + self.PS1_i = unpacked_attr[9] + self.DECam_g = unpacked_attr[10] + self.DECam_r = unpacked_attr[11] + self.DECam_i = unpacked_attr[12] + self.LCO_g = unpacked_attr[13] + self.LCO_r = unpacked_attr[14] + self.LCO_i = unpacked_attr[15] + self.Skymapper_g = unpacked_attr[16] + self.Skymapper_r = unpacked_attr[17] + self.Skymapper_i = unpacked_attr[18] + self.instrument = unpacked_attr[19] + self.TEMP_SRC = None + self.TEMP_FILT = None + self.FOUND_TEMP = False + + + + +#------------- FUNCTIONS -------------# +def send_query(query, conn): + cursor = conn.cursor() + cursor.execute(query) + result = cursor.fetchall() + return result + + +def get_target_list(): + """ + Checks all recent observations for those relevant to GW events + and organizes their information in preparation for subtraction. + Designed to run on schedule. + + **Args**: + + * none (none): none + + **Returns**: + + * target_list (list): list of targets to perform subtraction on + + """ + + + ## minimum to date to examine is 3 days ago ## + # d = datetime.datetime(2023, 9, 01) - datetime.timedelta(3) ## DEBUGGING + d = datetime.date.today() - datetime.timedelta(3) + + ## convert to pipeline-friendly fmt (YYYYMMDD) ## + d = d.strftime("%Y%m%d") + + ''' + grab ID, name, obs fpath/fname, filetype, + obsdate, and filter from obs in both + photlco AND o5 as well as any template info + for associated tampltes + ''' + query = "SELECT p.targetid, p.objname, p.filepath, p.filename, p.filetype, p.dayobs, p.filter, " + query += "o.PS1_g, o.PS1_r, o.PS1_i, " + query += "o.DECam_g, o.DECam_r, o.DECam_i, " + query += "o.LCO_g, o.LCO_r, o.LCO_i, " + query += "o.Skymapper_g, o.Skymapper_r, o.Skymapper_i, p.instrument " + query += "FROM photlco p INNER JOIN o4_galaxies o ON p.targetid = o.targetid " ## ensures that only targets in both are considered + query += "WHERE p.dayobs >= {0};".format(d) + + ## connect to database ## + conn = connect( + user = 'supernova', + password = 'supernova', + host = 'supernova.science.lco.global', + database = 'supernova' + ) + + result = send_query(query, conn) + + ## + potential_targets = [PotentialTarget(r) for r in result] + + target_list = [] + for target in potential_targets: + FOUND_TEMP = False + for temp in template_order: + if getattr(target, temp) is None or getattr(target, temp) == '--': + pass + else: + target.TEMP_SRC = temp.split('_')[0] + target.TEMP_FILT = temp.split('_')[1] + if target.TEMP_FILT == target.filter or target.TEMP_FILT+'p' == target.filter: + print("Template found with properties: {0},{1}".format(target.TEMP_SRC, target.TEMP_FILT)) + target.FOUND_TEMP = True + target_list.append(target) + FOUND_TEMP = True + break + if INGESTION_MODE == True and FOUND_TEMP == False: + target.TEMP_SRC = 'PS1' + target.TEMP_FILT = target.filter.replace('p', '') + target_list.append(target) + + + print("List of targets: ", target_list) + return target_list + + + + + +def run_subtraction(targets, ingestion_mode=False): + """ + Runs subtraction on narrowed list of targets provided by + get_target_list(). + + Info needed for subtraction: + - instrument + - target id + - filter + - date + - template date + + **Args**: + + * targets (list): list of targets to perform subtraction on + * ingestion_mode (bool): whether to perform on-the-fly template download + + **Returns**: + + * none (none): none + + """ + + + def subtract_LCO(target): + print("Perfoming LCO subtraction on {0}.".format(target.objname)) + print("No commands available.") + + + def subtract_DECam(target): + print("Perfoming DEcam subtraction on {0}.".format(target.objname)) + print("No commands available.") + + def subtract_PS1(target, ingestion_mode=False): + print("Perfoming PS1 subtraction on {0}.".format(target.objname)) + objname = target.objname + date = target.dayobs + target_id = target.targetid + conn = connect( + user = 'supernova', + password = 'supernova', + host = 'supernova.science.lco.global', + database = 'supernova' + ) + + if ingestion_mode==True: + tempdate = '20100101-20160101' + filt = target.filter.replace('p', '') + instrument = filter(str.isalpha, target.instrument.encode('utf-8')) + command = "lscloop.py -n {0} -e {1} && lscloop.py -n {0} -e {1} -s ingestps1 -f {4} && lscloop.py -n {0} -e {3} --filetype 4 -s psf --fwhm 5 --use-sextractor && lscloop.py -n {0} -e {3} --filetype 4 -s cosmic && lscloop.py -n {0} -e {1} -s cosmic && lscloop.py -n {0} -e {1} --normalize i --convolve t -T {5} --tempdate {3} --temptel PS1 --fixpix --difftype 0 -s diff -F".format(objname, date, target_id, tempdate, filt, instrument) + print(command) + # input() + + else: + filename = getattr(target, target.TEMP_SRC+'_'+target.TEMP_FILT).split('/')[-1] + query = "SELECT dayobs FROM photlco WHERE targetid={1} AND filename='{0}'".format(filename, target_id) + print(query) + tempdate = send_query(query, conn)[0][0].encode('utf-8') + filt = target.filter.replace('p', '') + instrument = filter(str.isalpha, target.instrument.encode('utf-8')) + command = "lscloop.py -n {0} -e {1} && lscloop.py -n {0} -e {3} --filetype 4 -s psf --fwhm 5 --use-sextractor && lscloop.py -n {0} -e {3} --filetype 4 -s cosmic && lscloop.py -n {0} -e {1} -s cosmic && lscloop.py -n {0} -e {1} --normalize i --convolve t -T {5} --tempdate {3} --temptel PS1 --fixpix --difftype 0 -s diff -F".format(objname, date, target_id, tempdate, filt, instrument) + + # command = "lscloop.py -n {0} -e {1} && lscloop.py -n {0} -e {1} -d {2} -s ingestps1 -f {4} && lscloop.py -n {0} -e {3} --filetype 4 -s psf --fwhm 5 --use-sextractor && lscloop.py -n {0} -e {3} --filetype 4 -s cosmic && lscloop.py -n {0} -e {1} -s cosmic && lscloop.py -n {0} -e {1} --normalize i --convolve t -T {5} --tempdate {3} --temptel PS1 --fixpix --difftype 0 -s diff --no_iraf".format(objname, date, tel_id, tempdate, filt, instrument) + + + call(command, shell=True) + + + + def subtract_Skymapper(target): + print("Perfoming Skymapper subtraction on {0}.".format(target.objname)) + print("No commands available.") + + + def subtract_SDSS(target): + print("Perfoming SDSS subtraction on {0}.".format(target.objname)) + print("No commands available.") + + objname = target.objname + date = target.dayobs + target_id = target.targetid + conn = connect( + user = 'supernova', + password = 'supernova', + host = 'supernova.science.lco.global', + database = 'supernova' + ) + filename = getattr(target, target.TEMP_SRC+'_'+target.TEMP_FILT).split('/')[-1] + if ingestion_mode==True: + tempdate = '20060101-20070101' + filt = target.filter.replace('p', '') + instrument = filter(str.isalpha, target.instrument.encode('utf-8')) + command = "lscloop.py -n {0} -e {1} && lscloop.py -n {0} -e {1} -s ingestsdss -f {4} && lscloop.py -n {0} -e {3} --filetype 4 -s psf --fwhm 5 --use-sextractor && lscloop.py -n {0} -e {3} --filetype 4 -s cosmic && lscloop.py -n {0} -e {1} -s cosmic && lscloop.py -n {0} -e {1} --normalize i --convolve t -T {5} --tempdate {3} --temptel PS1 --fixpix --difftype 0 -s diff -F".format(objname, date, target_id, tempdate, filt, instrument) + print(command) + # input() + + + # command = "lscloop.py -n {0} -e {1} && lscloop.py -n {0} -e {1} -d {2} -s ingestps1 -f {4} && lscloop.py -n {0} -e {3} --filetype 4 -s psf --fwhm 5 --use-sextractor && lscloop.py -n {0} -e {3} --filetype 4 -s cosmic && lscloop.py -n {0} -e {1} -s cosmic && lscloop.py -n {0} -e {1} --normalize i --convolve t -T {5} --tempdate {3} --temptel PS1 --fixpix --difftype 0 -s diff --no_iraf".format(objname, date, tel_id, tempdate, filt, instrument) + + + call(command, shell=True) + + + + subfunc_switchboard = { + 'LCO':subtract_LCO, + 'DECam':subtract_DECam, + 'PS1':subtract_PS1, + 'Skymapper':subtract_Skymapper, + 'SDSS':subtract_SDSS, + } + + print("Running subtraction on: ", targets) + logfile = open("/supernova/github/lcogtsnpipe/LOG.txt", "a") + ## iterate over targets ## + for target in targets: + + print("Loading target {0} with target ID {1} and object name {2}".format(target, target.targetid, target.objname)) + logfile.write("Performing subtraction on obj " + str(target.objname) + " with filename " + str(target.filename) + " at time " + str(datetime.datetime.now()) + "\n") + + + ## ingest template with filetype=4 + if target.FOUND_TEMP == True: + print(target.TEMP_SRC) + print(target.TEMP_FILT) + lsc.mysqldef.ingestredu([getattr(target, target.TEMP_SRC+'_'+target.TEMP_FILT)], 'no', filetype=4) + + ## associate with correct subtraction function ## + subtraction_func = subfunc_switchboard[target.TEMP_SRC] + + ## perform subtraction ## + subtraction_func(target, ingestion_mode=ingestion_mode) + + + logfile.write("Done\n") + logfile.close() + + + + + + +def main(): + global INGESTION_MODE + + target_list = get_target_list() + run_subtraction(target_list, ingestion_mode=INGESTION_MODE) + + +#------------- SWITCHBOARD -------------# +if __name__ == '__main__': + main() + diff --git a/trunk/setup.py b/trunk/setup.py index f450a771..c2d4f2fe 100644 --- a/trunk/setup.py +++ b/trunk/setup.py @@ -12,12 +12,13 @@ license='LICENSE.txt', description='lsc is a package to reduce PHTOMETRIC SN data', long_description=open('README.txt').read(), - requires=['numpy','astropy','matplotlib','MySQLdb', 'pyraf', 'reproject'], + # we also require mysql-connector-python-rf + requires=['numpy','astropy','matplotlib','MySQLdb', 'pyraf'], packages=['lsc'], package_dir={'': 'src'}, package_data={'lsc': ["standard/astrometry/*cat","standard/*txt","standard/stdlist/*txt", "standard/cat/*dat","standard/cat/*cat", "standard/cat/sloan/*cat","standard/cat/landolt/*cat","standard/cat/apass/*cat", "standard/cat/sloanprime/*cat","standard/cat/sloannatural/*cat","standard/cat/landoltnatural/*cat", - "standard/sex/*", 'configure']} + "standard/sex/*"]} )