allow arbitrary number of wcs iterations

basschute/Makefile

@@ -1,9 +1,12 @@
 #!/usr/bin/make
 
+# define escaped comma
+, := ,
+
 .PHONY: initproc badpix flats proc redo
 
 ifndef DATE
-	$(error DATE must be set!)
+$(error DATE must be set!)
 endif
 UTARGS := --night $(DATE)
 
@@ -36,8 +39,24 @@ else
 	PROCARGS := --nousepixflat --fixsaturation --nobiascorr --noweightmap
 endif
 
+ifdef WCSCNFG
+# check configuration files exist
+FNS := $(subst $(,), ,${WCSCNFG})
+MTCH := $(foreach f,$(FNS),$(wildcard $(f)))
+MISS := $(filter-out $(MTCH), $(FNS))
+N := $(words $(MISS))
+ifneq ($(N), 0)
+$(info No such configuration file(s)!)
+$(foreach f,$(MISS),$(info $(f)))
+$(error )
+endif
+WCSCNFG := --wcsconfig $(WCSCNFG)
+endif
+
+WCSARGS := $(WCSCNFG) 
+
 INITARGS := $(LOGARGS) $(DATAARGS) $(UTARGS) $(BANDARGS) \
-            $(MPARGS) $(VERBOSE) 
+            $(MPARGS) $(VERBOSE) $(WCSARGS)
 
 all_detrend: initproc badpix proc1 makeillum flats proc2
 
@@ -142,7 +161,7 @@ quickproc:
 
 # Obtain astrometric solutions
 wcs:
-	python basschute.py $(INITARGS) -s wcs --gaia $(XARGS)
+	python basschute.py $(INITARGS) -s wcs $(XARGS)
 
 
 # Generate object catalogs and PSF models with sextractor+psfex

bokpipe/bokastrom.py

@@ -10,16 +10,88 @@
 
 configDir = os.path.join(os.path.split(__file__)[0],'config')
 
+def read_cnfg(configName):
+	# read a scamp configuration file into a dictionary
+	cnfg={}
+	with open(configName) as f:
+		lines=f.readlines()
+		for line in lines:
+			# ignore empty lines
+			if not( line.strip() ):
+				continue
+			# ignore commented lines
+			if line.strip()[0]=='#':
+				continue
+			split = line.split(' ',1)
+			key=split[0]
+			# ignore comments
+			split = split[1].split('#',1)
+			val=split[0].strip()
+			cnfg[key]=val
+
+	return cnfg
+
 def put_wcs(imageFile,verbose=0):
 	configFile = os.path.join(configDir,'wcsput.missfits')
 	missfits_cmd = ['missfits','-c',configFile,imageFile]
 	if verbose > 1:
 		print ' '.join(missfits_cmd)
 	rv = subprocess.call(missfits_cmd)
 
-def scamp_solve(imageFile,catFile,refStarCatFile=None,
-                filt='g',savewcs=False,clobber=False,
-                check_plots=False,twopass=True,verbose=0,**kwargs):
+def scamp_solver(imageFile,catFile,verbose,**kwargs):
+	scamp_cmd_base = ['scamp',catFile,
+	                  '-c',os.path.join(configDir,'boksolve.scamp')]
+
+	def add_scamp_pars(scamp_pars):
+		scamp_cmd = copy(scamp_cmd_base)
+		for k,v in scamp_pars.items():
+			scamp_cmd.extend(['-'+k,str(v)])
+		return scamp_cmd
+
+	# copy in options passed from command-line (override existing)
+	scamp_pars = {}
+	for k,v in kwargs.items():
+		scamp_pars[k] = v
+	scamp_cmd = add_scamp_pars(scamp_pars)
+	if verbose >= 1:
+		outdev = None
+		if verbose >=2:
+			print ' '.join(scamp_cmd)
+	else:
+		outdev = open(os.devnull,'w')
+
+	rv = subprocess.call(scamp_cmd,stdout=outdev)
+
+def scamp_solve(imageFile,catFile,wcsCnfg,savewcs=False,clobber=False,verbose=0):
+
+	headf = catFile.replace('.fits','.head') 
+	tmpAhead = catFile.replace('.fits','.ahead')
+	wcsFile = imageFile.replace('.fits','.ahead')
+	if not clobber and os.path.exists(wcsFile):
+		if verbose > 0:
+			print wcsFile,' already exists, skipping'
+		return
+	try:
+		os.unlink(wcsFile)
+	except:
+		pass
+
+	for n, cnfg in enumerate(wcsCnfg):
+		scamp_pars = read_cnfg(cnfg)
+		if verbose >= 1:
+			print 'scamp_solve pass', '%i of %i for' % (n+1,len(wcsCnfg)), imageFile
+		scamp_solver(imageFile,catFile,verbose,**scamp_pars)
+		shutil.move(headf,tmpAhead)
+
+	shutil.move(tmpAhead,wcsFile)
+	if savewcs:
+		put_wcs(imageFile,verbose=verbose)
+	return
+
+def scamp_default_solve(imageFile,catFile,refStarCatFile=None,
+			filt='g',savewcs=False,clobber=False,
+			check_plots=False,twopass=True,verbose=0):
+
 	headf = catFile.replace('.fits','.head') 
 	wcsFile = imageFile.replace('.fits','.ahead')
 	if not clobber and os.path.exists(wcsFile):
@@ -30,14 +102,7 @@ def scamp_solve(imageFile,catFile,refStarCatFile=None,
 		refCatPath = os.path.dirname(refStarCatFile)
 		if len(refCatPath)==0:
 			refCatPath = '.'
-	#
-	scamp_cmd_base = ['scamp',catFile,
-	                  '-c',os.path.join(configDir,'boksolve.scamp')]
-	def add_scamp_pars(scamp_pars):
-		scamp_cmd = copy(scamp_cmd_base)
-		for k,v in scamp_pars.items():
-			scamp_cmd.extend(['-'+k,str(v)])
-		return scamp_cmd
+
 	try:
 		os.unlink(wcsFile)
 	except:
@@ -59,24 +124,17 @@ def add_scamp_pars(scamp_pars):
 		scamp_pars['ASTREF_CATALOG'] = 'FILE'
 	else:
 		scamp_pars['ASTREF_CATALOG'] = 'SDSS-R9'
+		print scamp_pars['ASTREF_CATALOG']
 		scamp_pars['ASTREF_BAND'] = filt
 		if refStarCatFile is not None:
 			scamp_pars['SAVE_REFCATALOG'] = 'Y'
 		# see below
 		#scamp_pars['ASTREFCAT_NAME'] = os.path.basename(refStarCatFile)
 		#scamp_pars['REFOUT_CATPATH'] = refCatPath
-	# copy in options passed from command-line (override existing)
-	for k,v in kwargs.items():
-		scamp_pars[k] = v
-	scamp_cmd = add_scamp_pars(scamp_pars)
 	if verbose >= 1:
 		print 'first pass scamp_solve for ',imageFile
-		outdev = None
-		if verbose >= 2:
-			print ' '.join(scamp_cmd)
-	else:
-		outdev = open(os.devnull,'w')
-	rv = subprocess.call(scamp_cmd,stdout=outdev)
+	scamp_solver(imageFile,catFile,verbose,**scamp_pars)
+
 	tmpAhead = catFile.replace('.fits','.ahead') 
 	shutil.move(headf,tmpAhead)
 	if refStarCatFile is not None and scamp_pars['ASTREF_CATALOG'] != 'FILE':
@@ -111,12 +169,11 @@ def add_scamp_pars(scamp_pars):
 	scamp_pars['MOSAIC_TYPE'] = 'FIX_FOCALPLANE'
 	if check_plots:
 		del scamp_pars['CHECKPLOT_TYPE']
-	scamp_cmd = add_scamp_pars(scamp_pars)
+
 	if verbose > 1:
-		print ' '.join(scamp_cmd)
-	elif verbose > 0:
 		print 'second pass scamp_solve for ',imageFile
-	rv = subprocess.call(scamp_cmd,stdout=outdev)
+	scamp_solver(imageFile,catFile,verbose,**scamp_pars)
+
 	shutil.move(headf,wcsFile)
 	os.unlink(tmpAhead)
 	#

bokpipe/bokpl.py

@@ -411,8 +411,6 @@ def make_supersky_flats(dataMap,byUtd=False,interpFill=True,**kwargs):
 	for filt,utd in filtAndUtd:
 		files,frames = dataMap.getFiles(imType='object',filt=filt,utd=utd,
 		                                with_frames=True)
-		if frames is None:
-			continue
 		_outfn = outfn = dataMap.storeCalibrator('skyflat',frames)
 		if interpFill:
 			_outfn = _outfn.replace('.fits','_raw.fits')
@@ -512,8 +510,9 @@ def process_all2(dataMap,skyArgs,noillumcorr=False,noskyflatcorr=False,
 	skySub.add_mask(dataMap.getCalMap('badpix4'))
 	skySub.process_files(files)
 
-def _wcs_worker(dataMap,inputType,savewcs,keepwcscat,
-                clobber,verbose,inp):
+def _wcs_worker(dataMap,inputType,wcsCnfg,savewcs,keepwcscat,
+	        clobber,verbose,inp):
+
 	try:
 		imFile,fieldName = inp
 		imageFile = dataMap(inputType)(imFile)
@@ -526,11 +525,15 @@ def _wcs_worker(dataMap,inputType,savewcs,keepwcscat,
 		# kwargs sent to the following are added sextractor/scamp parameters
 		#  (e.g., {'VERBOSE':'FULL'}), so remap the pipeline kwargs 
 		bokphot.sextract(imageFile,catFile,
-		                 clobber=clobber,verbose=verbose)
-		bokastrom.scamp_solve(imageFile,catFile,
-		                      dataMap.getScampRefCat(fieldName),
-		                      filt='r',savewcs=savewcs,
-		                      clobber=clobber,verbose=verbose)
+				 clobber=clobber,verbose=verbose)
+		if wcsCnfg is None:
+			bokastrom.scamp_default_solve(imageFile,catFile,
+						      dataMap.getScampRefCat(fieldName),
+						      filt='r',savewcs=savewcs,twopass=True,
+						      clobber=clobber,verbose=verbose)
+		else:
+			bokastrom.scamp_solve(imageFile,catFile,wcsCnfg,savewcs=savewcs,
+					      clobber=clobber,verbose=verbose)
 		if not keepwcscat:
 			os.unlink(catFile)
 	except:
@@ -539,7 +542,8 @@ def _wcs_worker(dataMap,inputType,savewcs,keepwcscat,
 def set_wcs(dataMap,inputType='sky',savewcs=False,keepwcscat=True,**kwargs):
 	procmap = kwargs.pop('procmap',map)
 	filesAndFields = dataMap.getFiles(imType='object',with_objnames=True)
-	p_wcs_worker = partial(_wcs_worker,dataMap,inputType,savewcs,keepwcscat,
+	p_wcs_worker = partial(_wcs_worker,dataMap,inputType,kwargs.get('wcsCnfg', None),
+			       savewcs,keepwcscat,
 	                       kwargs.get('clobber',False),
 	                       kwargs.get('verbose',0))
 	status = procmap(p_wcs_worker,zip(*filesAndFields))
@@ -576,6 +580,7 @@ def bokpipe(dataMap,**kwargs):
 	processes = kwargs.get('processes',1)
 	procmap = kwargs.get('procmap')
 	maxmem = kwargs.get('maxmem',5)
+	wcsCnfg = kwargs.get('wcsCnfg',None)
 	chunkSize = 10
 	if processes > 1:
 		pool = multiprocessing.Pool(processes)
@@ -585,8 +590,8 @@ def bokpipe(dataMap,**kwargs):
 		procmap = pool.map
 	else:
 		procmap = map
-	pipekwargs = {'clobber':redo,'verbose':verbose,'debug':debug,
-	              'processes':processes,'procmap':procmap,'maxmem':maxmem}
+	pipekwargs = {'clobber':redo,'verbose':verbose,'debug':debug,'processes':processes,
+	              'procmap':procmap,'maxmem':maxmem,'wcsCnfg':wcsCnfg}
 	# fixpix is sticking nan's into the images in unmasked pixels (???)
 	fixpix = False #True
 	writeccdims = kwargs.get('calccdims',False)
@@ -884,6 +889,8 @@ def init_pipeline_args(parser):
 	                help='write wcs to headers')
 	parser.add_argument('--wcscheck',action='store_true',
 	                help='make astrometry diagnostic files')
+	parser.add_argument('--wcsconfig',type=str,default=None,
+	                help='space separated list of optional wcs config files')
 	parser.add_argument('--maxflatcounts',type=int,
 	                help='maximum counts (ADU) to accept for a flat')
 	parser.add_argument('--cleancals',action='store_true',
@@ -910,11 +917,15 @@ def run_pipe(dataMap,args,**_kwargs):
 		steps = ['oscan','proc1','proc2','wcs','cat']
 	else:
 		steps = args.steps.split(',')
+	if args.wcsconfig is not None:
+		args.wcsconfig = args.wcsconfig.split(',')
+
 	verbose = 0 if args.verbose is None else args.verbose
 	# convert command-line arguments into dictionary
 	opts = vars(args)
 	kwargs = { k : opts[k] for k in opts if opts[k] != None }
 	kwargs['steps'] = steps
+	kwargs['wcsCnfg'] = args.wcsconfig
 	for k,v in _kwargs.items():
 		kwargs[k] = v
 	# run pipeline processes
@@ -945,3 +956,4 @@ def run_pipe(dataMap,args,**_kwargs):
 	else:
 		bokpipe(dataMap,**kwargs)
 
+

bokpipe/config/boksolve.scamp

@@ -95,7 +95,7 @@ FWHM_THRESHOLDS        1.5,10.0       # FWHM thresholds (in pixels) for sources
 AHEADER_SUFFIX         .ahead          # Filename extension for additional
                                        # INPUT headers
 HEADER_SUFFIX          .head           # Filename extension for OUTPUT headers
-VERBOSE_TYPE           QUIET           # QUIET, NORMAL, LOG or FULL
+VERBOSE_TYPE           NORMAL          # QUIET, NORMAL, LOG or FULL
 WRITE_XML              N               # Write XML file (Y/N)?
 XML_NAME               scamp.xml       # Filename for XML output
 NTHREADS               0               # Number of simultaneous threads for