Commit 60a29b8a authored by Monica Rainer's avatar Monica Rainer
Browse files

Change structure, add gofio.py

parent 8a9821e2
"""
GOFIO DRS v1.1.0
Written by Monica Rainer
Usage:
gofioDRS.py -h
gofioDRS.py [-g=<gofio_config>]
gofioDRS.py <date> [--dark --flat --une --fp --only_calib --use_flat=<flag>]
gofioDRS.py <date> <calib_date> [--dark --flat --une --fp --only_calib --use_flat=<flag>]
Options:
-h,--help : show this screen
-g,--cfg : path of the configuration file (optional)
date : date to be reduced, no configuration file possible (always first input)
calib_date : calibration date to be used (always second input)
--dark : darks are reduced
--flat : flats are reduced
--une : U-Ne lamps are reduced
--fp : FP lamps are reduced
--only_calib : only calibration are reduced, no science images
--use_flat=<flag> : flat-field is removed, options are global/order/nor/none [Default: order]
"""
import os, glob, sys, time
from docopt import docopt
from drslib.config import CONFIG, read_usr_config
from drslib.logger import DrsLogger
#from logger.logger import DrsLogger
from drslib import db, rawfiles, darkframes, flatframes, wlframes, nodding, stare, varie
# use watchdog to monitor file creation
from watchdog.observers import Observer
from watchdog.events import PatternMatchingEventHandler
# read docopt_arg and act accordingly
docopt_args = docopt(__doc__)
#print docopt_args
if docopt_args['--cfg']:
try:
USRCONFIG = read_usr_config(docopt_args['-g'])
except:
try:
USRCONFIG = read_usr_config(docopt_args['--cfg'])
except:
USRCONFIG = {}
USRCONFIG['OFFLINE'] = False
if USRCONFIG['OFFLINE']:
for key in USRCONFIG.keys():
CONFIG[key] = USRCONFIG[key]
CONFIG['RAW_DIR'] = os.path.join(CONFIG['BASE_RAW'],CONFIG['DATE'])
CONFIG['OFFLINE_DIR'] = os.path.join(CONFIG['BASE_RED_DIR'],CONFIG['DATE'])
CONFIG['RED_DIR'] = os.path.join(CONFIG['OFFLINE_DIR'],'offline')
CONFIG['RED_CALIB'] = os.path.join(CONFIG['RED_DIR'],'CALIB')
CONFIG['RED_STR'] = os.path.join(CONFIG['RED_DIR'],'STR')
CONFIG['LOG_FILE'] = os.path.join(CONFIG['RED_DIR'], 'drs.log')
CONFIG['RAMP_NAME'] = 'ramps' + CONFIG['DATE'] + '.db'
CONFIG['DB_RAMP'] = os.path.join(CONFIG['BASE_RAMP'],CONFIG['RAMP_NAME'])
CONFIG['DB_OFFLINE'] = ''.join(('db_',CONFIG['DATE'],'_offline.db'))
CONFIG['DB_OFFLINE_PATH'] = os.path.join(CONFIG['WEBUI_DB_DIR'], CONFIG['DB_OFFLINE'])
CONFIG['DB_NIGHT'] = CONFIG['DB_OFFLINE']
CONFIG['DB_NIGHT_PATH'] = CONFIG['DB_OFFLINE_PATH']
maskc = ''.join(('GIANOB_MASKC_',CONFIG['DATE'],'.fits'))
CONFIG['MASK_C'] = os.path.join(CONFIG['RES_DIR'], maskc)
if docopt_args['<date>']:
CONFIG['DATE'] = docopt_args['<date>']
CONFIG['OFFLINE'] = True
CONFIG['RAW_DIR'] = os.path.join(CONFIG['BASE_RAW'],CONFIG['DATE'])
CONFIG['OFFLINE_DIR'] = os.path.join(CONFIG['BASE_RED_DIR'],CONFIG['DATE'])
CONFIG['RED_DIR'] = os.path.join(CONFIG['OFFLINE_DIR'],'offline')
CONFIG['RED_CALIB'] = os.path.join(CONFIG['RED_DIR'],'CALIB')
CONFIG['RED_STR'] = os.path.join(CONFIG['RED_DIR'],'STR')
CONFIG['LOG_FILE'] = os.path.join(CONFIG['RED_DIR'], 'drs.log')
CONFIG['DO_CALIB'] = {'dark': docopt_args['--dark'], 'flat': docopt_args['--flat'], 'une': docopt_args['--une'], 'fp': docopt_args['--fp'], 'only_calib': docopt_args['--only_calib']}
#CONFIG['DO_CALIB'] = {'dark':True,'flat':True,'une':True,'fp':True,'only_calib':True}
#CONFIG['DO_CALIB'] = {'dark':False,'flat':False,'une':False,'fp':False,'only_calib':False}
#CONFIG['USE_FLAT'] = {'global':False, 'order':False, 'nor':False}
if docopt_args['--use_flat'] == 'global':
CONFIG['USE_FLAT'] = {'global':True, 'order':False, 'nor':False}
elif docopt_args['--use_flat'] == 'order':
CONFIG['USE_FLAT'] = {'global':False, 'order':True, 'nor':False}
elif docopt_args['--use_flat'] == 'nor':
CONFIG['USE_FLAT'] = {'global':False, 'order':False, 'nor':True}
else:
CONFIG['USE_FLAT'] = {'global':False, 'order':False, 'nor':False}
maskc = ''.join(('GIANOB_MASKC_',CONFIG['DATE'],'.fits'))
CONFIG['MASK_C'] = os.path.join(CONFIG['RES_DIR'], maskc)
CONFIG['DB_OFFLINE'] = ''.join(('db_',CONFIG['DATE'],'_offline.db'))
CONFIG['DB_OFFLINE_PATH'] = os.path.join(CONFIG['WEBUI_DB_DIR'], CONFIG['DB_OFFLINE'])
CONFIG['DB_NIGHT'] = CONFIG['DB_OFFLINE']
CONFIG['DB_NIGHT_PATH'] = CONFIG['DB_OFFLINE_PATH']
# second argument: how to query the calibration database
# to be set if the the user wants to reduce the night using
# another night's calibrations
if docopt_args['<calib_date>']:
CONFIG['DATE'] = docopt_args['<calib_date>']
#print docopt_args
#print CONFIG['DATE']
#print CONFIG['DO_CALIB']
#print CONFIG['USE_FLAT']
# remove the mask with the order position (if it exists)
# it will be re-build with the correct flat-field
try:
os.remove(CONFIG['MASK_C'])
except:
pass
# Create the reduction directory
try:
os.mkdir(CONFIG['BASE_RED_DIR'])
except OSError:
pass
if CONFIG['OFFLINE']:
try:
os.mkdir(CONFIG['OFFLINE_DIR'])
except OSError:
pass
try:
os.mkdir(CONFIG['RED_DIR'])
except OSError:
pass
try:
os.mkdir(CONFIG['RED_CALIB'])
os.mkdir(CONFIG['RED_STR'])
except OSError:
pass
# Create the calibration directory tree
try:
os.mkdir(CONFIG['BASE_CALIB_DIR'])
except OSError:
pass
try:
os.mkdir(CONFIG['CALIB_DIR'])
except OSError:
pass
drslogger = DrsLogger()
# Databases online/offline
dbcalib = db.create_db_calib()
if CONFIG['OFFLINE']:
try:
os.remove(CONFIG['DB_NIGHT_PATH'])
except:
pass
else:
dbramps = db.connect_db_ramp()
dbnight = db.create_db_night()
do_calib = CONFIG['DO_CALIB']
# watchdog append fits files to the raw list
class UpdateRawList(PatternMatchingEventHandler):
def process(self, event):
raws.append(event.src_path)
drslogger.log('File %s has arrived.' % (event.src_path))
def on_created(self, event):
ext = os.path.splitext(event.src_path)[1]
if ext == '.fts':
self.process(event)
# data reduction
def GianoBreduce(rawfits,lists,dbcalib,working,group,do_calib,stop):
if stop:
try:
raw = rawfiles.GBCheckRawFiles(None,None,None,None,None)
group = raw.group
working = raw.working
if working == CONFIG['SCIENCE']:
if group['nodstare'] == CONFIG['NODVALUE'] and group['pair']:
lists[CONFIG['SCIENCE']].pop()
if group:
drslogger.log('Reducing pending frames.')
print 'Press Ctrl-C again to stop the pipeline now.'
raw.command[group['nodstare']] = 'incomplete_group'
elif working:
drslogger.log('Reducing pending %s frames.' % working)
print 'Press Ctrl-C again to stop the pipeline now.'
raw.command[working] = True
else:
return
except KeyboardInterrupt:
drslogger.log('Pipeline stopped.')
sys.exit(0)
# check raw file and insert in the correct list
else:
drslogger.log('Checking the FITS file ' + str(os.path.basename(rawfits)))
raw = rawfiles.GBCheckRawFiles(rawfits,lists,working,group,do_calib)
raw.check_raw_file()
working = raw.working
group = raw.group
go = raw.go
#do_calib = raw.do_calib
for message in raw.messages:
drslogger.log(message)
#print group
# see if the previous function passed a command and execute it
if raw.command[CONFIG['DARK']]:
drslogger.log('----------------------------------------------')
drslogger.log(' *** DARK FRAMES *** ')
stamp = time.time()
darks = darkframes.GBDarks(rawlists[CONFIG['DARK']],dbcalib)
darks.process()
for message in darks.messages:
drslogger.log(message)
#reduced = ','.join(map(os.path.basename,darks.darks))
#quality = ','.join(map(str,darks.quality))
#db.insert_calib(dbnight, 'DARK', reduced, quality, stamp)
db.insert_calib(dbnight, 'DARK', darks.darks, darks.quality, stamp)
#if not CONFIG['OFFLINE']:
# for reduced in rawlists[CONFIG['DARK']]:
# db.insert_dbnight(dbnight, reduced)
rawlists[CONFIG['DARK']][:] = []
group.clear()
drslogger.log('----------------------------------------------')
elif raw.command[CONFIG['FLAT']]:
drslogger.log('----------------------------------------------')
drslogger.log(' *** FLAT-FIELD FRAMES *** ')
stamp = time.time()
flats = flatframes.GBFlats(rawlists[CONFIG['FLAT']],dbcalib)
flats.process()
for message in flats.messages:
drslogger.log(message)
#reduced = ','.join(map(os.path.basename,flats.flats))
#quality = ','.join(map(str,flats.quality))
#db.insert_calib(dbnight, 'FLAT', reduced, quality, stamp)
db.insert_calib(dbnight, 'FLAT', flats.flats, flats.quality, stamp)
#if not CONFIG['OFFLINE']:
# for reduced in rawlists[CONFIG['FLAT']]:
# db.insert_dbnight(dbnight, reduced)
rawlists[CONFIG['FLAT']][:] = []
group.clear()
drslogger.log('----------------------------------------------')
elif raw.command[CONFIG['WCAL_UNE']]:
drslogger.log('----------------------------------------------')
drslogger.log(' *** UNE LAMP FRAMES *** ')
stamp = time.time()
wls = wlframes.GBWls(rawlists[CONFIG['WCAL_UNE']],dbcalib)
wls.une_process()
for message in wls.messages:
drslogger.log(message)
#reduced = ','.join(map(os.path.basename,wls.wls))
#quality = ','.join(map(str,wls.quality))
#db.insert_calib(dbnight, 'WCAL_UNE', reduced, quality, stamp)
db.insert_calib(dbnight, 'WCAL_UNE', wls.wls, wls.quality, stamp)
#if not CONFIG['OFFLINE']:
# for reduced in rawlists[CONFIG['WCAL_UNE']]:
# db.insert_dbnight(dbnight, reduced)
rawlists[CONFIG['WCAL_UNE']][:] = []
group.clear()
drslogger.log('----------------------------------------------')
elif raw.command[CONFIG['WCAL_FP']]:
drslogger.log('----------------------------------------------')
drslogger.log(' *** FP LAMP FRAMES *** ')
stamp = time.time()
fps = wlframes.GBWls(rawlists[CONFIG['WCAL_FP']],dbcalib)
fps.fp_process()
for message in fps.messages:
drslogger.log(message)
#reduced = ','.join(map(os.path.basename,fps.wls))
#quality = ','.join(map(str,fps.quality))
#db.insert_calib(dbnight, 'WCAL_FP', reduced, quality, stamp)
db.insert_calib(dbnight, 'WCAL_FP', fps.wls, fps.quality, stamp)
#if not CONFIG['OFFLINE']:
# for reduced in rawlists[CONFIG['WCAL_FP']]:
# db.insert_dbnight(dbnight, reduced)
rawlists[CONFIG['WCAL_FP']][:] = []
group.clear()
drslogger.log('----------------------------------------------')
elif raw.command[CONFIG['NODVALUE']] == 'pair':
drslogger.log('----------------------------------------------')
drslogger.log(' *** NODDING PAIR *** ')
t1 = time.time()
nods = nodding.GBNodding(rawlists[CONFIG['SCIENCE']],group,dbcalib,dbnight)
nods.pair_process()
for message in nods.messages:
drslogger.log(message)
t2 = time.time()
drslogger.log(' *** Reduction: %s s ***' % (str(round((t2-t1),2)), ))
#if not CONFIG['OFFLINE']:
# for reduced in rawlists[CONFIG['SCIENCE']]:
# db.insert_dbnight(dbnight, reduced)
rawlists[CONFIG['SCIENCE']][:] = []
drslogger.log('----------------------------------------------')
elif raw.command[CONFIG['STAREVALUE']] == 'pair':
drslogger.log('----------------------------------------------')
drslogger.log(' *** STARE PAIR *** ')
t1 = time.time()
stares = stare.GBStare(rawlists[CONFIG['SCIENCE']],group,dbcalib,dbnight)
stares.pair_process()
for message in stares.messages:
drslogger.log(message)
t2 = time.time()
drslogger.log(' *** Reduction: %s s ***' % (str(round((t2-t1),2)), ))
#if not CONFIG['OFFLINE']:
# for reduced in rawlists[CONFIG['SCIENCE']]:
# db.insert_dbnight(dbnight, reduced)
rawlists[CONFIG['SCIENCE']][:] = []
drslogger.log('----------------------------------------------')
elif raw.command[CONFIG['NODVALUE']] == 'group':
drslogger.log('----------------------------------------------')
drslogger.log(' *** NODDING GROUP *** ')
t1 = time.time()
nods = nodding.GBNodding(rawlists[CONFIG['SCIENCE']],group,dbcalib,dbnight)
nods.group_process()
for message in nods.messages:
drslogger.log(message)
t2 = time.time()
drslogger.log(' *** Reduction: %s s ***' % (str(round((t2-t1),2)), ))
#if not CONFIG['OFFLINE']:
# for reduced in rawlists[CONFIG['SCIENCE']]:
# db.insert_dbnight(dbnight, reduced)
rawlists[CONFIG['SCIENCE']][:] = []
drslogger.log('----------------------------------------------')
elif raw.command[CONFIG['STAREVALUE']] == 'group':
drslogger.log('----------------------------------------------')
drslogger.log(' *** STARE GROUP *** ')
t1 = time.time()
stares = stare.GBStare(rawlists[CONFIG['SCIENCE']],group,dbcalib,dbnight)
stares.group_process()
for message in stares.messages:
drslogger.log(message)
t2 = time.time()
drslogger.log(' *** Reduction: %s s ***' % (str(round((t2-t1),2)), ))
#if not CONFIG['OFFLINE']:
# for reduced in rawlists[CONFIG['SCIENCE']]:
# db.insert_dbnight(dbnight, reduced)
rawlists[CONFIG['SCIENCE']][:] = []
drslogger.log('----------------------------------------------')
elif raw.command[CONFIG['NODVALUE']] == 'incomplete_group':
drslogger.log('----------------------------------------------')
drslogger.log(' *** NODDING INCOMPLETE GROUP *** ')
t1 = time.time()
nods = nodding.GBNodding(rawlists[CONFIG['SCIENCE']],group,dbcalib,dbnight)
nods.ingroup_process()
for message in nods.messages:
drslogger.log(message)
t2 = time.time()
drslogger.log(' *** Reduction: %s s ***' % (str(round((t2-t1),2)), ))
#if not CONFIG['OFFLINE']:
# for reduced in rawlists[CONFIG['SCIENCE']]:
# db.insert_dbnight(dbnight, reduced)
rawlists[CONFIG['SCIENCE']][:] = []
drslogger.log('----------------------------------------------')
elif raw.command[CONFIG['STAREVALUE']] == 'incomplete_group':
drslogger.log('----------------------------------------------')
drslogger.log(' *** STARE INCOMPLETE GROUP *** ')
t1 = time.time()
stares = stare.GBStare(rawlists[CONFIG['SCIENCE']],group,dbcalib,dbnight)
stares.ingroup_process()
for message in stares.messages:
drslogger.log(message)
t2 = time.time()
drslogger.log(' *** Reduction: %s s ***' % (str(round((t2-t1),2)), ))
#if not CONFIG['OFFLINE']:
# for reduced in rawlists[CONFIG['SCIENCE']]:
# db.insert_dbnight(dbnight, reduced)
rawlists[CONFIG['SCIENCE']][:] = []
drslogger.log('----------------------------------------------')
else:
pass
if stop:
return
#print group
return working, group, go, do_calib
if __name__ == "__main__":
rawdir = CONFIG['RAW_DIR']
if not os.path.exists(rawdir):
drslogger.log('The RAW directory %s does not exist. GOFIO will now close down.' % rawdir)
drslogger.log('Pipeline stopped.')
dbcalib.close()
if not CONFIG['OFFLINE']:
dbramps.close()
dbnight.close()
sys.exit(0)
rawlists = {CONFIG['DARK']:[],CONFIG['FLAT']:[],CONFIG['WCAL_UNE']:[],CONFIG['WCAL_FP']:[],CONFIG['SCIENCE']:[]}
# store the information on what the pipeline is working on (dark, flat, ...) and which science group/pair
working = 0
group = {}
stop = False
# check if there are already files in RAW_DIR, create list raws in temporal order
fits = os.path.join(rawdir,"*.fts")
raws = glob.glob(fits)
raws.sort()
# use watchdog to check when new files are created
observer = Observer()
observer.schedule(UpdateRawList(),rawdir)
observer.start()
try:
while True:
time.sleep(1)
# the raw files are processed
while raws:
if CONFIG['OFFLINE']:
working, group, go, do_calib = GianoBreduce(raws[0], rawlists, dbcalib, working, group, do_calib, stop)
if go:
raws.pop(0)
#raws.pop(0)
else:
for n in xrange(len(raws)):
if db.check_night(dbnight,os.path.basename(raws[n])):
raws.pop(n)
break
#elif db.check_raw(dbramps,raws[n]):
elif db.check_raw(dbramps,os.path.basename(raws[n])):
working, group, go, do_calib = GianoBreduce(raws[n], rawlists, dbcalib, working, group, do_calib, stop)
if go:
raws.pop(n)
break
else:
time.sleep(1)
if CONFIG['OFFLINE']:
drslogger.log('No more files to reduce.')
dbcalib.close()
try: os.remove(CONFIG['MASK_C'])
except: pass
sys.exit(0)
except KeyboardInterrupt:
observer.stop()
# reduce any file still pending!
if working:
drslogger.log('Processing the last pending files (if any).')
stop = True
if CONFIG['OFFLINE']:
GianoBreduce(raws[0], rawlists, dbcalib, working, group, do_calib, stop)
else:
for n in xrange(len(raws)):
if db.check_raw(dbramps,raws[n]):
GianoBreduce(raws[n], rawlists, dbcalib, working, group, do_calib, stop)
break
try: os.remove(CONFIG['MASK_C'])
except: pass
drslogger.log('Pipeline stopped.')
dbcalib.close()
if not CONFIG['OFFLINE']:
dbramps.close()
dbnight.close()
sys.exit(0)
observer.join()
gofio/gofioDRS.py
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment