Commit 60a29b8a authored by Monica Rainer's avatar Monica Rainer
Browse files

Change structure, add gofio.py

parent 8a9821e2
backup backup
calibrations
git_howto.txt git_howto.txt
gofio_latest gofioDRS.py
gofio_latest.tar
reduced reduced
varie varie
vradlib vradlib
...@@ -18,4 +16,5 @@ webuidatabases ...@@ -18,4 +16,5 @@ webuidatabases
*/*/*/*.pyc */*/*/*.pyc
*~ *~
.~* .~*
gofio/calibrations
...@@ -7,6 +7,7 @@ Python packages: ...@@ -7,6 +7,7 @@ Python packages:
- NumPy v1.12 - NumPy v1.12
- SciPy v0.19 - SciPy v0.19
- watchdog v0.8.2 - watchdog v0.8.2
- docopt v0.6.1
Output: Output:
- intermediate: *_str.fits, 2D images with orders straightened and bad pixel removed - intermediate: *_str.fits, 2D images with orders straightened and bad pixel removed
......
...@@ -220,6 +220,7 @@ CONFIG['MASK_C'] = os.path.join(CONFIG['RES_DIR'], 'GIANOB_MASKC.fits') ...@@ -220,6 +220,7 @@ CONFIG['MASK_C'] = os.path.join(CONFIG['RES_DIR'], 'GIANOB_MASKC.fits')
CONFIG['A_POS'] = 8. # guess pixel position for A CONFIG['A_POS'] = 8. # guess pixel position for A
CONFIG['B_POS'] = 23. # guess pixel position for B CONFIG['B_POS'] = 23. # guess pixel position for B
CONFIG['C_POS'] = 15. # guess pixel position for C CONFIG['C_POS'] = 15. # guess pixel position for C
CONFIG['Y_POS'] = 2 # max. pixel difference between expected and real pixel position
CONFIG['HWTM'] = 6. # hwtm limit for contamination between A and B CONFIG['HWTM'] = 6. # hwtm limit for contamination between A and B
CONFIG['S1D'] = True # flag for creation of s1d output (True/False) CONFIG['S1D'] = True # flag for creation of s1d output (True/False)
CONFIG['S1D_NORM'] = False # flag for normalization of s1d output (True/False) CONFIG['S1D_NORM'] = False # flag for normalization of s1d output (True/False)
......
import os, sys, time
from drslib.config import CONFIG, read_usr_config
from drslib.logger import DrsLogger
from drslib import db, rawfiles, darkframes, flatframes, wlframes, nodding, stare
def gofio_init(docopt_args):
if docopt_args['--cfg']:
try:
USRCONFIG = read_usr_config(docopt_args['-g'])
except:
try:
USRCONFIG = read_usr_config(docopt_args['--cfg'])
except:
USRCONFIG = {}
USRCONFIG['OFFLINE'] = False
if USRCONFIG['OFFLINE']:
for key in USRCONFIG.keys():
CONFIG[key] = USRCONFIG[key]
CONFIG['RAW_DIR'] = os.path.join(CONFIG['BASE_RAW'],CONFIG['DATE'])
CONFIG['OFFLINE_DIR'] = os.path.join(CONFIG['BASE_RED_DIR'],CONFIG['DATE'])
CONFIG['RED_DIR'] = os.path.join(CONFIG['OFFLINE_DIR'],'offline')
CONFIG['RED_CALIB'] = os.path.join(CONFIG['RED_DIR'],'CALIB')
CONFIG['RED_STR'] = os.path.join(CONFIG['RED_DIR'],'STR')
CONFIG['LOG_FILE'] = os.path.join(CONFIG['RED_DIR'], 'drs.log')
CONFIG['RAMP_NAME'] = 'ramps' + CONFIG['DATE'] + '.db'
CONFIG['DB_RAMP'] = os.path.join(CONFIG['BASE_RAMP'],CONFIG['RAMP_NAME'])
CONFIG['DB_OFFLINE'] = ''.join(('db_',CONFIG['DATE'],'_offline.db'))
CONFIG['DB_OFFLINE_PATH'] = os.path.join(CONFIG['WEBUI_DB_DIR'], CONFIG['DB_OFFLINE'])
CONFIG['DB_NIGHT'] = CONFIG['DB_OFFLINE']
CONFIG['DB_NIGHT_PATH'] = CONFIG['DB_OFFLINE_PATH']
maskc = ''.join(('GIANOB_MASKC_',CONFIG['DATE'],'.fits'))
CONFIG['MASK_C'] = os.path.join(CONFIG['RES_DIR'], maskc)
if docopt_args['<date>']:
CONFIG['DATE'] = docopt_args['<date>']
CONFIG['OFFLINE'] = True
CONFIG['RAW_DIR'] = os.path.join(CONFIG['BASE_RAW'],CONFIG['DATE'])
CONFIG['OFFLINE_DIR'] = os.path.join(CONFIG['BASE_RED_DIR'],CONFIG['DATE'])
CONFIG['RED_DIR'] = os.path.join(CONFIG['OFFLINE_DIR'],'offline')
CONFIG['RED_CALIB'] = os.path.join(CONFIG['RED_DIR'],'CALIB')
CONFIG['RED_STR'] = os.path.join(CONFIG['RED_DIR'],'STR')
CONFIG['LOG_FILE'] = os.path.join(CONFIG['RED_DIR'], 'drs.log')
CONFIG['DO_CALIB'] = {'dark': docopt_args['--dark'], 'flat': docopt_args['--flat'], 'une': docopt_args['--une'], 'fp': docopt_args['--fp'], 'only_calib': docopt_args['--only_calib']}
if docopt_args['--use_flat'] == 'global':
CONFIG['USE_FLAT'] = {'global':True, 'order':False, 'nor':False}
elif docopt_args['--use_flat'] == 'order':
CONFIG['USE_FLAT'] = {'global':False, 'order':True, 'nor':False}
elif docopt_args['--use_flat'] == 'nor':
CONFIG['USE_FLAT'] = {'global':False, 'order':False, 'nor':True}
else:
CONFIG['USE_FLAT'] = {'global':False, 'order':False, 'nor':False}
maskc = ''.join(('GIANOB_MASKC_',CONFIG['DATE'],'.fits'))
CONFIG['MASK_C'] = os.path.join(CONFIG['RES_DIR'], maskc)
CONFIG['DB_OFFLINE'] = ''.join(('db_',CONFIG['DATE'],'_offline.db'))
CONFIG['DB_OFFLINE_PATH'] = os.path.join(CONFIG['WEBUI_DB_DIR'], CONFIG['DB_OFFLINE'])
CONFIG['DB_NIGHT'] = CONFIG['DB_OFFLINE']
CONFIG['DB_NIGHT_PATH'] = CONFIG['DB_OFFLINE_PATH']
# second argument: how to query the calibration database
# to be set if the the user wants to reduce the night using
# another night's calibrations
if docopt_args['<calib_date>']:
CONFIG['DATE'] = docopt_args['<calib_date>']
# remove the mask with the order position (if it exists)
# it will be re-build with the correct flat-field
try:
os.remove(CONFIG['MASK_C'])
except:
pass
# Create the reduction directory
try:
os.mkdir(CONFIG['BASE_RED_DIR'])
except OSError:
pass
if CONFIG['OFFLINE']:
try:
os.mkdir(CONFIG['OFFLINE_DIR'])
except OSError:
pass
try:
os.mkdir(CONFIG['RED_DIR'])
except OSError:
pass
try:
os.mkdir(CONFIG['RED_CALIB'])
os.mkdir(CONFIG['RED_STR'])
except OSError:
pass
# Create the calibration directory tree
try:
os.mkdir(CONFIG['BASE_CALIB_DIR'])
except OSError:
pass
try:
os.mkdir(CONFIG['CALIB_DIR'])
except OSError:
pass
# initialize logger
drslogger = DrsLogger()
# Databases online/offline
dbcalib = db.create_db_calib()
if CONFIG['OFFLINE']:
dbramps = False
try:
os.remove(CONFIG['DB_NIGHT_PATH'])
except:
pass
else:
dbramps = db.connect_db_ramp()
dbnight = db.create_db_night()
if not os.path.exists(CONFIG['RAW_DIR']):
drslogger.log('The RAW directory %s does not exist. GOFIO will now close down.' % CONFIG['RAW_DIR'])
drslogger.log('Pipeline stopped.')
dbcalib.close()
if not CONFIG['OFFLINE']:
dbramps.close()
dbnight.close()
sys.exit(0)
rawlists = {CONFIG['DARK']:[],CONFIG['FLAT']:[],CONFIG['WCAL_UNE']:[],CONFIG['WCAL_FP']:[],CONFIG['SCIENCE']:[]}
# store the information on what the pipeline is working on (dark, flat, ...) and which science group/pair
working = 0
group = {}
stop = False
# --------------------------------
# End initializing GOFIO
# --------------------------------
return CONFIG, rawlists, dbcalib, working, group, stop, drslogger, dbnight, dbramps
# data reduction
def GianoBreduce(rawfits,rawlists,dbcalib,working,group,do_calib,stop,drslogger, dbnight):
if stop:
try:
raw = rawfiles.GBCheckRawFiles(None,None,None,None,None)
group = raw.group
working = raw.working
if working == CONFIG['SCIENCE']:
if group['nodstare'] == CONFIG['NODVALUE'] and group['pair']:
rawlists[CONFIG['SCIENCE']].pop()
if group:
drslogger.log('Reducing pending frames.')
print 'Press Ctrl-C again to stop the pipeline now.'
raw.command[group['nodstare']] = 'incomplete_group'
elif working:
drslogger.log('Reducing pending %s frames.' % working)
print 'Press Ctrl-C again to stop the pipeline now.'
raw.command[working] = True
else:
return
except KeyboardInterrupt:
drslogger.log('Pipeline stopped.')
sys.exit(0)
# check raw file and insert in the correct list
else:
drslogger.log('Checking the FITS file ' + str(os.path.basename(rawfits)))
raw = rawfiles.GBCheckRawFiles(rawfits,rawlists,working,group,do_calib)
raw.check_raw_file()
working = raw.working
group = raw.group
go = raw.go
#do_calib = raw.do_calib
for message in raw.messages:
drslogger.log(message)
#print group
# see if the previous function passed a command and execute it
if raw.command[CONFIG['DARK']]:
drslogger.log('----------------------------------------------')
drslogger.log(' *** DARK FRAMES *** ')
stamp = time.time()
darks = darkframes.GBDarks(rawlists[CONFIG['DARK']],dbcalib)
darks.process()
for message in darks.messages:
drslogger.log(message)
db.insert_calib(dbnight, 'DARK', darks.darks, darks.quality, stamp)
rawlists[CONFIG['DARK']][:] = []
group.clear()
drslogger.log('----------------------------------------------')
elif raw.command[CONFIG['FLAT']]:
drslogger.log('----------------------------------------------')
drslogger.log(' *** FLAT-FIELD FRAMES *** ')
stamp = time.time()
flats = flatframes.GBFlats(rawlists[CONFIG['FLAT']],dbcalib)
flats.process()
for message in flats.messages:
drslogger.log(message)
db.insert_calib(dbnight, 'FLAT', flats.flats, flats.quality, stamp)
rawlists[CONFIG['FLAT']][:] = []
group.clear()
drslogger.log('----------------------------------------------')
elif raw.command[CONFIG['WCAL_UNE']]:
drslogger.log('----------------------------------------------')
drslogger.log(' *** UNE LAMP FRAMES *** ')
stamp = time.time()
wls = wlframes.GBWls(rawlists[CONFIG['WCAL_UNE']],dbcalib)
wls.une_process()
for message in wls.messages:
drslogger.log(message)
db.insert_calib(dbnight, 'WCAL_UNE', wls.wls, wls.quality, stamp)
rawlists[CONFIG['WCAL_UNE']][:] = []
group.clear()
drslogger.log('----------------------------------------------')
elif raw.command[CONFIG['WCAL_FP']]:
drslogger.log('----------------------------------------------')
drslogger.log(' *** FP LAMP FRAMES *** ')
stamp = time.time()
fps = wlframes.GBWls(rawlists[CONFIG['WCAL_FP']],dbcalib)
fps.fp_process()
for message in fps.messages:
drslogger.log(message)
db.insert_calib(dbnight, 'WCAL_FP', fps.wls, fps.quality, stamp)
rawlists[CONFIG['WCAL_FP']][:] = []
group.clear()
drslogger.log('----------------------------------------------')
elif raw.command[CONFIG['NODVALUE']] == 'pair':
drslogger.log('----------------------------------------------')
drslogger.log(' *** NODDING PAIR *** ')
t1 = time.time()
nods = nodding.GBNodding(rawlists[CONFIG['SCIENCE']],group,dbcalib,dbnight)
nods.pair_process()
for message in nods.messages:
drslogger.log(message)
t2 = time.time()
drslogger.log(' *** Reduction: %s s ***' % (str(round((t2-t1),2)), ))
rawlists[CONFIG['SCIENCE']][:] = []
drslogger.log('----------------------------------------------')
elif raw.command[CONFIG['STAREVALUE']] == 'pair':
drslogger.log('----------------------------------------------')
drslogger.log(' *** STARE PAIR *** ')
t1 = time.time()
stares = stare.GBStare(rawlists[CONFIG['SCIENCE']],group,dbcalib,dbnight)
stares.pair_process()
for message in stares.messages:
drslogger.log(message)
t2 = time.time()
drslogger.log(' *** Reduction: %s s ***' % (str(round((t2-t1),2)), ))
rawlists[CONFIG['SCIENCE']][:] = []
drslogger.log('----------------------------------------------')
elif raw.command[CONFIG['NODVALUE']] == 'group':
drslogger.log('----------------------------------------------')
drslogger.log(' *** NODDING GROUP *** ')
t1 = time.time()
nods = nodding.GBNodding(rawlists[CONFIG['SCIENCE']],group,dbcalib,dbnight)
nods.group_process()
for message in nods.messages:
drslogger.log(message)
t2 = time.time()
drslogger.log(' *** Reduction: %s s ***' % (str(round((t2-t1),2)), ))
rawlists[CONFIG['SCIENCE']][:] = []
drslogger.log('----------------------------------------------')
elif raw.command[CONFIG['STAREVALUE']] == 'group':
drslogger.log('----------------------------------------------')
drslogger.log(' *** STARE GROUP *** ')
t1 = time.time()
stares = stare.GBStare(rawlists[CONFIG['SCIENCE']],group,dbcalib,dbnight)
stares.group_process()
for message in stares.messages:
drslogger.log(message)
t2 = time.time()
drslogger.log(' *** Reduction: %s s ***' % (str(round((t2-t1),2)), ))
rawlists[CONFIG['SCIENCE']][:] = []
drslogger.log('----------------------------------------------')
elif raw.command[CONFIG['NODVALUE']] == 'incomplete_group':
drslogger.log('----------------------------------------------')
drslogger.log(' *** NODDING INCOMPLETE GROUP *** ')
t1 = time.time()
nods = nodding.GBNodding(rawlists[CONFIG['SCIENCE']],group,dbcalib,dbnight)
nods.ingroup_process()
for message in nods.messages:
drslogger.log(message)
t2 = time.time()
drslogger.log(' *** Reduction: %s s ***' % (str(round((t2-t1),2)), ))
rawlists[CONFIG['SCIENCE']][:] = []
drslogger.log('----------------------------------------------')
elif raw.command[CONFIG['STAREVALUE']] == 'incomplete_group':
drslogger.log('----------------------------------------------')
drslogger.log(' *** STARE INCOMPLETE GROUP *** ')
t1 = time.time()
stares = stare.GBStare(rawlists[CONFIG['SCIENCE']],group,dbcalib,dbnight)
stares.ingroup_process()
for message in stares.messages:
drslogger.log(message)
t2 = time.time()
drslogger.log(' *** Reduction: %s s ***' % (str(round((t2-t1),2)), ))
rawlists[CONFIG['SCIENCE']][:] = []
drslogger.log('----------------------------------------------')
else:
pass
if stop:
return
#print group
return working, group, go, do_calib, rawlists
""" """
Last modified: 2017-03-07
Reduction of the single AB nodding: Reduction of the single AB nodding:
- check the image quality (signal in well defined region) - check the image quality (signal in well defined region)
- check that the exposure times of A and B are the same - check that the exposure times of A and B are the same
......
""" """
Last modified: 2017-03-08
The FITS files are divided in darks, flats, UNes, FPs and science The FITS files are divided in darks, flats, UNes, FPs and science
using the header keywords (defined in config.py). using the header keywords (defined in config.py).
If all the images of one group have been taken, the command to If all the images of one group have been taken, the command to
process them is passed to the pipeline. process them is passed to the pipeline.
If a different kind of image arrives, the previous ones are processed If a different kind of image arrives, the previous ones are processed
even if the group is incomplete. even if the group is incomplete.
WARNING: the dark files are 4 times the expected number, because for WARNING: the dark files are n times the expected number, because for
each observations the ramp-processor creates 4 images with different each observations the ramp-processor creates n images with different
exposure times (10, 30, 60 and 100 seconds). exposure times (10, 30, 60, 100 seconds, and so on).
""" """
from drslib.config import CONFIG from drslib.config import CONFIG
......
""" """
Last modified: 2017-03-07
Reduction of the single Obj-Sky group: Reduction of the single Obj-Sky group:
- check the Obj image quality (signal in well defined region) - check the Obj image quality (signal in well defined region)
- check that the exposure times of Obj and Sky are the same - check that the exposure times of Obj and Sky are the same
......
""" """
Last modified: 2017-06-07
- badpix: bad pixels removals - badpix: bad pixels removals
- stdcombine: weights for flat and dark combiner - stdcombine: weights for flat and dark combiner
- optExtract: optimal extraction - optExtract: optimal extraction
...@@ -186,7 +184,7 @@ def optExtract(data,gain,ron,slit_pos,ordine): ...@@ -186,7 +184,7 @@ def optExtract(data,gain,ron,slit_pos,ordine):
hwtm = math.sqrt(2*math.log(10)) * abs(sigmagauss) # half-width at tenth-maximum hwtm = math.sqrt(2*math.log(10)) * abs(sigmagauss) # half-width at tenth-maximum
if abs(x0 - slit_pos) > 2: if abs(x0 - slit_pos) > CONFIG['Y_POS']:
x0 = slit_pos x0 = slit_pos
# if bad seeing, there can be overlapping between A and B # if bad seeing, there can be overlapping between A and B
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment