Commit c8cfe94d authored by Monica Rainer's avatar Monica Rainer
Browse files

Merge branch 'v1.1.0' into develop

parents 30cf7cee 2ba58775
...@@ -11,6 +11,7 @@ DA_FARE.txt ...@@ -11,6 +11,7 @@ DA_FARE.txt
GIARPS_GOFIO_v1.docx GIARPS_GOFIO_v1.docx
GIARPS_GOFIO_v1.odt GIARPS_GOFIO_v1.odt
patchfile patchfile
setup.py
vrad_telluric.py vrad_telluric.py
webuidatabases webuidatabases
*/*.pyc */*.pyc
......
...@@ -66,16 +66,18 @@ CONFIG['OFFLINE'] = False ...@@ -66,16 +66,18 @@ CONFIG['OFFLINE'] = False
# Directories # Directories
# #
#CONFIG['BASE_RAW'] = '/mnt/nfs/RAW'
CONFIG['BASE_RAW'] = '/home/monica/Documenti/gianob/gianobdrs/varie/commissioning_marzo/raw'
CONFIG['BASE_RAMP'] = '/home/avet/gianorp/databases'
CONFIG['USR_DIR'] = os.environ.get('HOME') CONFIG['USR_DIR'] = os.environ.get('HOME')
CONFIG['TMP_DIR'] = tempfile.gettempdir() CONFIG['TMP_DIR'] = tempfile.gettempdir()
CONFIG['APP_DIR'] = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) CONFIG['APP_DIR'] = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
#CONFIG['BASE_RAW'] = '/mnt/nfs/RAW'
CONFIG['BASE_RAW'] = '/home/monica/Documenti/gianob/gianobdrs/varie/commissioning_marzo/raw'
CONFIG['BASE_RED_DIR'] = os.path.join(CONFIG['APP_DIR'],'reduced') CONFIG['BASE_RED_DIR'] = os.path.join(CONFIG['APP_DIR'],'reduced')
#CONFIG['BASE_RED_DIR'] = '/mnt/nfs/REDUCED' #CONFIG['BASE_RED_DIR'] = '/mnt/nfs/REDUCED'
CONFIG['BASE_RAMP'] = '/home/avet/gianorp/databases'
CONFIG['RAW_DIR'] = os.path.join(CONFIG['BASE_RAW'],CONFIG['DATE']) CONFIG['RAW_DIR'] = os.path.join(CONFIG['BASE_RAW'],CONFIG['DATE'])
CONFIG['OFFLINE_DIR'] = False CONFIG['OFFLINE_DIR'] = False
CONFIG['RED_DIR'] = os.path.join(CONFIG['BASE_RED_DIR'],CONFIG['DATE']) CONFIG['RED_DIR'] = os.path.join(CONFIG['BASE_RED_DIR'],CONFIG['DATE'])
...@@ -154,6 +156,7 @@ CONFIG['HJD'] = (' '.join((CONFIG['KEY_DRS'],'HJD')),'HJD of the combined image' ...@@ -154,6 +156,7 @@ CONFIG['HJD'] = (' '.join((CONFIG['KEY_DRS'],'HJD')),'HJD of the combined image'
CONFIG['AIRMASS'] = (' '.join((CONFIG['KEY_DRS'],'AIRMASS')),'Airmass of the combined image') CONFIG['AIRMASS'] = (' '.join((CONFIG['KEY_DRS'],'AIRMASS')),'Airmass of the combined image')
CONFIG['MASTERFLAT'] = (' '.join((CONFIG['KEY_DRS'],'FLAT')),'Flat-field used for the reduction') CONFIG['MASTERFLAT'] = (' '.join((CONFIG['KEY_DRS'],'FLAT')),'Flat-field used for the reduction')
CONFIG['MASTERLAMP'] = (' '.join((CONFIG['KEY_DRS'],'LAMP')),'Calibration lamp used for the reduction') CONFIG['MASTERLAMP'] = (' '.join((CONFIG['KEY_DRS'],'LAMP')),'Calibration lamp used for the reduction')
CONFIG['KEY_WEXT'] = (' '.join((CONFIG['KEY_DRS'],'WEXT')),'Standard extraction window [pixel]')
# Wavelength calibration # Wavelength calibration
CONFIG['CAL_FUNC'] = {'Oliva':True, 'Poly3':False} # function used for wavelength calibration (Tino Oliva Polynomial or 3-degree polynomial) CONFIG['CAL_FUNC'] = {'Oliva':True, 'Poly3':False} # function used for wavelength calibration (Tino Oliva Polynomial or 3-degree polynomial)
...@@ -253,7 +256,7 @@ CONFIG['FP_EXPT'] = 60.0 ...@@ -253,7 +256,7 @@ CONFIG['FP_EXPT'] = 60.0
CONFIG['DARKLIST'] = {10:[],30:[],60:[],100:[],200:[],300:[],600:[]} # exptime dei dark CONFIG['DARKLIST'] = {10:[],30:[],60:[],100:[],200:[],300:[],600:[]} # exptime dei dark
CONFIG['NDARK'] = len(CONFIG['DARKLIST']) # quanti tipi di dark (esempio: 10,30,60,100 sec = 4) meglio sovrastimare CONFIG['NDARK'] = len(CONFIG['DARKLIST']) # quanti tipi di dark (esempio: 10,30,60,100 sec = 4) meglio sovrastimare
CONFIG['WEXT'] = 5 # extraction window for wavelength calibration, it will extract 2*wext around the center of the order CONFIG['WEXT'] = 5 # extraction half-window for wavelength calibration, it will extract 2*wext around the center of the order
# #
# CCD info # CCD info
......
...@@ -28,7 +28,7 @@ def create_db_calib(): ...@@ -28,7 +28,7 @@ def create_db_calib():
.format(table=CONFIG['DB_CALIB_TBL'],column=CONFIG['DB_CALIB_COLS'][colname], datatype=CONFIG['DB_CALIB_DATATYPE'][colname])) .format(table=CONFIG['DB_CALIB_TBL'],column=CONFIG['DB_CALIB_COLS'][colname], datatype=CONFIG['DB_CALIB_DATATYPE'][colname]))
except: except:
#print CONFIG['DB_CALIB_TBL'] #print CONFIG['DB_CALIB_TBL']
print colname #print colname
#print CONFIG['DB_CALIB_DATATYPE'][colname] #print CONFIG['DB_CALIB_DATATYPE'][colname]
pass pass
......
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
Last modified: 2017-03-07 Last modified: 2017-03-07
Reduction of the flat-fields: Reduction of the flat-fields:
- check the exposure time - NOT ANYMORE: check the exposure time
- check the flux in a chosen area - check the flux in a chosen area
- subtract the masterdark - subtract the masterdark
- remove bad pixel using the bad pixel mask - remove bad pixel using the bad pixel mask
...@@ -176,7 +176,6 @@ class GBFlats(): ...@@ -176,7 +176,6 @@ class GBFlats():
""" """
Straighten the masterflat, normalize it and create the masked array Straighten the masterflat, normalize it and create the masked array
for A, B and C extraction for A, B and C extraction
MODIFIED: no normalization, only straightened and mask C creation
""" """
# straighten the flat # straighten the flat
......
...@@ -474,6 +474,7 @@ def UNe_calibrate(lamp,order,select_lines,all_lines,use_oliva=CONFIG['CAL_FUNC'] ...@@ -474,6 +474,7 @@ def UNe_calibrate(lamp,order,select_lines,all_lines,use_oliva=CONFIG['CAL_FUNC']
messages = [] messages = []
#print order #print order
calib_failed = False
# define gaussian function: # define gaussian function:
def gaussian(x,p,c,sg): def gaussian(x,p,c,sg):
...@@ -710,7 +711,7 @@ def UNe_calibrate(lamp,order,select_lines,all_lines,use_oliva=CONFIG['CAL_FUNC'] ...@@ -710,7 +711,7 @@ def UNe_calibrate(lamp,order,select_lines,all_lines,use_oliva=CONFIG['CAL_FUNC']
if use_oliva: if use_oliva:
#coeffs = OrderedDict({'k1':coeffk1,'k2':coeffk2,'k3':coeffk3,'l0':coeffl0,'xc':coeffxc0,'rms':coeffrms}) #coeffs = OrderedDict({'k1':coeffk1,'k2':coeffk2,'k3':coeffk3,'l0':coeffl0,'xc':coeffxc0,'rms':coeffrms})
coeffs.update({'k1':coeffk1,'k2':coeffk2,'k3':coeffk3,'l0':coeffl0,'xc':coeffxc0,'rms':coeffrms}) coeffs.update({'k1':coeffk1,'k2':coeffk2,'k3':coeffk3,'l0':coeffl0,'xc':coeffxc0,'rms':coeffrms})
calib_failed = False #calib_failed = False
#calib = lambdafit(pixrange,lambda0,xc0) #calib = lambdafit(pixrange,lambda0,xc0)
#coeffs = OrderedDict({'k1':k1,'k2':k2,'k3':k3,'l0':lambda0,'xc':xc0,'rms':rvrmse.value}) #coeffs = OrderedDict({'k1':k1,'k2':k2,'k3':k3,'l0':lambda0,'xc':xc0,'rms':rvrmse.value})
...@@ -750,7 +751,8 @@ def UNe_calibrate(lamp,order,select_lines,all_lines,use_oliva=CONFIG['CAL_FUNC'] ...@@ -750,7 +751,8 @@ def UNe_calibrate(lamp,order,select_lines,all_lines,use_oliva=CONFIG['CAL_FUNC']
coeffc3 = fitpoly3[3] coeffc3 = fitpoly3[3]
coeffrms = round(rvrmse.value,2) coeffrms = round(rvrmse.value,2)
calib_failed = False #calib_failed = False
#coeffs = OrderedDict({'c0':coeffc0,'c1':coeffc1,'c2':coeffc2,'c3':coeffc3,'rms':coeffrms}) #coeffs = OrderedDict({'c0':coeffc0,'c1':coeffc1,'c2':coeffc2,'c3':coeffc3,'rms':coeffrms})
coeffs.update({'c0':coeffc0,'c1':coeffc1,'c2':coeffc2,'c3':coeffc3,'rms_poly':coeffrms}) coeffs.update({'c0':coeffc0,'c1':coeffc1,'c2':coeffc2,'c3':coeffc3,'rms_poly':coeffrms})
......
...@@ -9,7 +9,7 @@ Reduction of the UNe lamps: ...@@ -9,7 +9,7 @@ Reduction of the UNe lamps:
- if more than one lamps, combine them and adjust ron and gain - if more than one lamps, combine them and adjust ron and gain
- save the masterlamp in the calibration database - save the masterlamp in the calibration database
- straigthen the masterlamp - straigthen the masterlamp
- extract the central 4 pixels of each order for quality check - extract the central 10 pixels of each order for quality check??
- UNe wavelength calibration (TO DO: FP) - UNe wavelength calibration (TO DO: FP)
- TO DO: UNe line diagnostics (checking the ratio between specific emission lines) - TO DO: UNe line diagnostics (checking the ratio between specific emission lines)
""" """
...@@ -120,7 +120,7 @@ class GBWls(): ...@@ -120,7 +120,7 @@ class GBWls():
else: else:
wlname = hea[CONFIG['KEYS']['IMANAME']].replace('.fts','_FP.fits') wlname = hea[CONFIG['KEYS']['IMANAME']].replace('.fts','_FP.fits')
wlname = wlname.replace(' ', '').replace('-', '_') #wlname = wlname.replace(' ', '').replace('-', '_')
nome = os.path.join(CONFIG['CALIB_DIR'],wlname) nome = os.path.join(CONFIG['CALIB_DIR'],wlname)
mwl.header = hea mwl.header = hea
...@@ -236,13 +236,20 @@ class GBWls(): ...@@ -236,13 +236,20 @@ class GBWls():
#print wcalib #print wcalib
wlc = ccdproc.CCDData.read(wcalib, unit=u.adu) wlc = ccdproc.CCDData.read(wcalib, unit=u.adu)
wl.header = wlc.header
for key in CONFIG['WLCOEFFS']:
keyword = ''.join((CONFIG['WLCOEFFS'][key][0],str(x+32)))
try:
wl.header[keyword] = (wlc.header[keyword],CONFIG['WLCOEFFS'][key][1])
except:
pass
# save the extracted lamp as FITS file # save the extracted lamp as FITS file
wlextract = ccdproc.CCDData(ordini, unit=u.adu) wlextract = ccdproc.CCDData(ordini, unit=u.adu)
wlextract.data = np.asarray(wlextract.data, dtype='float32') wlextract.data = np.asarray(wlextract.data, dtype='float32')
wlextract.header = wl.header wlextract.header = wl.header
wlextract.header[CONFIG['KEY_WEXT'][0]] = (CONFIG['WEXT']*2,CONFIG['KEY_WEXT'][1])
hdu = fits.PrimaryHDU(data=wlextract.data,header=wlextract.header) hdu = fits.PrimaryHDU(data=wlextract.data,header=wlextract.header)
wlextr = fits.HDUList([hdu]) wlextr = fits.HDUList([hdu])
wlextr.writeto(extracted,clobber=True) wlextr.writeto(extracted,clobber=True)
......
""" """
GOFIO DRS GOFIO DRS v1.1.0
Last modified: 2017-03-07
Written by Monica Rainer Written by Monica Rainer
Usage:
gofioDRS.py -h
gofioDRS.py [-g=<gofio_config>]
gofioDRS.py <date> [--dark --flat --une --fp --only_calib --use_flat=<flag>]
gofioDRS.py <date> <calib_date> [--dark --flat --une --fp --only_calib --use_flat=<flag>]
Options:
-h,--help : show this screen
-g,--cfg : path of the configuration file (optional)
date : date to be reduced, no configuration file possible (always first input)
calib_date : calibration date to be used (always second input)
--dark : darks are reduced
--flat : flats are reduced
--une : U-Ne lamps are reduced
--fp : FP lamps are reduced
--only_calib : only calibration are reduced, no science images
--use_flat=<flag> : flat-field is removed, options are global/order/nor/none [Default: order]
""" """
import os, glob, sys, time import os, glob, sys, time
from drslib.config import CONFIG from docopt import docopt
from drslib.config import CONFIG, read_usr_config
from logger.logger import DrsLogger from logger.logger import DrsLogger
from drslib import db, rawfiles, darkframes, flatframes, wlframes, nodding, stare, varie from drslib import db, rawfiles, darkframes, flatframes, wlframes, nodding, stare, varie
...@@ -15,11 +35,43 @@ from drslib import db, rawfiles, darkframes, flatframes, wlframes, nodding, star ...@@ -15,11 +35,43 @@ from drslib import db, rawfiles, darkframes, flatframes, wlframes, nodding, star
from watchdog.observers import Observer from watchdog.observers import Observer
from watchdog.events import PatternMatchingEventHandler from watchdog.events import PatternMatchingEventHandler
# first argument: night to be reduced, using the calibration in the database
# to be used at the end of the night, after the calibration lamp has been # read docopt_arg and act accordingly
# observed and put in the database
try: docopt_args = docopt(__doc__)
CONFIG['DATE'] = sys.argv[1] #print docopt_args
if docopt_args['--cfg']:
try:
USRCONFIG = read_usr_config(docopt_args['-g'])
except:
try:
USRCONFIG = read_usr_config(docopt_args['--cfg'])
except:
USRCONFIG = {}
USRCONFIG['OFFLINE'] = False
if USRCONFIG['OFFLINE']:
for key in USRCONFIG.keys():
CONFIG[key] = USRCONFIG[key]
CONFIG['RAW_DIR'] = os.path.join(CONFIG['BASE_RAW'],CONFIG['DATE'])
CONFIG['OFFLINE_DIR'] = os.path.join(CONFIG['BASE_RED_DIR'],CONFIG['DATE'])
CONFIG['RED_DIR'] = os.path.join(CONFIG['OFFLINE_DIR'],'offline')
CONFIG['RED_CALIB'] = os.path.join(CONFIG['RED_DIR'],'CALIB')
CONFIG['RED_STR'] = os.path.join(CONFIG['RED_DIR'],'STR')
CONFIG['LOG_FILE'] = os.path.join(CONFIG['RED_DIR'], 'drs.log')
CONFIG['RAMP_NAME'] = 'ramps' + CONFIG['DATE'] + '.db'
CONFIG['DB_RAMP'] = os.path.join(CONFIG['BASE_RAMP'],CONFIG['RAMP_NAME'])
CONFIG['DB_OFFLINE'] = ''.join(('db_',CONFIG['DATE'],'_offline.db'))
CONFIG['DB_OFFLINE_PATH'] = os.path.join(CONFIG['WEBUI_DB_DIR'], CONFIG['DB_OFFLINE'])
CONFIG['DB_NIGHT'] = CONFIG['DB_OFFLINE']
CONFIG['DB_NIGHT_PATH'] = CONFIG['DB_OFFLINE_PATH']
if docopt_args['<date>']:
CONFIG['DATE'] = docopt_args['<date>']
CONFIG['OFFLINE'] = True CONFIG['OFFLINE'] = True
CONFIG['RAW_DIR'] = os.path.join(CONFIG['BASE_RAW'],CONFIG['DATE']) CONFIG['RAW_DIR'] = os.path.join(CONFIG['BASE_RAW'],CONFIG['DATE'])
CONFIG['OFFLINE_DIR'] = os.path.join(CONFIG['BASE_RED_DIR'],CONFIG['DATE']) CONFIG['OFFLINE_DIR'] = os.path.join(CONFIG['BASE_RED_DIR'],CONFIG['DATE'])
...@@ -27,9 +79,18 @@ try: ...@@ -27,9 +79,18 @@ try:
CONFIG['RED_CALIB'] = os.path.join(CONFIG['RED_DIR'],'CALIB') CONFIG['RED_CALIB'] = os.path.join(CONFIG['RED_DIR'],'CALIB')
CONFIG['RED_STR'] = os.path.join(CONFIG['RED_DIR'],'STR') CONFIG['RED_STR'] = os.path.join(CONFIG['RED_DIR'],'STR')
CONFIG['LOG_FILE'] = os.path.join(CONFIG['RED_DIR'], 'drs.log') CONFIG['LOG_FILE'] = os.path.join(CONFIG['RED_DIR'], 'drs.log')
CONFIG['DO_CALIB'] = {'dark': docopt_args['--dark'], 'flat': docopt_args['--flat'], 'une': docopt_args['--une'], 'fp': docopt_args['--fp'], 'only_calib': docopt_args['--only_calib']}
#CONFIG['DO_CALIB'] = {'dark':True,'flat':True,'une':True,'fp':True,'only_calib':True} #CONFIG['DO_CALIB'] = {'dark':True,'flat':True,'une':True,'fp':True,'only_calib':True}
CONFIG['DO_CALIB'] = {'dark':False,'flat':False,'une':False,'fp':False,'only_calib':False} #CONFIG['DO_CALIB'] = {'dark':False,'flat':False,'une':False,'fp':False,'only_calib':False}
#CONFIG['USE_FLAT'] = {'global':False, 'order':False, 'nor':False} #CONFIG['USE_FLAT'] = {'global':False, 'order':False, 'nor':False}
if docopt_args['--use_flat'] == 'global':
CONFIG['USE_FLAT'] = {'global':True, 'order':False, 'nor':False}
elif docopt_args['--use_flat'] == 'order':
CONFIG['USE_FLAT'] = {'global':False, 'order':True, 'nor':False}
elif docopt_args['--use_flat'] == 'nor':
CONFIG['USE_FLAT'] = {'global':False, 'order':False, 'nor':True}
else:
CONFIG['USE_FLAT'] = {'global':False, 'order':False, 'nor':False}
maskc = ''.join(('GIANOB_MASKC_',CONFIG['DATE'],'.fits')) maskc = ''.join(('GIANOB_MASKC_',CONFIG['DATE'],'.fits'))
CONFIG['MASK_C'] = os.path.join(CONFIG['RES_DIR'], maskc) CONFIG['MASK_C'] = os.path.join(CONFIG['RES_DIR'], maskc)
...@@ -39,17 +100,18 @@ try: ...@@ -39,17 +100,18 @@ try:
CONFIG['DB_NIGHT'] = CONFIG['DB_OFFLINE'] CONFIG['DB_NIGHT'] = CONFIG['DB_OFFLINE']
CONFIG['DB_NIGHT_PATH'] = CONFIG['DB_OFFLINE_PATH'] CONFIG['DB_NIGHT_PATH'] = CONFIG['DB_OFFLINE_PATH']
except:
pass
# second argument: how to query the calibration database # second argument: how to query the calibration database
# to be set if the the user wants to reduce the night using # to be set if the the user wants to reduce the night using
# another night's calibrations # another night's calibrations
try: if docopt_args['<calib_date>']:
CONFIG['DATE'] = sys.argv[2] CONFIG['DATE'] = docopt_args['<calib_date>']
except:
pass
#print docopt_args
#print CONFIG['DATE']
#print CONFIG['DO_CALIB']
#print CONFIG['USE_FLAT']
# remove the mask with the order position (if it exists) # remove the mask with the order position (if it exists)
# it will be re-build with the correct flat-field # it will be re-build with the correct flat-field
...@@ -59,6 +121,10 @@ except: ...@@ -59,6 +121,10 @@ except:
pass pass
# Create the reduction directory # Create the reduction directory
try:
os.mkdir(CONFIG['BASE_RED_DIR'])
except OSError:
pass
if CONFIG['OFFLINE']: if CONFIG['OFFLINE']:
try: try:
...@@ -78,13 +144,19 @@ except OSError: ...@@ -78,13 +144,19 @@ except OSError:
pass pass
# Create the calibration directory # Create the calibration directory tree
try:
os.mkdir(CONFIG['BASE_CALIB_DIR'])
except OSError:
pass
try: try:
os.mkdir(CONFIG['CALIB_DIR']) os.mkdir(CONFIG['CALIB_DIR'])
except OSError: except OSError:
pass pass
drslogger = DrsLogger() drslogger = DrsLogger()
# Databases online/offline # Databases online/offline
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment