Commit c6755255 authored by Monica Rainer's avatar Monica Rainer
Browse files

Automatically compute the straighten option DY

parent 1d20c652
backup
fringing
git_howto.txt
gofioDRS.py
reduced
......@@ -17,4 +18,6 @@ webuidatabases
*~
.~*
gofio/calibrations
gofio/resources/straight_giano_2D
gofio/resources/straight_giano_2D_v1_1_old.f
......@@ -8,8 +8,59 @@ Python packages:
- SciPy v0.19
- watchdog v0.8.2
- docopt v0.6.1
Other requirements:
- fortran77 compiler
- libcfitsio library
Installation:
- preserve the directory tree:
gofio/
__init__.py
__version__.py
gofioDRS.py
drslib/
__init__.py
clean_db.py
config.py
darkframes.py
db.py
flatframes.py
gofio.py
logger.py
nodding.py
rawfiles.py
stare.py
varie.py
wlframes.py
berv/
__init__.py
astroTimeLegacy.py
baryvel.py
resources/
GIANOB_MASKC.fits
UNe_lines_GIANO_selected.txt
UNe_observed_lines_GIANOB_18nov2016.txt
badpix_mask.fits
straight_giano_2D_v1_1.f
- compile the fortran77 straight_giano_2D_v1_1.f file:
gfortran -o straight_giano_2D straight_giano_2D_v1_1.f -lcfitsio
- modify the configuration file drslib/config.py in order to set the
right directory path for raw and reduced files (CONFIG['BASE_RAW']
and CONFIG['BASE_RED_DIR'])
- run "python gofioDRS.py -h" for help
Output:
- intermediate: *_str.fits, 2D images with orders straightened and bad pixel removed
- final output: *_ms1d.fits. The spectra are flat-fielded, extracted, calibrated, and the cosmic rays are removed. The echelle orders are not merged. The FITS table has 50 rows and 4 columns, and it stores each echelle order in a row. The columns are: the number of the echelle order (from 32 - the reddest - to 81 - the bluest), the wavelength in nanometers, the fluxes and the signal-to-noise ratios.
- final output (optional): *_s1d.fits. The spectra are flat-fielded, extracted, calibrated, and the cosmic rays are removed. The echelle orders are merged. The barycentric correction is applied. These are monodimensional spectra with constant step in wavelength.
- intermediate: *_str.fits, 2D images with orders straightened
and bad pixel removed
- final output: *_ms1d.fits. The spectra are flat-fielded, extracted,
calibrated, and the cosmic rays are removed.
The echelle orders are not merged.
The FITS table has 50 rows and 4 columns, and it stores each echelle
order in a row. The columns are: the number of the echelle order
(from 32 - the reddest - to 81 - the bluest), the wavelength in nanometers,
the fluxes and the signal-to-noise ratios.
- final output (optional): *_s1d.fits. The spectra are flat-fielded, extracted,
calibrated, and the cosmic rays are removed. The echelle orders are merged.
The barycentric correction is applied. These are monodimensional spectra with
constant step in wavelength.
......@@ -91,8 +91,8 @@ CONFIG['BADPIX_MASK'] = os.path.join(CONFIG['RES_DIR'], 'badpix_mask.fits')
CONFIG['LOGGER'] = {'TERM':True,'FILE':True} # logger writes to terminal and/or to file
CONFIG['LOG_FILE'] = os.path.join(CONFIG['RED_DIR'], 'drs.log')
CONFIG['STRAIGHT'] = os.path.join(CONFIG['RES_DIR'],'straight_giano_2D')
#CONFIG['STRAIGHT_OPT'] = ['I=1']
CONFIG['STRAIGHT_OPT'] = ['I=1','DY=-5']
CONFIG['STRAIGHT_OPT'] = ['I=1'] # the option DY is computed automatically
#CONFIG['STRAIGHT_OPT'] = ['I=1','DY=-5'] # define DY only if you want to override the automatic procedure
CONFIG['BASE_CALIB_DIR'] = os.path.join(CONFIG['APP_DIR'], 'calibrations')
CONFIG['CALIB_DIR'] = os.path.join(CONFIG['BASE_CALIB_DIR'], 'calfiles')
CONFIG['CALIB_DB_DIR'] = os.path.join(CONFIG['BASE_CALIB_DIR'], 'database')
......@@ -185,12 +185,13 @@ CONFIG['DB_CALIB'] = 'drs_calib.db'
CONFIG['DB_CALIB_PATH'] = os.path.join(CONFIG['CALIB_DB_DIR'], CONFIG['DB_CALIB'])
CONFIG['DB_CALIB_TBL'] = 'calibrations'
CONFIG['DB_CALIB_COLS'] = OrderedDict([('01_data', 'date_calib'), ('dark10', 'masterdark10'), ('dark30', 'masterdark30'), ('dark60', 'masterdark60'), ('dark100', 'masterdark100'), ('dark200', 'masterdark200'), ('dark300', 'masterdark300'), ('dark600', 'masterdark600'),('flat', 'masterflat'), ('flatstr', 'masterflat_str'), ('flatnor', 'masterflat_nor'), ('une', 'wl_une'), ('une_str', 'wl_une_str'), ('une_calib', 'wl_une_calib'), ('fp', 'wl_fp'), ('fp_str', 'wl_fp_str'), ('fp_calib','wl_fp_calib')])
CONFIG['DB_CALIB_COLS'] = OrderedDict([('01_data', 'date_calib'), ('dark10', 'masterdark10'), ('dark30', 'masterdark30'), ('dark60', 'masterdark60'), ('dark100', 'masterdark100'), ('dark200', 'masterdark200'), ('dark300', 'masterdark300'), ('dark600', 'masterdark600'),('flat', 'masterflat'), ('flatstr', 'masterflat_str'), ('flatnor', 'masterflat_nor'), ('shiftY', 'shift_Y') ,('une', 'wl_une'), ('une_str', 'wl_une_str'), ('une_calib', 'wl_une_calib'), ('fp', 'wl_fp'), ('fp_str', 'wl_fp_str'), ('fp_calib','wl_fp_calib')])
CONFIG['DB_CALIB_DATATYPE'] = OrderedDict()
for colname in CONFIG['DB_CALIB_COLS']:
CONFIG['DB_CALIB_DATATYPE'][colname] = 'TEXT'
CONFIG['DB_CALIB_DATATYPE']['01_data'] = 'TEXT unique'
CONFIG['DB_CALIB_DATATYPE']['shiftY'] = 'INTEGER'
# Night reduction database
......@@ -222,7 +223,7 @@ CONFIG['MASK_C'] = os.path.join(CONFIG['RES_DIR'], 'GIANOB_MASKC.fits')
CONFIG['A_POS'] = 8. # guess pixel position for A
CONFIG['B_POS'] = 23. # guess pixel position for B
CONFIG['C_POS'] = 15. # guess pixel position for C
CONFIG['Y_POS'] = 2 # max. pixel difference between expected and real pixel position
CONFIG['Y_POS'] = 4 # max. pixel difference between expected and real pixel position
CONFIG['HWTM'] = 6. # hwtm limit for contamination between A and B
CONFIG['S1D'] = True # flag for creation of s1d output (True/False)
CONFIG['S1D_NORM'] = False # flag for normalization of s1d output (True/False)
......@@ -253,7 +254,7 @@ CONFIG['NDARK'] = len(CONFIG['DARKLIST']) # quanti tipi di dark (esempio: 10,30,
CONFIG['WEXT'] = 10 # extraction half-window for wavelength calibration, it will extract 2*wext around the center of the order
#
# CCD info
# Detector info
#
CONFIG['RON'] = 5.0 # electrons
......@@ -262,6 +263,7 @@ CONFIG['XCCD'] = 2048
CONFIG['YCCD'] = 2048
CONFIG['N_ORD'] = 50 # number of echelle orders
CONFIG['W_ORD'] = 41 # width of straighten order in pixel
CONFIG['SHIFT_Y'] = 20 # bottom position of flat signal on the lowest order and the left side of the detector
#
# Quality checks
......
......@@ -78,6 +78,7 @@ class GBFlats():
using ron and gain, correct ron and gain --> ron_eff and gain_eff
Write the result in the database.
"""
if self.flatlist:
exptime = self.flatlist[0].header[CONFIG['KEYS']['EXPTIME']]
darkname = 'dark' + str(int(exptime))
......@@ -117,7 +118,7 @@ class GBFlats():
for flat in self.flatlist:
# subdtract the masterdak
# subdtract the masterdark
if use_dark:
subflat = ccdproc.subtract_dark(flat,mdark,exposure_time=CONFIG['KEYS']['EXPTIME'],exposure_unit=u.second)
self.messages.append('Masterdark subtracted.')
......@@ -160,19 +161,44 @@ class GBFlats():
masterflat = fits.HDUList([hdu])
masterflat.writeto(nome,clobber=True)
mflat = None
db.insert_dbfile(self.dbconn,'flat',nome)
self.messages.append('Masterflat %s was created and inserted in the calibration database' % (str(os.path.basename(nome))))
shiftY = varie.shiftY(mflat.data)
db.insert_dbfile(self.dbconn,'shiftY',shiftY)
mflat = None
else:
self.messages.append('There are no available flats. The masterflat will be taken from the calibration database.')
if db.check_dbfile(self.dbconn,'flat'):
try:
db.copy_dbfile(self.dbconn,'flat')
try:
db.copy_dbfile(self.dbconn,'shiftY')
shift = db.extract_dbfile(self.dbconn,'shiftY')
if not shift:
cal_flat = db.extract_dbfile(self.dbconn,'flat')
mflat = ccdproc.CCDData.read(cal_flat, unit=u.adu)
shiftY = varie.shiftY(mflat.data)
db.insert_dbfile(self.dbconn,'shiftY',shiftY)
except:
cal_flat = db.extract_dbfile(self.dbconn,'flat')
mflat = ccdproc.CCDData.read(cal_flat, unit=u.adu)
shiftY = varie.shiftY(mflat.data)
db.insert_dbfile(self.dbconn,'shiftY',shiftY)
except:
self.messages.append('There are no flats in the calibration database.')
return False
if db.check_dbfile(self.dbconn,'shiftY'):
cal_flat = db.extract_dbfile(self.dbconn,'flat')
mflat = ccdproc.CCDData.read(cal_flat, unit=u.adu)
shiftY = varie.shiftY(mflat.data)
db.insert_dbfile(self.dbconn,'shiftY',shiftY)
return True
......@@ -189,9 +215,21 @@ class GBFlats():
nor = flat.replace('.fits','_nor.fits')
#extracted = flat.replace('.fits','_ext.fits')
shift = db.extract_dbfile(self.dbconn,'shiftY')
if not shift:
cal_flat = db.extract_dbfile(self.dbconn,'flat')
mflat = ccdproc.CCDData.read(cal_flat, unit=u.adu)
shift = varie.shiftY(mflat.data)
db.insert_dbfile(self.dbconn,'shiftY',shift)
shiftY = ''.join(('DY=',str(shift - CONFIG['SHIFT_Y'])))
#print shiftY
#args = [CONFIG['STRAIGHT'],flat,straight,CONFIG['STRAIGHT_OPT']]
args = [CONFIG['STRAIGHT'],flat,straight]
args.extend(CONFIG['STRAIGHT_OPT'])
args.extend(shiftY)
subprocess.call(args)
red_straight = os.path.join(CONFIG['RED_CALIB'],os.path.basename(straight))
......
......@@ -60,6 +60,18 @@ def gofio_init(docopt_args):
else:
CONFIG['USE_FLAT'] = {'global':False, 'order':False, 'nor':False}
if docopt_args['--s1d'] == 'norm':
CONFIG['S1D'] = True
CONFIG['S1D_NORM'] = True
elif docopt_args['--s1d'] == 'no':
CONFIG['S1D'] = False
CONFIG['S1D_NORM'] = False
else:
CONFIG['S1D'] = True
CONFIG['S1D_NORM'] = False
maskc = ''.join(('GIANOB_MASKC_',CONFIG['DATE'],'.fits'))
CONFIG['MASK_C'] = os.path.join(CONFIG['RES_DIR'], maskc)
......
......@@ -177,10 +177,32 @@ class GBNodding():
# straighten
straight = fitsfile.replace('.fits','_str.fits')
#args = [CONFIG['STRAIGHT'],fitsfile,straight,CONFIG['STRAIGHT_OPT']]
shift = db.extract_dbfile(self.dbconn,'shiftY')
if not shift:
try:
cal_flat = db.extract_dbfile(self.dbconn,'flat')
mflat = ccdproc.CCDData.read(cal_flat, unit=u.adu)
shift = varie.shiftY(mflat.data)
db.insert_dbfile(self.dbconn,'shiftY',shift)
except:
try:
db.copy_dbfile(self.dbconn,'shiftY')
shift = db.extract_dbfile(self.dbconn,'shiftY')
if not shift:
shift = CONFIG['SHIFT_Y']
except:
shift = CONFIG['SHIFT_Y']
shiftY = ''.join(('DY=',str(shift - CONFIG['SHIFT_Y'])))
#print shiftY
args = [CONFIG['STRAIGHT'],fitsfile,straight]
args.extend(CONFIG['STRAIGHT_OPT'])
args.extend(shiftY)
subprocess.call(args)
str_file = os.path.join(CONFIG['RED_STR'],os.path.basename(straight))
try: shutil.copyfile(straight,str_file)
except: pass
......
......@@ -13,8 +13,8 @@ exposure times (10, 30, 60, 100 seconds, and so on).
from drslib.config import CONFIG
from astropy.io import fits
import os
import drslib.varie as varie
import drslib.db as db
#import drslib.varie as varie
#import drslib.db as db
class GBCheckRawFiles():
def __init__(self, rawfile,lists,working,group,do_calib):
......
......@@ -325,9 +325,30 @@ class GBStare():
# straighten
straight = fitsfile.replace('.fits','_str.fits')
#args = [CONFIG['STRAIGHT'],fitsfile,straight,CONFIG['STRAIGHT_OPT']]
shift = db.extract_dbfile(self.dbconn,'shiftY')
if not shift:
try:
cal_flat = db.extract_dbfile(self.dbconn,'flat')
mflat = ccdproc.CCDData.read(cal_flat, unit=u.adu)
shift = varie.shiftY(mflat.data)
db.insert_dbfile(self.dbconn,'shiftY',shift)
except:
try:
db.copy_dbfile(self.dbconn,'shiftY')
shift = db.extract_dbfile(self.dbconn,'shiftY')
if not shift:
shift = CONFIG['SHIFT_Y']
except:
shift = CONFIG['SHIFT_Y']
shiftY = ''.join(('DY=',str(shift - CONFIG['SHIFT_Y'])))
#print shiftY
args = [CONFIG['STRAIGHT'],fitsfile,straight]
args.extend(CONFIG['STRAIGHT_OPT'])
args.extend(shiftY)
subprocess.call(args)
str_file = os.path.join(CONFIG['RED_STR'],os.path.basename(straight))
try: shutil.copyfile(straight,str_file)
......
......@@ -90,6 +90,29 @@ def stdcombine(x,axis):
#return np.ma.sqrt(np.ma.mean((np.ma.absolute(x- np.ma.mean(x)) /CONFIG['GAIN']) + ((CONFIG['RON']/CONFIG['GAIN']) ** 2)))
return np.ma.sqrt(np.ma.mean((np.ma.absolute(x- np.ma.median(x)) /CONFIG['GAIN']) + ((CONFIG['RON']/CONFIG['GAIN']) ** 2)))
#-------------- Define straighten option vertical shift -------------------
def shiftY(fdata):
# search for shift defined in the straighten options in config.py
try:
for opt in CONFIG['STRAIGHT_OPT']:
try:
dy = opt.rindex('DY=')
ypos = int(opt[dy-1:])
shift = CONFIG['SHIFT_Y'] + ypos
return shift
# if the shift is not defined, compute it
except:
column = fdata[0:140,0] # the bottom 140 pixel of the first column on the left of the detector
background = np.sort(column)[20] # value of the 20th pixel after arranging them in ascending order
#print column
for i in xrange(len(column)):
if column[i] > 3*background:
if np.median(column[i+1:i+6]) > 3*background:
shift = i
return shift
# if the shift computation fails, use the default definition
return CONFIG['SHIFT_Y']
#--------------------- Build the extraction mask -------------------
def buildMaskC(fdata):
......
......@@ -164,9 +164,29 @@ class GBWls():
straight = lamp.replace('.fits','_str.fits')
extracted = lamp.replace('.fits','_ext.fits')
#args = [CONFIG['STRAIGHT'],lamp,straight,CONFIG['STRAIGHT_OPT']]
shift = db.extract_dbfile(self.dbconn,'shiftY')
if not shift:
try:
cal_flat = db.extract_dbfile(self.dbconn,'flat')
mflat = ccdproc.CCDData.read(cal_flat, unit=u.adu)
shift = varie.shiftY(mflat.data)
db.insert_dbfile(self.dbconn,'shiftY',shift)
except:
try:
db.copy_dbfile(self.dbconn,'shiftY')
shift = db.extract_dbfile(self.dbconn,'shiftY')
if not shift:
shift = CONFIG['SHIFT_Y']
except:
shift = CONFIG['SHIFT_Y']
shiftY = ''.join(('DY=',str(shift - CONFIG['SHIFT_Y'])))
#print shiftY
args = [CONFIG['STRAIGHT'],lamp,straight]
args.extend(CONFIG['STRAIGHT_OPT'])
args.extend(shiftY)
subprocess.call(args)
red_straight = os.path.join(CONFIG['RED_CALIB'],os.path.basename(straight))
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment