Skip to content

GitLab

Commit cd7c5785 authored by Monica Rainer's avatar Monica Rainer
Browse files

New name/format for the output, fix bug in nodding.py

parent 63fe626d
Hide whitespace changes
Inline Side-by-side
Showing with 176 additions and 70 deletions
README.md
View file @ cd7c5785
......@@ -10,5 +10,5 @@ Python packages:
Output:
- intermediate: *_str.fits, 2D images with orders straightened and bad pixel removed
- final output: *_e2ds.fits. The spectra are flat-fielded, extracted, calibrated, and the cosmic rays are removed. The echelle orders are not merged. The FITS files contain three images 50 orders x 2048 pixels: the primary image contains the fluxes, the second contains the wavelength calibration and the third contains the signal-to-noise ratio.
- final output: *_ms1d.fits. The spectra are flat-fielded, extracted, calibrated, and the cosmic rays are removed. The echelle orders are not merged. The FITS table has 50 rows and 4 columns, and it stores each echelle order in a row. The columns are: the number of the echelle order (from 32 - the reddest - to 81 - the bluest), the wavelength in nanometers, the fluxes and the signal-to-noise ratios.
- final output (optional): *_s1d.fits. The spectra are flat-fielded, extracted, calibrated, and the cosmic rays are removed. The echelle orders are merged. The barycentric correction is applied. These are monodimensional spectra with constant step in wavelength.
drslib/config.py
View file @ cd7c5785
......@@ -96,6 +96,11 @@ CONFIG['CALIB_DIR'] = os.path.join(CONFIG['BASE_CALIB_DIR'], 'calfiles')
CONFIG['CALIB_DB_DIR'] = os.path.join(CONFIG['BASE_CALIB_DIR'], 'database')
CONFIG['NIGHT_DB_DIR'] = os.path.join(CONFIG['APP_DIR'], 'databases')
# Output name
CONFIG['MERGED'] = 's1d' # suffix for merged spectra
CONFIG['UNMERGED'] = 'ms1d' # suffix for unmerged spectra
#
# Header configuration
#
......@@ -150,7 +155,8 @@ CONFIG['MASTERFLAT'] = (' '.join((CONFIG['KEY_DRS'],'FLAT')),'Flat-field used fo
CONFIG['MASTERLAMP'] = (' '.join((CONFIG['KEY_DRS'],'LAMP')),'Calibration lamp used for the reduction')
# Wavelength calibration
CONFIG['CAL_FUNC'] = {'Oliva':True, 'Poly3':False} # function used for wavelength calibration (Tino Oliva Polynomial or 3-degree polynomial)
#CONFIG['CAL_FUNC'] = {'Oliva':True, 'Poly3':False} # function used for wavelength calibration (Tino Oliva Polynomial or 3-degree polynomial)
CONFIG['CAL_FUNC'] = {'Oliva':False, 'Poly3':True}
CONFIG['WLFIT'] = (' '.join((CONFIG['KEY_DRS'],'CAL WLFUNC')),'Function used for wavelength calibration')
if CONFIG['CAL_FUNC']['Oliva']:
......
drslib/nodding.py
View file @ cd7c5785
......@@ -391,11 +391,18 @@ class GBNodding():
nomebase = os.path.splitext(aname)[0]
#qui = aname.rindex('.')
if 'grp' in fitsfile:
calname = '_'.join((nomebase,'Agrp_e2ds.fits'))
calname1d = '_'.join((nomebase,'Agrp_s1d.fits'))
msfx = ''.join(('Agrp_',CONFIG['UNMERGED'],'.fits'))
sfx = ''.join(('Agrp_',CONFIG['MERGED'],'.fits'))
#calname = '_'.join((nomebase,msfx))
#calname1d = '_'.join((nomebase,sfx))
else:
calname = '_'.join((nomebase,'A_e2ds.fits'))
calname1d = '_'.join((nomebase,'A_s1d.fits'))
msfx = ''.join(('A_',CONFIG['UNMERGED'],'.fits'))
sfx = ''.join(('A_',CONFIG['MERGED'],'.fits'))
#calname = '_'.join((nomebase,'A_e2ds.fits'))
#calname1d = '_'.join((nomebase,'A_s1d.fits'))
calname = '_'.join((nomebase,msfx))
calname1d = '_'.join((nomebase,sfx))
#calname = str(os.path.basename(fitsfile)).replace('_AB','_A')
elif slit_pos == CONFIG['B_POS']:
......@@ -404,11 +411,18 @@ class GBNodding():
nomebase = os.path.splitext(bname)[0]
#qui = bname.rindex('.')
if 'grp' in fitsfile:
calname = '_'.join((nomebase,'Bgrp_e2ds.fits'))
calname1d = '_'.join((nomebase,'Bgrp_s1d.fits'))
msfx = ''.join(('Bgrp_',CONFIG['UNMERGED'],'.fits'))
sfx = ''.join(('Bgrp_',CONFIG['MERGED'],'.fits'))
#calname = '_'.join((nomebase,msfx))
#calname1d = '_'.join((nomebase,sfx))
else:
calname = '_'.join((nomebase,'B_e2ds.fits'))
calname1d = '_'.join((nomebase,'B_s1d.fits'))
msfx = ''.join(('B_',CONFIG['UNMERGED'],'.fits'))
sfx = ''.join(('B_',CONFIG['MERGED'],'.fits'))
#calname = '_'.join((nomebase,'B_e2ds.fits'))
#calname1d = '_'.join((nomebase,'B_s1d.fits'))
calname = '_'.join((nomebase,msfx))
calname1d = '_'.join((nomebase,sfx))
#calname = str(os.path.basename(fitsfile)).replace('_AB','_B')
else:
print 'Wrong slit position!'
......@@ -450,15 +464,26 @@ class GBNodding():
for o in xrange(CONFIG['N_ORD']):
waves[o] = varie.wcalib(heaspe,o)
spefits = fits.PrimaryHDU(optSpectrum,header=heaspe)
wavefits = fits.ImageHDU(waves,name='WAVE')
snrfits = fits.ImageHDU(snr,name='SNR')
#spefits = fits.PrimaryHDU(optSpectrum,header=heaspe)
#wavefits = fits.ImageHDU(waves,name='WAVE')
#snrfits = fits.ImageHDU(snr,name='SNR')
#results = fits.HDUList([spefits,wavefits,snrfits])
#calname = os.path.join(CONFIG['RED_DIR'],calname)
#results.writeto(calname,clobber=True)
#results = fits.HDUList([spefits])
results = fits.HDUList([spefits,wavefits,snrfits])
orders=np.arange(CONFIG['N_ORD'])+32
c1 = fits.Column(name='ORDER', format='I', array=orders)
c2 = fits.Column(name='WAVE', format=''.join((str(CONFIG['YCCD']),'D')), unit='nm', array=waves)
c3 = fits.Column(name='FLUX', format=''.join((str(CONFIG['YCCD']),'D')), array=optSpectrum)
c4 = fits.Column(name='SNR', format=''.join((str(CONFIG['YCCD']),'D')), array=snr)
tbhdu = fits.BinTableHDU.from_columns([c1, c2, c3, c4],header=heaspe)
calname = os.path.join(CONFIG['RED_DIR'],calname)
results.writeto(calname,clobber=True)
tbhdu.writeto(calname,clobber=True)
#t1 = time.time()
......@@ -494,23 +519,48 @@ class GBNodding():
def combine(self,acalib,bcalib,fsnr):
abnome = acalib.replace('_A_e2ds.fits','_AB_e2ds.fits')
abnome1d = acalib.replace('_A_e2ds.fits','_AB_s1d.fits')
abnome = abnome.replace('_Agrp_e2ds.fits','_ABgrp_e2ds.fits')
abnome1d = abnome1d.replace('_Agrp_e2ds.fits','_ABgrp_s1d.fits')
#abnome = acalib.replace('_A_e2ds.fits','_AB_e2ds.fits')
#abnome1d = acalib.replace('_A_e2ds.fits','_AB_s1d.fits')
#abnome = abnome.replace('_Agrp_e2ds.fits','_ABgrp_e2ds.fits')
#abnome1d = abnome1d.replace('_Agrp_e2ds.fits','_ABgrp_s1d.fits')
old = ''.join(('_A_',CONFIG['UNMERGED'],'.fits'))
oldgrp = ''.join(('_Agrp_',CONFIG['UNMERGED'],'.fits'))
sfx = ''.join(('_AB_',CONFIG['MERGED'],'.fits'))
msfx = ''.join(('_AB_',CONFIG['UNMERGED'],'.fits'))
grp = ''.join(('_ABgrp_',CONFIG['MERGED'],'.fits'))
mgrp = ''.join(('_ABgrp_',CONFIG['UNMERGED'],'.fits'))
abnome = acalib.replace(old,msfx)
abnome1d = acalib.replace(old,sfx)
abnome = abnome.replace(oldgrp,mgrp)
abnome1d = abnome1d.replace(oldgrp,grp)
#print abnome
#print abnome1d
acal = fits.open(acalib)
afluxes = acal[0].data
awaves = acal[0].header
#roneff = acal[0].header[CONFIG['RON_EFF'][0]]
#gaineff = acal[0].header[CONFIG['GAIN_EFF'][0]]
roneff = math.sqrt(2)*acal[0].header[CONFIG['RON_EFF'][0]]
gaineff = 2*acal[0].header[CONFIG['GAIN_EFF'][0]]
#afluxes = acal[0].data
#awaves = acal[0].header
#roneff = math.sqrt(2)*acal[0].header[CONFIG['RON_EFF'][0]]
#gaineff = 2*acal[0].header[CONFIG['GAIN_EFF'][0]]
adata = acal[1].data
#print adata.shape
afluxes = adata.field(2)
#print afluxes
#print afluxes.shape
awaves = acal[1].header
roneff = math.sqrt(2)*acal[1].header[CONFIG['RON_EFF'][0]]
gaineff = 2*acal[1].header[CONFIG['GAIN_EFF'][0]]
bcal = fits.open(bcalib)
bfluxes = bcal[0].data
bwaves = bcal[0].header
#bfluxes = bcal[0].data
#bwaves = bcal[0].header
bdata = bcal[1].data
bfluxes = bdata.field(2)
bwaves = bcal[1].header
abcalib = np.zeros((CONFIG['N_ORD'],CONFIG['YCCD']))
......@@ -526,7 +576,8 @@ class GBNodding():
abcalib = np.asarray(abcalib, dtype='float32')
abhea = acal[0].header
#abhea = acal[0].header
abhea = acal[1].header
abhea[CONFIG['RON_EFF'][0]] = (roneff,CONFIG['RON_EFF'][1])
abhea[CONFIG['GAIN_EFF'][0]] = (gaineff,CONFIG['GAIN_EFF'][1])
#abhea[CONFIG['TEXP_EFF'][0]] = (,CONFIG['TEXP_EFF'][1])
......@@ -556,11 +607,15 @@ class GBNodding():
try:
am_a = acal[0].header[CONFIG['AIRMASS'][0]]
am_b = bcal[0].header[CONFIG['AIRMASS'][0]]
#am_a = acal[0].header[CONFIG['AIRMASS'][0]]
#am_b = bcal[0].header[CONFIG['AIRMASS'][0]]
am_a = acal[1].header[CONFIG['AIRMASS'][0]]
am_b = bcal[1].header[CONFIG['AIRMASS'][0]]
except:
am_a = acal[0].header[CONFIG['KEYS']['AM']]
am_b = bcal[0].header[CONFIG['KEYS']['AM']]
#am_a = acal[0].header[CONFIG['KEYS']['AM']]
#am_b = bcal[0].header[CONFIG['KEYS']['AM']]
am_a = acal[1].header[CONFIG['KEYS']['AM']]
am_b = bcal[1].header[CONFIG['KEYS']['AM']]
am = (am_a+am_b)/2.0
......@@ -576,15 +631,26 @@ class GBNodding():
for o in xrange(CONFIG['N_ORD']):
waves[o] = varie.wcalib(heaspe,o)
spefits = fits.PrimaryHDU(abcalib,header=heaspe)
wavefits = fits.ImageHDU(waves,name='WAVE')
snrfits = fits.ImageHDU(snr,name='SNR')
#spefits = fits.PrimaryHDU(abcalib,header=heaspe)
#wavefits = fits.ImageHDU(waves,name='WAVE')
#snrfits = fits.ImageHDU(snr,name='SNR')
#results = fits.HDUList([spefits,wavefits,snrfits])
#results.writeto(abnome,clobber=True)
#results = fits.HDUList([spefits])
results = fits.HDUList([spefits,wavefits,snrfits])
orders=np.arange(CONFIG['N_ORD'])+32
c1 = fits.Column(name='ORDER', format='I', array=orders)
c2 = fits.Column(name='WAVE', format=''.join((str(CONFIG['YCCD']),'D')), unit='nm', array=waves)
c3 = fits.Column(name='FLUX', format=''.join((str(CONFIG['YCCD']),'D')), array=abcalib)
c4 = fits.Column(name='SNR', format=''.join((str(CONFIG['YCCD']),'D')), array=snr)
tbhdu = fits.BinTableHDU.from_columns([c1, c2, c3, c4],header=heaspe)
tbhdu.writeto(abnome,clobber=True)
results.writeto(abnome,clobber=True)
if CONFIG['S1D']:
......@@ -715,16 +781,19 @@ class GBNodding():
def group_process(self):
warnings.simplefilter('ignore', category=AstropyWarning)
self.pair_process()
if self.group['noddings'] and len(self.group['noddings'])>1 :
ab, heaA, heaB = self.group_avg()
acalib, fsnr, straight = self.reduce(ab,CONFIG['A_POS'], heaA)
bcalib, fsnr, straight = self.reduce(ab,CONFIG['B_POS'], heaB)
self.combine(acalib,bcalib,fsnr)
acalib = None
bcalib = None
os.remove(ab)
os.remove(straight)
else:
try:
if self.group['noddings'] and len(self.group['noddings'])>1 :
ab, heaA, heaB = self.group_avg()
acalib, fsnr, straight = self.reduce(ab,CONFIG['A_POS'], heaA)
bcalib, fsnr, straight = self.reduce(ab,CONFIG['B_POS'], heaB)
self.combine(acalib,bcalib,fsnr)
acalib = None
bcalib = None
os.remove(ab)
os.remove(straight)
else:
self.messages.append('There are no available spectra in this nodding group.')
except:
self.messages.append('There are no available spectra in this nodding group.')
self.group.clear()
return
......@@ -732,16 +801,19 @@ class GBNodding():
def ingroup_process(self):
warnings.simplefilter('ignore', category=AstropyWarning)
self.nodding[:] = []
if self.group['noddings'] and len(self.group['noddings'])>1:
ab, heaA, heaB = self.group_avg()
acalib, fsnr, straight = self.reduce(ab,CONFIG['A_POS'], heaA)
bcalib, fsnr, straight = self.reduce(ab,CONFIG['B_POS'], heaB)
self.combine(acalib,bcalib,fsnr)
acalib = None
bcalib = None
os.remove(ab)
os.remove(straight)
else:
try:
if self.group['noddings'] and len(self.group['noddings'])>1:
ab, heaA, heaB = self.group_avg()
acalib, fsnr, straight = self.reduce(ab,CONFIG['A_POS'], heaA)
bcalib, fsnr, straight = self.reduce(ab,CONFIG['B_POS'], heaB)
self.combine(acalib,bcalib,fsnr)
acalib = None
bcalib = None
os.remove(ab)
os.remove(straight)
else:
self.messages.append('There are no available spectra in this incomplete nodding group.')
except:
self.messages.append('There are no available spectra in this incomplete nodding group.')
self.group.clear()
return
......
drslib/rawfiles.py
View file @ cd7c5785
......@@ -117,6 +117,12 @@ class GBCheckRawFiles():
self.group['ngroup'] = groupn
self.group['n'] = groupi
# if len(self.lists[CONFIG['DARK']]) > groupn*int(CONFIG['NDARK']):
# self.messages.append('A new group of %s has started, the previous one will now be reduced.' % (obstype))
# self.command[CONFIG['DARK']] = True
# self.working = 0
# return
self.lists[CONFIG['DARK']].append(self.rawfile)
self.messages.append('%s out of %s %s files observed.' % (len(self.lists[CONFIG['DARK']]),groupn*int(CONFIG['NDARK']),obstype))
......@@ -124,6 +130,7 @@ class GBCheckRawFiles():
self.command[CONFIG['DARK']] = True
self.messages.append('All expected %s have been observed. They will now be reduced.' % (obstype))
self.working = 0
return
elif obstype == CONFIG['DARK']:
......
drslib/stare.py
View file @ cd7c5785
......@@ -520,8 +520,12 @@ class GBStare():
self.messages.append('%s cosmics were removed.' % str(all_cosmics),)
redname = os.path.join(CONFIG['RED_DIR'],str(os.path.basename(fitsfile)))
calname = redname.replace('.fits','_e2ds.fits')
calname1d = redname.replace('.fits','_s1d.fits')
sfx = ''.join(('_',CONFIG['MERGED'],'.fits'))
msfx = ''.join(('_',CONFIG['UNMERGED'],'.fits'))
calname = redname.replace('.fits',msfx)
calname1d = redname.replace('.fits',sfx)
#calname = redname.replace('.fits','_e2ds.fits')
#calname1d = redname.replace('.fits','_s1d.fits')
heaspe = fits.Header(imstr.header)
......@@ -555,16 +559,24 @@ class GBStare():
for o in xrange(CONFIG['N_ORD']):
waves[o] = varie.wcalib(heaspe,o)
spefits = fits.PrimaryHDU(optSpectrum,header=heaspe)
wavefits = fits.ImageHDU(waves,name='WAVE')
snrfits = fits.ImageHDU(snr,name='SNR')
#spefits = fits.PrimaryHDU(optSpectrum,header=heaspe)
#wavefits = fits.ImageHDU(waves,name='WAVE')
#snrfits = fits.ImageHDU(snr,name='SNR')
#results = fits.HDUList([spefits,wavefits,snrfits])
#calname = os.path.join(CONFIG['RED_DIR'],calname)
#results.writeto(calname,clobber=True)
#results = fits.HDUList([spefits])
results = fits.HDUList([spefits,wavefits,snrfits])
orders=np.arange(CONFIG['N_ORD'])+32
c1 = fits.Column(name='ORDER', format='I', array=orders)
c2 = fits.Column(name='WAVE', format=''.join((str(CONFIG['YCCD']),'D')), unit='nm', array=waves)
c3 = fits.Column(name='FLUX', format=''.join((str(CONFIG['YCCD']),'D')), array=optSpectrum)
c4 = fits.Column(name='SNR', format=''.join((str(CONFIG['YCCD']),'D')), array=snr)
tbhdu = fits.BinTableHDU.from_columns([c1, c2, c3, c4],header=heaspe)
calname = os.path.join(CONFIG['RED_DIR'],calname)
results.writeto(calname,clobber=True)
tbhdu.writeto(calname,clobber=True)
if CONFIG['S1D']:
#s1d = varie.create_s1d(optSpectrum,snr,heaspe)
......
drslib/wlframes.py
View file @ cd7c5785
......@@ -105,7 +105,8 @@ class GBWls():
combinewl = ccdproc.Combiner(self.wlcorr)
# mask the pixels using ron/gain
combinewl.sigma_clipping(func=np.ma.mean, dev_func=varie.stdcombine)
#combinewl.sigma_clipping(func=np.ma.mean, dev_func=varie.stdcombine)
combinewl.sigma_clipping(func=np.ma.median, dev_func=varie.stdcombine)
#combineflat.sigma_clipping(func=np.ma.mean)
mwl = combinewl.average_combine()
......
gofioDRS.py
View file @ cd7c5785
......@@ -310,11 +310,11 @@ if __name__ == "__main__":
if not os.path.exists(rawdir):
drslogger.log('The RAW directory %s does not exist. GOFIO will now close down.' % rawdir)
drslogger.log('Pipeline stopped.')
sys.exit(0)
dbcalib.close()
if not CONFIG['OFFLINE']:
dbramps.close()
dbnight.close()
sys.exit(0)
rawlists = {CONFIG['DARK']:[],CONFIG['FLAT']:[],CONFIG['WCAL_UNE']:[],CONFIG['WCAL_FP']:[],CONFIG['SCIENCE']:[]}
......@@ -345,6 +345,8 @@ if __name__ == "__main__":
working, group, go, do_calib = GianoBreduce(raws[0], rawlists, dbcalib, working, group, do_calib, stop)
if go:
raws.pop(0)
#raws.pop(0)
else:
for n in xrange(len(raws)):
if db.check_night(dbnight,raws[n]):
......@@ -359,6 +361,12 @@ if __name__ == "__main__":
else:
time.sleep(1)
if CONFIG['OFFLINE']:
drslogger.log('No more files to reduce.')
dbcalib.close()
try: os.remove(CONFIG['MASK_C'])
except: pass
sys.exit(0)
except KeyboardInterrupt:
......@@ -377,11 +385,11 @@ if __name__ == "__main__":
try: os.remove(CONFIG['MASK_C'])
except: pass
drslogger.log('Pipeline stopped.')
sys.exit(0)
dbcalib.close()
if not CONFIG['OFFLINE']:
dbramps.close()
dbnight.close()
sys.exit(0)
observer.join()
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment