Commit 0d402123 authored by Avet Harutyunyan's avatar Avet Harutyunyan
Browse files

- gofioDRS.pt is modified to avoid a problem with "raws" variable in online...

- gofioDRS.pt is modified to avoid a problem with "raws" variable in online mode (crashing due to not declared "raws"). However, this is just a workaroud, a better solution has to be found.

- nodding.py and stare.py are modified to write the full FITS header contents into the primary hdu (hdu[0], instead of hdu[1], as it is in the previous version. Needed for a better compatibility with the archiving.

- nodding.py and stare.py are modified in order to write a correct filename into IMANAME and FILENAME header cards.
parent 2de7aff1
......@@ -456,12 +456,13 @@ class GBNodding():
calname = '_'.join((nomebase,msfx))
calname1d = '_'.join((nomebase,sfx))
#calname = str(os.path.basename(fitsfile)).replace('_AB','_B')
else:
else:
print 'Wrong slit position!'
heaspe = fits.Header(hea_ima)
heaspe[CONFIG['KEYS']['FILENAME']] = calname
heaspe[CONFIG['KEYS']['IMANAME']] = calname
drs_mjd = float(heaspe[CONFIG['KEYS']['MJD']]) + (float(heaspe[CONFIG['KEYS']['EXPTIME']])/(2*86400))
heaspe[CONFIG['DRS_MJD'][0]] = (drs_mjd,CONFIG['DRS_MJD'][1])
......@@ -517,13 +518,14 @@ class GBNodding():
orders=np.arange(CONFIG['N_ORD'])+32
c1 = fits.Column(name='ORDER', format='I', array=orders)
c2 = fits.Column(name='WAVE', format=''.join((str(CONFIG['YCCD']),'D')), unit='nm', array=waves)
c3 = fits.Column(name='FLUX', format=''.join((str(CONFIG['YCCD']),'D')), array=optSpectrum)
c3 = fits.Column(name='FLUX', format=''.join((str(CONFIG['YCCD']),'D')), unit='ADU', array=optSpectrum)
c4 = fits.Column(name='SNR', format=''.join((str(CONFIG['YCCD']),'D')), array=snr)
tbhdu = fits.BinTableHDU.from_columns([c1, c2, c3, c4],header=heaspe)
tbhdu = fits.BinTableHDU.from_columns([c1, c2, c3, c4])
calname = os.path.join(CONFIG['RED_DIR'],calname)
tbhdu.writeto(calname,clobber=True)
prihdu = fits.PrimaryHDU(data=None, header=heaspe)
hdulist_out = fits.HDUList([prihdu, tbhdu])
hdulist_out.writeto(calname, overwrite=True)
#t1 = time.time()
......@@ -536,6 +538,7 @@ class GBNodding():
#s1d = varie.create_s1d(optSpectrum,snr,heaspe)
s1d, startval = varie.create_s1d(optSpectrum,heaspe)
heaspe[CONFIG['KEYS']['FILENAME']] = calname1d
heaspe[CONFIG['KEYS']['IMANAME']] = calname1d
heaspe['CRPIX1'] = (1.,'Reference pixel')
heaspe['CRVAL1'] = (startval,'Coordinates at reference pixel')
heaspe['CDELT1'] = (CONFIG['S1D_STEP'],'Coordinates increment per pixel')
......@@ -599,9 +602,9 @@ class GBNodding():
afluxes = adata.field(2)
#print afluxes
#print afluxes.shape
awaves = acal[1].header
roneff = math.sqrt(2)*acal[1].header[CONFIG['RON_EFF'][0]]
gaineff = 2*acal[1].header[CONFIG['GAIN_EFF'][0]]
awaves = acal[0].header
roneff = math.sqrt(2)*acal[0].header[CONFIG['RON_EFF'][0]]
gaineff = 2*acal[0].header[CONFIG['GAIN_EFF'][0]]
bcal = fits.open(bcalib)
......@@ -609,7 +612,7 @@ class GBNodding():
#bwaves = bcal[0].header
bdata = bcal[1].data
bfluxes = bdata.field(2)
bwaves = bcal[1].header
bwaves = bcal[0].header
abcalib = np.zeros((CONFIG['N_ORD'],CONFIG['YCCD']))
......@@ -626,7 +629,7 @@ class GBNodding():
abcalib = np.asarray(abcalib, dtype='float32')
#abhea = acal[0].header
abhea = acal[1].header
abhea = acal[0].header
abhea[CONFIG['RON_EFF'][0]] = (roneff,CONFIG['RON_EFF'][1])
abhea[CONFIG['GAIN_EFF'][0]] = (gaineff,CONFIG['GAIN_EFF'][1])
#abhea[CONFIG['TEXP_EFF'][0]] = (,CONFIG['TEXP_EFF'][1])
......@@ -656,15 +659,15 @@ class GBNodding():
try:
#am_a = acal[0].header[CONFIG['AIRMASS'][0]]
#am_b = bcal[0].header[CONFIG['AIRMASS'][0]]
am_a = acal[1].header[CONFIG['AIRMASS'][0]]
am_b = bcal[1].header[CONFIG['AIRMASS'][0]]
am_a = acal[0].header[CONFIG['AIRMASS'][0]]
am_b = bcal[0].header[CONFIG['AIRMASS'][0]]
#am_a = acal[1].header[CONFIG['AIRMASS'][0]]
#am_b = bcal[1].header[CONFIG['AIRMASS'][0]]
except:
#am_a = acal[0].header[CONFIG['KEYS']['AM']]
#am_b = bcal[0].header[CONFIG['KEYS']['AM']]
am_a = acal[1].header[CONFIG['KEYS']['AM']]
am_b = bcal[1].header[CONFIG['KEYS']['AM']]
am_a = acal[0].header[CONFIG['KEYS']['AM']]
am_b = bcal[0].header[CONFIG['KEYS']['AM']]
#am_a = acal[1].header[CONFIG['KEYS']['AM']]
#am_b = bcal[1].header[CONFIG['KEYS']['AM']]
am = (am_a+am_b)/2.0
......@@ -702,12 +705,15 @@ class GBNodding():
orders=np.arange(CONFIG['N_ORD'])+32
c1 = fits.Column(name='ORDER', format='I', array=orders)
c2 = fits.Column(name='WAVE', format=''.join((str(CONFIG['YCCD']),'D')), unit='nm', array=waves)
c3 = fits.Column(name='FLUX', format=''.join((str(CONFIG['YCCD']),'D')), array=abcalib)
c3 = fits.Column(name='FLUX', format=''.join((str(CONFIG['YCCD']),'D')), unit='ADU', array=abcalib)
c4 = fits.Column(name='SNR', format=''.join((str(CONFIG['YCCD']),'D')), array=snr)
tbhdu = fits.BinTableHDU.from_columns([c1, c2, c3, c4],header=heaspe)
tbhdu.writeto(abnome,clobber=True)
tbhdu = fits.BinTableHDU.from_columns([c1, c2, c3, c4])
heaspe[CONFIG['KEYS']['IMANAME']] = os.path.split(abnome)[1]
heaspe[CONFIG['KEYS']['FILENAME']] = os.path.split(abnome)[1]
prihdu = fits.PrimaryHDU(data=None, header=heaspe)
hdulist_out = fits.HDUList([prihdu, tbhdu])
hdulist_out.writeto(abnome, overwrite=True)
try: obj_name = heaspe[CONFIG['KEYS']['OBJECT']]
......@@ -724,6 +730,9 @@ class GBNodding():
heaspe['CTYPE1'] = ('Nanometers','Units of coordinates')
heaspe['BUNIT'] = ('Relative Flux','Units of data values')
heaspe[CONFIG['KEYS']['IMANAME']] = os.path.split(abnome1d)[1]
heaspe[CONFIG['KEYS']['FILENAME']] = os.path.split(abnome1d)[1]
s1dfits = fits.PrimaryHDU(s1d,header=heaspe)
s1dfits.writeto(abnome1d,clobber=True)
......@@ -903,6 +912,3 @@ class GBNodding():
self.messages.append('There are no available spectra in this incomplete nodding group.')
self.group.clear()
return
......@@ -144,7 +144,7 @@ class GBStare():
#print len(self.starelist)
self.mjd = np.average(np.asarray(mjd_obj)) + (exp_common/(2*86400))
sky_time = abs(np.asarray(sky_time) - self.mjd)
sky_time = abs(np.asarray(sky_time) - self.mjd)
# Skip sky images with exposure time different than Obj
#print len(self.skylist)
......@@ -610,13 +610,17 @@ class GBStare():
orders=np.arange(CONFIG['N_ORD'])+32
c1 = fits.Column(name='ORDER', format='I', array=orders)
c2 = fits.Column(name='WAVE', format=''.join((str(CONFIG['YCCD']),'D')), unit='nm', array=waves)
c3 = fits.Column(name='FLUX', format=''.join((str(CONFIG['YCCD']),'D')), array=optSpectrum)
c3 = fits.Column(name='FLUX', format=''.join((str(CONFIG['YCCD']),'D')), unit='ADU', array=optSpectrum)
c4 = fits.Column(name='SNR', format=''.join((str(CONFIG['YCCD']),'D')), array=snr)
tbhdu = fits.BinTableHDU.from_columns([c1, c2, c3, c4],header=heaspe)
tbhdu = fits.BinTableHDU.from_columns([c1, c2, c3, c4])
heaspe[CONFIG['KEYS']['IMANAME']] = calname
heaspe[CONFIG['KEYS']['FILENAME']] = calname
calname = os.path.join(CONFIG['RED_DIR'],calname)
tbhdu.writeto(calname,clobber=True)
prihdu = fits.PrimaryHDU(data=None, header=heaspe)
hdulist_out = fits.HDUList([prihdu, tbhdu])
hdulist_out.writeto(calname, overwrite=True)
try: obj_name = heaspe[CONFIG['KEYS']['OBJECT']]
except: obj_name = 'NONE'
......@@ -633,6 +637,8 @@ class GBStare():
heaspe['BUNIT'] = ('Relative Flux','Units of data values')
s1dfits = fits.PrimaryHDU(s1d,header=heaspe)
heaspe[CONFIG['KEYS']['FILENAME']] = calname1d
heaspe[CONFIG['KEYS']['IMANAME']] = calname1d
calname1d = os.path.join(CONFIG['RED_DIR'],calname1d)
s1dfits.writeto(calname1d,clobber=True)
rid = varie.random_id(12)
......@@ -644,7 +650,7 @@ class GBStare():
if imstr.header[CONFIG['KEYS']['EXTMODE']] == CONFIG['EXTPAIR']:
return calname, straight, dbreduced
return calname, straight, dbreduced
elif 'grp' in fitsfile:
return calname, straight, dbreduced
......@@ -707,5 +713,3 @@ class GBStare():
self.group.clear()
return
"""
GOFIO DRS
Written by Monica Rainer
Usage:
gofioDRS.py -h
gofioDRS.py [-g=<gofio_config>]
gofioDRS.py <date> [--dark --flat --une --fp --all_calib --only_calib --use_flat=<flag> --s1d=<s1d>]
gofioDRS.py <date> <calib_date> [--dark --flat --une --fp --all_calib --only_calib --use_flat=<flag> --s1d=<s1d>]
Options:
-h,--help : show this screen
-g,--cfg : path of the configuration file (optional)
date : date to be reduced, no configuration file possible
(always first input)
calib_date : calibration date to be used (always second input)
--dark : darks are reduced
--flat : flats are reduced
--une : U-Ne lamps are reduced
--fp : FP lamps are reduced
--all_calib : all the calibrations are reduced
--only_calib : only calibrations are reduced, no science images
(the calibrations to be reduced must be defined
using the above parameters)
--use_flat=<flag> : flat-field is removed,
options are global/order/nor/none [Default: order]
--s1d=<s1d> : s1d outputs are created, either normalized or not
options are yes/no/norm [Default: yes]
"""
import os, glob, sys, time
from docopt import docopt
from drslib.logger import DrsLogger
from drslib import db, gofio
# use watchdog to monitor file creation
from watchdog.observers import Observer
from watchdog.events import PatternMatchingEventHandler
# --------------------------------
# read docopt args
# --------------------------------
docopt_args = docopt(__doc__)
raws = []
# watchdog append fits files to the raw list
class UpdateRawList(PatternMatchingEventHandler):
def process(self, event):
raws.append(event.src_path)
drslogger.log('File %s has arrived.' % (event.src_path))
def on_created(self, event):
ext = os.path.splitext(event.src_path)[1]
if ext == '.fts':
self.process(event)
def run_gofio(CONFIG, rawlists, dbcalib, working, group, stop, drslogger, dbnight, dbramps):
do_calib = CONFIG['DO_CALIB']
rawdir = CONFIG['RAW_DIR']
# check if there are already files in RAW_DIR, create list raws in temporal order
fits = os.path.join(rawdir,"*.fts")
rawfiles = glob.glob(fits)
rawfiles.sort()
for file in rawfiles:
raws.append(file)
# use watchdog to check when new files are created
observer = Observer()
observer.schedule(UpdateRawList(),rawdir)
observer.start()
try:
while True:
time.sleep(1)
# the raw files are processed
while raws:
if CONFIG['OFFLINE']:
working, group, go, do_calib, rawlists = gofio.GianoBreduce(raws[0], rawlists, dbcalib, working, group, do_calib, stop, drslogger, dbnight)
if go:
raws.pop(0)
#raws.pop(0)
else:
for n in xrange(len(raws)):
if db.check_night(dbnight,os.path.basename(raws[n])):
raws.pop(n)
break
#elif db.check_raw(dbramps,raws[n]):
elif db.check_raw(dbramps,os.path.basename(raws[n])):
working, group, go, do_calib, rawlists = gofio.GianoBreduce(raws[n], rawlists, dbcalib, working, group, do_calib, stop, drslogger, dbnight)
if go:
raws.pop(n)
break
else:
time.sleep(1)
if CONFIG['OFFLINE']:
drslogger.log('No more files to reduce.')
dbcalib.close()
try: os.remove(CONFIG['MASK_C'])
except: pass
sys.exit(0)
except KeyboardInterrupt:
observer.stop()
# reduce any file still pending!
if working:
drslogger.log('Processing the last pending files (if any).')
stop = True
if CONFIG['OFFLINE']:
gofio.GianoBreduce(raws[0], rawlists, dbcalib, working, group, do_calib, stop, drslogger, dbnight)
else:
for n in xrange(len(raws)):
if db.check_raw(dbramps,raws[n]):
gofio.GianoBreduce(raws[n], rawlists, dbcalib, working, group, do_calib, stop, drslogger, dbnight)
break
try: os.remove(CONFIG['MASK_C'])
except: pass
drslogger.log('Pipeline stopped.')
dbcalib.close()
if not CONFIG['OFFLINE']:
dbramps.close()
dbnight.close()
sys.exit(0)
observer.join()
if __name__ == "__main__":
CONFIG, rawlists, dbcalib, working, group, stop, drslogger, dbnight, dbramps = gofio.gofio_init(docopt_args)
run_gofio(CONFIG, rawlists, dbcalib, working, group, stop, drslogger, dbnight, dbramps)
gofio/gofioDRS.py
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment