Skip to content

GitLab

Commit 357a2e8e authored by Andrea Bignamini's avatar Andrea Bignamini
Browse files

Merge develop into master

parents 4ef49468 4c5b4072
Hide whitespace changes
Inline Side-by-side
Showing with 313 additions and 25 deletions
gofio/drslib/config.py
View file @ 357a2e8e
......@@ -167,6 +167,10 @@ CONFIG['MASTERFLAT'] = (' '.join((CONFIG['KEY_DRS'],'FLAT')),'Flat-field used fo
CONFIG['MASTERLAMP'] = (' '.join((CONFIG['KEY_DRS'],'LAMP')),'Calibration lamp used for the reduction')
CONFIG['KEY_WEXT'] = (' '.join((CONFIG['KEY_DRS'],'WEXT')),'Standard extraction window [pixel]')
CONFIG['DRS_VERSION'] = (' '.join((CONFIG['KEY_DRS'],'VERSION')),'GOFIO version')
CONFIG['RED'] = (' '.join((CONFIG['KEY_DRS'],'RED')),'GOFIO reduced file')
CONFIG['RED_TYPE'] = (' '.join((CONFIG['KEY_DRS'],'RED TYPE')),'GOFIO reduced file type')
CONFIG['RED_SUBTYPE'] = (' '.join((CONFIG['KEY_DRS'],'RED SUBTYPE')),'GOFIO reduced file subtype')
CONFIG['RED_SLIT'] = (' '.join((CONFIG['KEY_DRS'],'RED SLIT')),'GOFIO reduced file slit')
# Wavelength calibration
......
gofio/drslib/darkframes.py
View file @ 357a2e8e
......@@ -15,6 +15,7 @@ from astropy.utils.exceptions import AstropyWarning
from ccdproc import CCDData, Combiner
from drslib.config import CONFIG
from drslib import db
from drslib import metadata
import numpy as np
import os, shutil
......@@ -105,6 +106,10 @@ class GBDarks():
self.masterdarks[exptime].header[CONFIG['DRS_VERSION'][0]] = (CONFIG['VERSION'], CONFIG['DRS_VERSION'][1])
# Add metadata to header
self.masterdarks[exptime].header = metadata.add_metadata(self.masterdarks[exptime].header)
hdu = fits.PrimaryHDU(data=self.masterdarks[exptime].data,header=self.masterdarks[exptime].header)
mdark = fits.HDUList([hdu])
......
gofio/drslib/flatframes.py
View file @ 357a2e8e
......@@ -21,6 +21,7 @@ from astropy.utils.exceptions import AstropyWarning
import ccdproc
from drslib.config import CONFIG
from drslib import db, varie
from drslib import metadata
import numpy as np
import math, os, subprocess, shutil
......@@ -159,6 +160,9 @@ class GBFlats():
mflat.header[CONFIG['DRS_VERSION'][0]] = (CONFIG['VERSION'], CONFIG['DRS_VERSION'][1])
# Add metadata to header
mflat.header = metadata.add_metadata(mflat.header)
hdu = fits.PrimaryHDU(data=mflat.data,header=mflat.header)
masterflat = fits.HDUList([hdu])
masterflat.writeto(nome,clobber=True)
......@@ -201,9 +205,41 @@ class GBFlats():
shiftY = varie.shiftY(mflat.data)
db.insert_dbfile(self.dbconn,'shiftY',shiftY)
# Since masterflat has been copied from calibration database
# also str and nor will be copied
self.copy_straighten()
return False
return True
def copy_straighten(self):
"""
The masterflat has been copied from calibration database
This function also copy str and nor file from calibration database
"""
# Get masterflat name from calibdb
# flat is a fullpath name
flat = db.extract_dbfile(self.dbconn,'flat')
# Generate str and nor fullpath name from calibdb
straight = flat.replace('.fits','_str.fits')
nor = flat.replace('.fits','_nor.fits')
# Insert str and nor in calibdb for this night
db.insert_dbfile(self.dbconn,'flatstr',straight)
db.insert_dbfile(self.dbconn,'flatnor',nor)
# Copy str and nor to RED_CALIB directory of current night
red_straight = os.path.join(CONFIG['RED_CALIB'],os.path.basename(straight))
shutil.copyfile(straight,red_straight)
red_nor = os.path.join(CONFIG['RED_CALIB'],os.path.basename(nor))
shutil.copyfile(nor,red_nor)
return
def straighten(self):
"""
Straighten the masterflat, normalize it and create the masked array
......@@ -245,13 +281,22 @@ class GBFlats():
#print args
subprocess.call(args)
red_straight = os.path.join(CONFIG['RED_CALIB'],os.path.basename(straight))
shutil.copyfile(straight,red_straight)
self.messages.append('Orders straightened.')
db.insert_dbfile(self.dbconn,'flatstr',straight)
mflat = ccdproc.CCDData.read(straight, unit=u.adu)
# Update FILENAME in header then
# add metadata to header and save straight file
mflat.header[CONFIG['KEYS']['FILENAME']] = os.path.basename(straight)
mflat.header = metadata.add_metadata(mflat.header)
hdu = fits.PrimaryHDU(data=mflat.data, header=mflat.header)
str_fits = fits.HDUList([hdu])
str_fits.writeto(straight, overwrite=True)
fdata = mflat.data
norflat = np.ones((CONFIG['YCCD'],CONFIG['XCCD']), dtype='float32')
......@@ -312,10 +357,20 @@ class GBFlats():
fnor = ccdproc.CCDData(norflat, unit=u.adu)
fnor.header = mflat.header
#fn = fnor.to_hdu()
hdu = fits.PrimaryHDU(data=fnor.data,header=fnor.header)
# Update FILENAME in header then
# add metadata to header and save nor file
fnor.header[CONFIG['KEYS']['FILENAME']] = os.path.basename(nor)
fnor.header = metadata.add_metadata(fnor.header)
hdu = fits.PrimaryHDU(data=fnor.data, header=fnor.header)
fn = fits.HDUList([hdu])
fn.writeto(nor,clobber=True)
fn.writeto(nor, overwrite=True)
# Copy straight and nor to RED_CALIB directory
red_straight = os.path.join(CONFIG['RED_CALIB'],os.path.basename(straight))
shutil.copyfile(straight,red_straight)
red_nor = os.path.join(CONFIG['RED_CALIB'],os.path.basename(nor))
shutil.copyfile(nor,red_nor)
......
gofio/drslib/logger.py
View file @ 357a2e8e
......@@ -5,6 +5,7 @@ Implemented in the GIANOB DRS by Monica Rainer
import logging
from drslib.config import CONFIG as CFG
from sys import stdout
class DrsLogger():
def __init__(self):
......@@ -29,7 +30,7 @@ class DrsLogger():
def addconsolelogger(self):
"""The logger to the terminal is defined"""
console_handler = logging.StreamHandler()
console_handler = logging.StreamHandler(stream=stdout)
formatter = logging.Formatter(fmt=self.conslogfmt)
console_handler.setFormatter(formatter)
self.logger.addHandler(console_handler)
......
gofio/drslib/metadata.py 0 → 100644
View file @ 357a2e8e
from drslib.config import CONFIG
def add_metadata(header):
filename = header['FILENAME']
dataDict = {'RED': True,
'RED_TYPE': None,
'RED_SUBTYPE': None,
'RED_SLIT': None}
if 'DARK' in filename:
dataDict['RED_TYPE'] = 'DARK'
if '010_DARK' in filename:
dataDict['RED_SUBTYPE'] = 'masterdark10'
elif '030_DARK' in filename:
dataDict['RED_SUBTYPE'] = 'masterdark30'
elif '060_DARK' in filename:
dataDict['RED_SUBTYPE'] = 'masterdark60'
elif '100_DARK' in filename:
dataDict['RED_SUBTYPE'] = 'masterdark100'
elif '200_DARK' in filename:
dataDict['RED_SUBTYPE'] = 'masterdark200'
elif '300_DARK' in filename:
dataDict['RED_SUBTYPE'] = 'masterdark300'
elif '600_DARK' in filename:
dataDict['RED_SUBTYPE'] = 'masterdark600'
else:
dataDict['RED_SUBTYPE'] = 'masterdark'
elif 'FLAT' in filename:
dataDict['RED_TYPE'] = 'FLAT'
if 'FLAT_nor' in filename:
dataDict['RED_SUBTYPE'] = 'masterflat_nor'
elif 'FLAT_str' in filename:
dataDict['RED_SUBTYPE'] = 'masterflat_str'
else:
dataDict['RED_SUBTYPE'] = 'masterflat'
elif 'UNE' in filename:
dataDict['RED_TYPE'] = 'UNE'
if 'UNE_ext' in filename:
dataDict['RED_SUBTYPE'] = 'wl_une_calib'
elif 'UNE_str' in filename:
dataDict['RED_SUBTYPE'] = 'wl_une_str'
else:
dataDict['RED_SUBTYPE'] = 'wl_une'
elif 'FP' in filename:
dataDict['RED_TYPE'] = 'FP'
if 'FP_ext' in filename:
dataDict['RED_SUBTYPE'] = 'wl_une_calib'
elif 'FP_str' in filename:
dataDict['RED_SUBTYPE'] = 'wl_une_str'
else:
dataDict['RED_SUBTYPE'] = 'wl_une'
elif 'str' in filename:
dataDict['RED_TYPE'] = 'INTERMEDIATE'
dataDict['RED_SUBTYPE'] = None
else:
dataDict['RED_TYPE'] = 'SCIENCE'
if 'ms1d' in filename:
dataDict['RED_SUBTYPE'] = 'ms1d'
else:
dataDict['RED_SUBTYPE'] = 's1d'
if '_AB_' in filename:
dataDict['RED_SLIT'] = 'AB'
elif '_A_' in filename:
dataDict['RED_SLIT'] = 'A'
elif '_B_' in filename:
dataDict['RED_SLIT'] = 'B'
# Insert in only keys with not None values
for key in dataDict.keys():
if dataDict[key]:
header[CONFIG[key][0]] = (dataDict[key], CONFIG[key][1])
# header[CONFIG['RED_TYPE'][0]] = (dataDict['RED_TYPE'], CONFIG['RED_TYPE'][1])
# header[CONFIG['RED_SUBTYPE'][0]] = (dataDict['RED_SUBTYPE'], CONFIG['RED_SUBTYPE'][1])
# header[CONFIG['RED_SLIT'][0]] = (dataDict['RED_SLIT'], CONFIG['RED_SLIT'][1])
return header
\ No newline at end of file
gofio/drslib/nodding.py
View file @ 357a2e8e
......@@ -21,6 +21,7 @@ Reduction of the nodding group:
from drslib.config import CONFIG
from drslib import db, varie
from drslib import metadata
from astropy import units as u
from astropy.io import ascii, fits
......@@ -82,14 +83,10 @@ class GBNodding():
return True
def createAB(self):
def create_nodcorr(self):
"""
Create nodding images A-B and save them in temporary directory if
the keyword SPEXTMODE is set to GRPAVG_EXT.
Create nodcorr
"""
hea = self.nodlist[0].header
badpix = ccdproc.CCDData.read(CONFIG['BADPIX_MASK'], unit=u.adu)
bad_mask=badpix.data
inverse_mask=np.logical_not(bad_mask)
......@@ -106,6 +103,46 @@ class GBNodding():
self.nodcorr[nod.header[CONFIG['KEYS']['SLIT']]] = nod_corr
nod = None
return
def check_nodcorr(self):
"""
Check if nodcorr is not corrupted
"""
try:
nodA = self.nodcorr[CONFIG['A']]
except Exception as e:
nodA = None
try:
nodB = self.nodcorr[CONFIG['B']]
except Exception as e:
nodB = None
if not nodA and not nodB:
self.messages.append('The nodding pair is corrupted. Both slits A and B are missing. Skipping pair processing.')
return False
else:
if not nodA:
self.messages.append('The nodding pair is corrupted. Slit A is missing. Skipping pair processing.')
return False
if not nodB:
self.messages.append('The nodding pair is corrupted. Slit B is missing. Skipping pair processing.')
return False
return True
def createAB(self):
"""
Create nodding images A-B and save them in temporary directory if
the keyword SPEXTMODE is set to GRPAVG_EXT.
"""
hea = self.nodlist[0].header
# Create A-B image
nodA = self.nodcorr[CONFIG['A']]
......@@ -215,13 +252,24 @@ class GBNodding():
subprocess.call(args)
# Read straight file
imstr = ccdproc.CCDData.read(straight, unit=u.adu)
# Update FILENAME in header then
# add metadata to header and save straight file
imstr.header[CONFIG['KEYS']['FILENAME']] = os.path.basename(straight)
imstr.header = metadata.add_metadata(imstr.header)
hdu = fits.PrimaryHDU(data=imstr.data, header=imstr.header)
str_fits = fits.HDUList([hdu])
str_fits.writeto(straight, overwrite=True)
str_file = os.path.join(CONFIG['RED_STR'],os.path.basename(straight))
try: shutil.copyfile(straight,str_file)
except: pass
self.messages.append('%s: orders straightened (nodding %s).' % (str(os.path.basename(fitsfile)),slit,))
imstr = ccdproc.CCDData.read(straight, unit=u.adu)
hea_ima = hea
for key in CONFIG['STRAIGHT_PAR']:
hea_ima[CONFIG['STRAIGHT_PAR'][key]] = imstr.header[CONFIG['STRAIGHT_PAR'][key]]
......@@ -576,6 +624,9 @@ class GBNodding():
heaspe[CONFIG['DRS_VERSION'][0]] = (CONFIG['VERSION'], CONFIG['DRS_VERSION'][1])
# Add metadata to header
heaspe = metadata.add_metadata(heaspe)
#tbhdu = fits.BinTableHDU.from_columns([c1, c2, c3, c4],header=heaspe)
tbhdu = fits.BinTableHDU.from_columns([c1, c2, c3, c4])
prihdu = fits.PrimaryHDU(data=None, header=heaspe)
......@@ -604,6 +655,9 @@ class GBNodding():
heaspe['CTYPE1'] = ('Nanometers','Units of coordinates')
heaspe['BUNIT'] = ('Relative Flux','Units of data values')
# Add metadata to header
heaspe = metadata.add_metadata(heaspe)
s1dfits = fits.PrimaryHDU(s1d,header=heaspe)
calname1d = os.path.join(CONFIG['RED_DIR'],calname1d)
s1dfits.writeto(calname1d,clobber=True)
......@@ -779,6 +833,9 @@ class GBNodding():
heaspe[CONFIG['DRS_VERSION'][0]] = (CONFIG['VERSION'], CONFIG['DRS_VERSION'][1])
# Add metadata to header
heaspe = metadata.add_metadata(heaspe)
#tbhdu = fits.BinTableHDU.from_columns([c1, c2, c3, c4],header=heaspe)
tbhdu = fits.BinTableHDU.from_columns([c1, c2, c3, c4])
prihdu = fits.PrimaryHDU(data=None, header=heaspe)
......@@ -803,6 +860,9 @@ class GBNodding():
heaspe['CTYPE1'] = ('Nanometers','Units of coordinates')
heaspe['BUNIT'] = ('Relative Flux','Units of data values')
# Add metadata to header
heaspe = metadata.add_metadata(heaspe)
s1dfits = fits.PrimaryHDU(s1d,header=heaspe)
s1dfits.writeto(abnome1d,clobber=True)
......@@ -910,6 +970,10 @@ class GBNodding():
warnings.simplefilter('ignore', category=AstropyWarning)
if self.qualitycheck():
#t1 = time.time()
# Create and check nodcorr
self.create_nodcorr()
if not self.check_nodcorr():
return
ab, heaA, heaB = self.createAB()
#t2 = time.time()
#print 'Bad pixels, create nodding: %s s' % str(t2-t1)
......
gofio/drslib/stare.py
View file @ 357a2e8e
......@@ -30,6 +30,7 @@ Reduction of all the Obj-Sky group:
from drslib.config import CONFIG
from drslib import db, varie
from drslib import metadata
from astropy import units as u
from astropy.io import ascii, fits
......@@ -386,15 +387,25 @@ class GBStare():
args.extend(shiftY)
subprocess.call(args)
# Read straight file
imstr = ccdproc.CCDData.read(straight, unit=u.adu)
# Update FILENAME in header then
# add metadata to header and save straight file
imstr.header[CONFIG['KEYS']['FILENAME']] = os.path.basename(straight)
imstr.header = metadata.add_metadata(imstr.header)
hdu = fits.PrimaryHDU(data=imstr.data, header=imstr.header)
str_fits = fits.HDUList([hdu])
str_fits.writeto(straight, overwrite=True)
str_file = os.path.join(CONFIG['RED_STR'],os.path.basename(straight))
try: shutil.copyfile(straight,str_file)
except: pass
self.messages.append('%s: orders straightened.' % str(os.path.basename(fitsfile)),)
imstr = ccdproc.CCDData.read(straight, unit=u.adu)
imflat = imstr.data.copy()
try: nspec = imstr.header[CONFIG['KEYS']['NCOMBINE']]
except: nspec = 1
......@@ -701,6 +712,9 @@ class GBStare():
heaspe[CONFIG['DRS_VERSION'][0]] = (CONFIG['VERSION'], CONFIG['DRS_VERSION'][1])
# Add metadata to header
heaspe = metadata.add_metadata(heaspe)
#tbhdu = fits.BinTableHDU.from_columns([c1, c2, c3, c4],header=heaspe)
tbhdu = fits.BinTableHDU.from_columns([c1, c2, c3, c4])
prihdu = fits.PrimaryHDU(data=None, header=heaspe)
......@@ -726,6 +740,9 @@ class GBStare():
heaspe['CTYPE1'] = ('Nanometers','Units of coordinates')
heaspe['BUNIT'] = ('Relative Flux','Units of data values')
# Add metadata to header
heaspe = metadata.add_metadata(heaspe)
s1dfits = fits.PrimaryHDU(s1d,header=heaspe)
calname1d = os.path.join(CONFIG['RED_DIR'],calname1d)
s1dfits.writeto(calname1d,clobber=True)
......
gofio/drslib/wlframes.py
View file @ 357a2e8e
......@@ -23,6 +23,7 @@ from astropy.utils.exceptions import AstropyWarning
import ccdproc
from drslib.config import CONFIG
from drslib import db, varie
from drslib import metadata
import numpy as np
import math, os, subprocess, shutil
......@@ -141,6 +142,9 @@ class GBWls():
mwl.header[CONFIG['DRS_VERSION'][0]] = (CONFIG['VERSION'], CONFIG['DRS_VERSION'][1])
# Add metadata to header
mwl.header = metadata.add_metadata(mwl.header)
hdu = fits.PrimaryHDU(data=mwl.data,header=mwl.header)
masterwl = fits.HDUList([hdu])
masterwl.writeto(self.mlamp,clobber=True)
......@@ -160,9 +164,45 @@ class GBWls():
except:
self.messages.append('There are no %s lamps in the calibration database.' %(unefp))
return False
# Since masterlamp has been copied from calibration database
# also str and extr will be copied
self.copy_straighten(unefp)
return False
return True
def copy_straighten(self,unefp):
"""
The masterlamp has been copied from calibration database
This function also copy str and extr file from calibration database
"""
# Get masterlamp name from calibdb
# lamp is a fullpath name
lamp = self.mlamp
# Generate str and extr fullpath name from calibdb
straight = lamp.replace('.fits','_str.fits')
extracted = lamp.replace('.fits','_ext.fits')
# Insert str and extr in calibdb for this night
strname = '_'.join((unefp,'str'))
db.insert_dbfile(self.dbconn,strname,straight)
calname = '_'.join((unefp,'calib'))
db.insert_dbfile(self.dbconn,calname,extracted)
# Copy straight and extracted to RED_CALIB directory
red_straight = os.path.join(CONFIG['RED_CALIB'],os.path.basename(straight))
shutil.copyfile(straight,red_straight)
red_extr = os.path.join(CONFIG['RED_CALIB'],os.path.basename(extracted))
shutil.copyfile(extracted,red_extr)
return
def extract(self,unefp):
# straighten the lamp's orders
# lamp = db.extract_dbfile(self.dbconn,unefp)
......@@ -206,9 +246,6 @@ class GBWls():
subprocess.call(args)
red_straight = os.path.join(CONFIG['RED_CALIB'],os.path.basename(straight))
shutil.copyfile(straight,red_straight)
#self.messages.append('%s: orders straightened.' % (str(os.path.basename(lamp))))
#strname = '_'.join((unefp,'str'))
......@@ -217,6 +254,16 @@ class GBWls():
wl = ccdproc.CCDData.read(straight, unit=u.adu)
# Update FILENAME in header then
# add metadata to header and save straight file
wl.header[CONFIG['KEYS']['FILENAME']] = os.path.basename(straight)
wl.header = metadata.add_metadata(wl.header)
hdu = fits.PrimaryHDU(data=wl.data, header=wl.header)
str_fits = fits.HDUList([hdu])
str_fits.writeto(straight, overwrite=True)
wldata = wl.data
wl.header[CONFIG['WLFIT'][0]] = (CONFIG['WLFIT_FUNC'],CONFIG['WLFIT'][1])
......@@ -307,14 +354,21 @@ class GBWls():
wlextract.header = wl.header
wlextract.header[CONFIG['KEY_WEXT'][0]] = (CONFIG['WEXT']*2,CONFIG['KEY_WEXT'][1])
hdu = fits.PrimaryHDU(data=wlextract.data,header=wlextract.header)
# Update FILENAME in header then
# add metadata to header and save extracted file
wlextract.header[CONFIG['KEYS']['FILENAME']] = os.path.basename(extracted)
wlextract.header = metadata.add_metadata(wlextract.header)
hdu = fits.PrimaryHDU(data=wlextract.data, header=wlextract.header)
wlextr = fits.HDUList([hdu])
wlextr.writeto(extracted,clobber=True)
wlextr.writeto(extracted, overwrite=True)
# Copy straight and extracted to RED_CALIB directory
red_straight = os.path.join(CONFIG['RED_CALIB'],os.path.basename(straight))
shutil.copyfile(straight,red_straight)
red_extr = os.path.join(CONFIG['RED_CALIB'],os.path.basename(extracted))
shutil.copyfile(extracted,red_extr)
if self.quality[0] == 'OK':
db.insert_dbfile(self.dbconn,unefp,self.mlamp)
self.messages.append('%s masterlamp %s was created and inserted in the calibration database.' % (unefp,str(os.path.basename(self.mlamp))))
......
gofio/gofioDRS.py
View file @ 357a2e8e
......@@ -78,9 +78,11 @@ def run_gofio(CONFIG, rawlists, dbcalib, working, group, stop, drslogger, dbnigh
self.process(event)
# use watchdog to check when new files are created
observer = Observer()
observer.schedule(UpdateRawList(),rawdir)
observer.start()
if not CONFIG['OFFLINE']:
observer = Observer()
observer.schedule(UpdateRawList(),rawdir)
observer.start()
try:
while True:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment