Skip to content

GitLab

Commit 6e1e7801 authored by Monica Rainer's avatar Monica Rainer
Browse files

Add calib DB, fix bug in straighten orders

parent 8662a3a9
Hide whitespace changes
Inline Side-by-side
Showing with 129 additions and 39 deletions
drslib/config.py
View file @ 6e1e7801
......@@ -89,7 +89,8 @@ CONFIG['BADPIX_MASK'] = os.path.join(CONFIG['RES_DIR'], 'badpix_mask.fits')
CONFIG['LOGGER'] = {'TERM':True,'FILE':True} # logger writes to terminal and/or to file
CONFIG['LOG_FILE'] = os.path.join(CONFIG['RED_DIR'], 'drs.log')
CONFIG['STRAIGHT'] = os.path.join(CONFIG['RES_DIR'],'straight_giano_2D')
CONFIG['STRAIGHT_OPT'] = 'I=1'
CONFIG['STRAIGHT_OPT'] = ['I=1']
#CONFIG['STRAIGHT_OPT'] = ['I=1','DY=-5']
CONFIG['BASE_CALIB_DIR'] = os.path.join(CONFIG['APP_DIR'], 'calibrations')
CONFIG['CALIB_DIR'] = os.path.join(CONFIG['BASE_CALIB_DIR'], 'calfiles')
CONFIG['CALIB_DB_DIR'] = os.path.join(CONFIG['BASE_CALIB_DIR'], 'database')
......@@ -196,14 +197,16 @@ CONFIG['DB_OFFLINE_PATH'] = os.path.join(CONFIG['WEBUI_DB_DIR'], CONFIG['DB_OFFL
CONFIG['DB_2D_TBL'] = 'spec2dfiles'
CONFIG['DB_1D_TBL'] = 'spec1dfiles'
CONFIG['DB_CAL_TBL'] = 'calib'
CONFIG['DB_2D_COLS'] = OrderedDict([('names', 'spec_2d_names'), ('stamp', 'process_timestamp')])
CONFIG['DB_1D_COLS'] = OrderedDict([('path', 'spec_1d_path'), ('stamp', 'process_timestamp'), ('id', 'spec_1d_id'), ('type', 'spec_1d_type'), ('stat', 'arch_stat'), ('name', 'objname'), ('slit', 'slitpos'), ('snry', 'snr1'), ('snrj', 'snr2'), ('snrh', 'snr3'), ('snrk', 'snr4')])
CONFIG['DB_CAL_COLS'] = OrderedDict([('type', 'calib_type'), ('names', 'calib_names'), ('status', 'calib_status'), ('stamp', 'process_timestamp')])
CONFIG['DB_2D_DATATYPE'] = {'names': 'TEXT unique', 'stamp': 'REAL'}
CONFIG['DB_1D_DATATYPE'] = {'path': 'TEXT unique', 'stamp': 'REAL', 'id': 'TEXT', 'type': 'TEXT', 'stat': 'INTEGER', 'name': 'TEXT', 'slit': 'TEXT', 'snry': 'REAL', 'snrj': 'REAL', 'snrh': 'REAL', 'snrk': 'REAL'}
CONFIG['DB_CAL_DATATYPE'] = {'type': 'TEXT', 'names': 'TEXT unique', 'status': 'TEXT', 'stamp': 'REAL'}
CONFIG['DB_NIGHT'] = CONFIG['DB_ONLINE']
......
drslib/darkframes.py
View file @ 6e1e7801
......@@ -35,6 +35,7 @@ class GBDarks():
exptime = int(dark.header[CONFIG['KEYS']['EXPTIME']])
if exptime not in self.darklists.keys():
self.messages.append('Dark frame %s has exposure time %s sec: skipped.' % (str(os.path.basename(frame)),str(dark.header[CONFIG['KEYS']['EXPTIME']]),))
self.quality.append('FAILED')
continue
# check the four dark quadrants, compute mean and std
......@@ -68,9 +69,11 @@ class GBDarks():
if (np.asarray(mean) > qcheck).any():
self.messages.append('Dark frame %s failed quality check' % (str(os.path.basename(frame))),)
dark = None
self.quality.append('FAILED')
else:
self.darklists[exptime].append(dark)
self.quality.append('OK')
#print frame
#print 'Quadrant 1: mean ' + str(mean[0])
......
drslib/db.py
View file @ 6e1e7801
......@@ -43,6 +43,13 @@ def create_db_night():
",".join([CONFIG['DB_1D_COLS'][colname] + ' ' + CONFIG['DB_1D_DATATYPE'][colname] for colname in CONFIG['DB_1D_COLS']]) + \
')')
cursor.execute("CREATE TABLE IF NOT EXISTS " + \
CONFIG['DB_CAL_TBL'] + \
"(" + \
",".join([CONFIG['DB_CAL_COLS'][colname] + ' ' + CONFIG['DB_CAL_DATATYPE'][colname] for colname in CONFIG['DB_CAL_COLS']]) + \
')')
dbcalib.commit()
return dbcalib
......@@ -170,7 +177,8 @@ def check_raw(dbramps,rawfile):
cursor = dbramps.cursor()
try:
cursor.execute("SELECT * FROM {table} WHERE {fpath}=?"\
#cursor.execute("SELECT * FROM {table} WHERE {fpath}=?"\
cursor.execute("SELECT * FROM {table} WHERE instr({fpath},?) > 0"\
.format(table=CONFIG['DB_RAMP_TBL'],fpath=CONFIG['DB_RAMP_COLS']['fpath']),(rawfile,))
filename = cursor.fetchone()[0]
if filename:
......@@ -199,26 +207,37 @@ def check_night(dbnight,rawfile):
"""
cursor = dbnight.cursor()
try:
cursor.execute("SELECT * FROM {table} WHERE instr({fpath},?) > 0"\
.format(table=CONFIG['DB_CAL_TBL'],fpath=CONFIG['DB_CAL_COLS']['names']),(rawfile,))
fstamp = cursor.fetchone()[0]
if fstamp:
return True
except:
pass
try:
cursor.execute("SELECT {stamp} FROM {table} WHERE instr({fpath},?) > 0"\
.format(stamp=CONFIG['DB_2D_COLS']['stamp'],table=CONFIG['DB_2D_TBL'],fpath=CONFIG['DB_2D_COLS']['names']),(rawfile,))
stamp = cursor.fetchone()[0]
if stamp:
fstamp = cursor.fetchone()[0]
if fstamp:
cursor.execute("SELECT * FROM {table} WHERE {stamp}=?"\
.format(table=CONFIG['DB_1D_TBL'],stamp=CONFIG['DB_1D_COLS']['stamp']),(stamp,))
.format(table=CONFIG['DB_1D_TBL'],stamp=CONFIG['DB_1D_COLS']['stamp']),(fstamp,))
reduced = cursor.fetchone()[0]
if reduced:
return True
else:
return False
else:
return False
except:
return False
pass
return False
def insert_dbnight(dbnight, reduced, stamp):
#reduced = ','.join(map(os.path.basename,reduced))
reduced = ','.join(map(str,reduced))
cursor = dbnight.cursor()
cursor.execute("UPDATE {table} SET {fpath}=?, {fstamp}=? WHERE instr({fpath},?) > 0"\
......@@ -233,8 +252,21 @@ def insert_dbreduced(dbnight, reduced, stamp):
cursor = dbnight.cursor()
cursor.execute("INSERT OR IGNORE INTO {table} ({path},{stamp},{rid},{ftype},{stat},{name},{slit},{snry},{snrj},{snrh},{snrk}) VALUES (?,?,?,?,?,?,?,?,?,?,?)"\
.format(table=CONFIG['DB_1D_TBL'],path=CONFIG['DB_1D_COLS']['path'], stamp=CONFIG['DB_1D_COLS']['stamp'], rid=CONFIG['DB_1D_COLS']['id'], ftype=CONFIG['DB_1D_COLS']['type'], stat=CONFIG['DB_1D_COLS']['stat'], name=CONFIG['DB_1D_COLS']['name'], slit=CONFIG['DB_1D_COLS']['slit'], snry=CONFIG['DB_1D_COLS']['snry'], snrj=CONFIG['DB_1D_COLS']['snrj'], snrh=CONFIG['DB_1D_COLS']['snrh'], snrk=CONFIG['DB_1D_COLS']['snrk']),(reduced['path'], stamp, reduced['id'], reduced['type'], 0, reduced['name'], reduced['slit'], reduced['snry'], reduced['snrj'], reduced['snrh'], reduced['snrk'], ))
cursor.execute("INSERT OR IGNORE INTO {table} ({path},{fstamp},{rid},{ftype},{stat},{name},{slit},{snry},{snrj},{snrh},{snrk}) VALUES (?,?,?,?,?,?,?,?,?,?,?)"\
.format(table=CONFIG['DB_1D_TBL'],path=CONFIG['DB_1D_COLS']['path'], fstamp=CONFIG['DB_1D_COLS']['stamp'], rid=CONFIG['DB_1D_COLS']['id'], ftype=CONFIG['DB_1D_COLS']['type'], stat=CONFIG['DB_1D_COLS']['stat'], name=CONFIG['DB_1D_COLS']['name'], slit=CONFIG['DB_1D_COLS']['slit'], snry=CONFIG['DB_1D_COLS']['snry'], snrj=CONFIG['DB_1D_COLS']['snrj'], snrh=CONFIG['DB_1D_COLS']['snrh'], snrk=CONFIG['DB_1D_COLS']['snrk']),(reduced['path'], stamp, reduced['id'], reduced['type'], 0, reduced['name'], reduced['slit'], reduced['snry'], reduced['snrj'], reduced['snrh'], reduced['snrk'], ))
dbnight.commit()
def insert_calib(dbnight, caltype, frames, quality, stamp):
#frames = ','.join(map(os.path.basename,frames))
frames = ','.join(map(str,frames))
quality = ','.join(map(str,quality))
cursor = dbnight.cursor()
cursor.execute("INSERT OR IGNORE INTO {table} ({ftype},{names},{status},{fstamp}) VALUES (?,?,?,?)"\
.format(table=CONFIG['DB_CAL_TBL'],ftype=CONFIG['DB_CAL_COLS']['type'], names=CONFIG['DB_CAL_COLS']['names'], status=CONFIG['DB_CAL_COLS']['status'], fstamp=CONFIG['DB_CAL_COLS']['stamp']),(caltype, frames, quality, stamp, ))
dbnight.commit()
drslib/flatframes.py
View file @ 6e1e7801
......@@ -66,8 +66,10 @@ class GBFlats():
if mean < CONFIG['FLATSIGNAL']:
self.messages.append('Flat frame %s failed quality check: signal too low (%s)' % (str(os.path.basename(frame)),str(mean)),)
flat = None
self.quality.append('FAILED')
else:
self.flatlist.append(flat)
self.quality.append('OK')
return
def masterflat(self):
......@@ -183,7 +185,9 @@ class GBFlats():
straight = flat.replace('.fits','_str.fits')
nor = flat.replace('.fits','_nor.fits')
args = [CONFIG['STRAIGHT'],flat,straight,CONFIG['STRAIGHT_OPT']]
#args = [CONFIG['STRAIGHT'],flat,straight,CONFIG['STRAIGHT_OPT']]
args = [CONFIG['STRAIGHT'],flat,straight]
args.extend(CONFIG['STRAIGHT_OPT'])
subprocess.call(args)
red_straight = os.path.join(CONFIG['RED_CALIB'],os.path.basename(straight))
......
drslib/nodding.py
View file @ 6e1e7801
......@@ -179,7 +179,9 @@ class GBNodding():
# straighten
straight = fitsfile.replace('.fits','_str.fits')
args = [CONFIG['STRAIGHT'],fitsfile,straight,CONFIG['STRAIGHT_OPT']]
#args = [CONFIG['STRAIGHT'],fitsfile,straight,CONFIG['STRAIGHT_OPT']]
args = [CONFIG['STRAIGHT'],fitsfile,straight]
args.extend(CONFIG['STRAIGHT_OPT'])
subprocess.call(args)
str_file = os.path.join(CONFIG['RED_STR'],os.path.basename(straight))
try: shutil.copyfile(straight,str_file)
......@@ -436,13 +438,13 @@ class GBNodding():
heaspe[CONFIG['DRS_MJD'][0]] = (drs_mjd,CONFIG['DRS_MJD'][1])
try:
if masterflat:
heaspe[CONFIG['MASTERFLAT'][0]] = (os.path.basename(masterflat),CONFIG['MASTERFLAT'][1])
except: pass
heaspe[CONFIG['MASTERFLAT'][0]] = (os.path.basename(masterflat),CONFIG['MASTERFLAT'][1])
except:
heaspe[CONFIG['MASTERFLAT'][0]] = ('None',CONFIG['MASTERFLAT'][1])
try:
if masterlamp:
heaspe[CONFIG['MASTERLAMP'][0]] = (os.path.basename(masterlamp),CONFIG['MASTERLAMP'][1])
except: pass
heaspe[CONFIG['MASTERLAMP'][0]] = (os.path.basename(masterlamp),CONFIG['MASTERLAMP'][1])
except:
heaspe[CONFIG['MASTERLAMP'][0]] = ('None',CONFIG['MASTERLAMP'][1])
snr[snr==np.inf] = 0
snr[snr==-np.inf] = 0
......@@ -790,9 +792,10 @@ class GBNodding():
def pair_process(self):
reduced = ','.join(map(str,self.nodding))
#reduced = ','.join(map(os.path.basename,self.nodding))
stamp = time.time()
db.insert_dbnight(self.dbnight, reduced, stamp)
#db.insert_dbnight(self.dbnight, reduced, stamp)
db.insert_dbnight(self.dbnight, self.nodding, stamp)
warnings.simplefilter('ignore', category=AstropyWarning)
if self.qualitycheck():
......
drslib/rawfiles.py
View file @ 6e1e7801
......@@ -117,11 +117,11 @@ class GBCheckRawFiles():
self.group['ngroup'] = groupn
self.group['n'] = groupi
# if len(self.lists[CONFIG['DARK']]) > groupn*int(CONFIG['NDARK']):
# self.messages.append('A new group of %s has started, the previous one will now be reduced.' % (obstype))
# self.command[CONFIG['DARK']] = True
# self.working = 0
# return
if len(self.lists[CONFIG['DARK']]) > groupn*int(CONFIG['NDARK']):
self.messages.append('A new group of %s has started, the previous one will now be reduced.' % (obstype))
self.command[CONFIG['DARK']] = True
self.working = 0
return
self.lists[CONFIG['DARK']].append(self.rawfile)
self.messages.append('%s out of %s %s files observed.' % (len(self.lists[CONFIG['DARK']]),groupn*int(CONFIG['NDARK']),obstype))
......@@ -160,6 +160,12 @@ class GBCheckRawFiles():
self.group['ngroup'] = groupn
self.group['n'] = groupi
if len(self.lists[CONFIG['FLAT']]) > groupn:
self.messages.append('A new group of %s has started, the previous one will now be reduced.' % (obstype))
self.command[CONFIG['FLAT']] = True
self.working = 0
return
self.lists[CONFIG['FLAT']].append(self.rawfile)
self.messages.append('%s out of %s %s files observed.' % (len(self.lists[CONFIG['FLAT']]),groupn,obstype))
......@@ -197,6 +203,12 @@ class GBCheckRawFiles():
self.group['ngroup'] = groupn
self.group['n'] = groupi
if len(self.lists[CONFIG['WCAL_UNE']]) > groupn:
self.messages.append('A new group of %s has started, the previous one will now be reduced.' % (obstype))
self.command[CONFIG['WCAL_UNE']] = True
self.working = 0
return
self.lists[CONFIG['WCAL_UNE']].append(self.rawfile)
self.messages.append('%s out of %s %s files observed.' % (len(self.lists[CONFIG['WCAL_UNE']]),groupn,obstype))
......@@ -233,6 +245,12 @@ class GBCheckRawFiles():
self.group['ngroup'] = groupn
self.group['n'] = groupi
if len(self.lists[CONFIG['WCAL_FP']]) > groupn:
self.messages.append('A new group of %s has started, the previous one will now be reduced.' % (obstype))
self.command[CONFIG['WCAL_FP']] = True
self.working = 0
return
self.lists[CONFIG['WCAL_FP']].append(self.rawfile)
self.messages.append('%s out of %s %s files observed.' % (len(self.lists[CONFIG['WCAL_FP']]),groupn,obstype))
......
drslib/stare.py
View file @ 6e1e7801
......@@ -327,8 +327,9 @@ class GBStare():
# straighten
straight = fitsfile.replace('.fits','_str.fits')
args = [CONFIG['STRAIGHT'],fitsfile,straight,CONFIG['STRAIGHT_OPT']]
#args = [CONFIG['STRAIGHT'],fitsfile,straight,CONFIG['STRAIGHT_OPT']]
args = [CONFIG['STRAIGHT'],fitsfile,straight]
args.extend(CONFIG['STRAIGHT_OPT'])
subprocess.call(args)
str_file = os.path.join(CONFIG['RED_STR'],os.path.basename(straight))
try: shutil.copyfile(straight,str_file)
......@@ -533,13 +534,13 @@ class GBStare():
heaspe = fits.Header(imstr.header)
try:
if masterflat:
heaspe[CONFIG['MASTERFLAT'][0]] = (os.path.basename(masterflat),CONFIG['MASTERFLAT'][1])
except: pass
heaspe[CONFIG['MASTERFLAT'][0]] = (os.path.basename(masterflat),CONFIG['MASTERFLAT'][1])
except:
heaspe[CONFIG['MASTERFLAT'][0]] = ('None',CONFIG['MASTERFLAT'][1])
try:
if masterlamp:
heaspe[CONFIG['MASTERLAMP'][0]] = (os.path.basename(masterlamp),CONFIG['MASTERLAMP'][1])
except: pass
heaspe[CONFIG['MASTERLAMP'][0]] = (os.path.basename(masterlamp),CONFIG['MASTERLAMP'][1])
except:
heaspe[CONFIG['MASTERLAMP'][0]] = ('None',CONFIG['MASTERLAMP'][1])
snr[snr==np.inf] = 0
snr[snr==-np.inf] = 0
......@@ -624,9 +625,10 @@ class GBStare():
return calname, False, dbreduced
def pair_process(self):
reduced = ','.join(map(str,self.stare))
#reduced = ','.join(map(os.path.basename,self.stare))
stamp = time.time()
db.insert_dbnight(self.dbnight, reduced, stamp)
#db.insert_dbnight(self.dbnight, reduced, stamp)
db.insert_dbnight(self.dbnight, self.stare, stamp)
warnings.simplefilter('ignore', category=AstropyWarning)
if self.qualitycheck():
......
drslib/wlframes.py
View file @ 6e1e7801
......@@ -46,6 +46,7 @@ class GBWls():
wl = ccdproc.CCDData.read(frame, unit=u.adu)
self.wllist.append(wl)
self.quality.append('OK')
return
def masterlamp(self,unefp):
......@@ -156,7 +157,9 @@ class GBWls():
straight = lamp.replace('.fits','_str.fits')
extracted = lamp.replace('.fits','_ext.fits')
args = [CONFIG['STRAIGHT'],lamp,straight,CONFIG['STRAIGHT_OPT']]
#args = [CONFIG['STRAIGHT'],lamp,straight,CONFIG['STRAIGHT_OPT']]
args = [CONFIG['STRAIGHT'],lamp,straight]
args.extend(CONFIG['STRAIGHT_OPT'])
subprocess.call(args)
red_straight = os.path.join(CONFIG['RED_CALIB'],os.path.basename(straight))
......
gofioDRS.py
View file @ 6e1e7801
......@@ -29,6 +29,7 @@ try:
CONFIG['LOG_FILE'] = os.path.join(CONFIG['RED_DIR'], 'drs.log')
#CONFIG['DO_CALIB'] = {'dark':True,'flat':True,'une':True,'fp':True,'only_calib':True}
CONFIG['DO_CALIB'] = {'dark':False,'flat':False,'une':False,'fp':False,'only_calib':False}
#CONFIG['USE_FLAT'] = {'global':False, 'order':False, 'nor':False}
maskc = ''.join(('GIANOB_MASKC_',CONFIG['DATE'],'.fits'))
CONFIG['MASK_C'] = os.path.join(CONFIG['RES_DIR'], maskc)
......@@ -158,10 +159,15 @@ def GianoBreduce(rawfits,lists,dbcalib,working,group,do_calib,stop):
if raw.command[CONFIG['DARK']]:
drslogger.log('----------------------------------------------')
drslogger.log(' *** DARK FRAMES *** ')
stamp = time.time()
darks = darkframes.GBDarks(rawlists[CONFIG['DARK']],dbcalib)
darks.process()
for message in darks.messages:
drslogger.log(message)
#reduced = ','.join(map(os.path.basename,darks.darks))
#quality = ','.join(map(str,darks.quality))
#db.insert_calib(dbnight, 'DARK', reduced, quality, stamp)
db.insert_calib(dbnight, 'DARK', darks.darks, darks.quality, stamp)
#if not CONFIG['OFFLINE']:
# for reduced in rawlists[CONFIG['DARK']]:
# db.insert_dbnight(dbnight, reduced)
......@@ -172,10 +178,15 @@ def GianoBreduce(rawfits,lists,dbcalib,working,group,do_calib,stop):
elif raw.command[CONFIG['FLAT']]:
drslogger.log('----------------------------------------------')
drslogger.log(' *** FLAT-FIELD FRAMES *** ')
stamp = time.time()
flats = flatframes.GBFlats(rawlists[CONFIG['FLAT']],dbcalib)
flats.process()
for message in flats.messages:
drslogger.log(message)
#reduced = ','.join(map(os.path.basename,flats.flats))
#quality = ','.join(map(str,flats.quality))
#db.insert_calib(dbnight, 'FLAT', reduced, quality, stamp)
db.insert_calib(dbnight, 'FLAT', flats.flats, flats.quality, stamp)
#if not CONFIG['OFFLINE']:
# for reduced in rawlists[CONFIG['FLAT']]:
# db.insert_dbnight(dbnight, reduced)
......@@ -186,10 +197,15 @@ def GianoBreduce(rawfits,lists,dbcalib,working,group,do_calib,stop):
elif raw.command[CONFIG['WCAL_UNE']]:
drslogger.log('----------------------------------------------')
drslogger.log(' *** UNE LAMP FRAMES *** ')
stamp = time.time()
wls = wlframes.GBWls(rawlists[CONFIG['WCAL_UNE']],dbcalib)
wls.une_process()
for message in wls.messages:
drslogger.log(message)
#reduced = ','.join(map(os.path.basename,wls.wls))
#quality = ','.join(map(str,wls.quality))
#db.insert_calib(dbnight, 'WCAL_UNE', reduced, quality, stamp)
db.insert_calib(dbnight, 'WCAL_UNE', wls.wls, wls.quality, stamp)
#if not CONFIG['OFFLINE']:
# for reduced in rawlists[CONFIG['WCAL_UNE']]:
# db.insert_dbnight(dbnight, reduced)
......@@ -200,10 +216,15 @@ def GianoBreduce(rawfits,lists,dbcalib,working,group,do_calib,stop):
elif raw.command[CONFIG['WCAL_FP']]:
drslogger.log('----------------------------------------------')
drslogger.log(' *** FP LAMP FRAMES *** ')
stamp = time.time()
fps = wlframes.GBWls(rawlists[CONFIG['WCAL_FP']],dbcalib)
fps.fp_process()
for message in fps.messages:
drslogger.log(message)
#reduced = ','.join(map(os.path.basename,fps.wls))
#quality = ','.join(map(str,fps.quality))
#db.insert_calib(dbnight, 'WCAL_FP', reduced, quality, stamp)
db.insert_calib(dbnight, 'WCAL_FP', fps.wls, fps.quality, stamp)
#if not CONFIG['OFFLINE']:
# for reduced in rawlists[CONFIG['WCAL_FP']]:
# db.insert_dbnight(dbnight, reduced)
......@@ -363,10 +384,11 @@ if __name__ == "__main__":
else:
for n in xrange(len(raws)):
if db.check_night(dbnight,raws[n]):
if db.check_night(dbnight,os.path.basename(raws[n])):
raws.pop(n)
break
elif db.check_raw(dbramps,raws[n]):
#elif db.check_raw(dbramps,raws[n]):
elif db.check_raw(dbramps,os.path.basename(raws[n])):
working, group, go, do_calib = GianoBreduce(raws[n], rawlists, dbcalib, working, group, do_calib, stop)
if go:
raws.pop(n)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment