Skip to content

GitLab

Commit 59e78c6d authored by Monica Rainer's avatar Monica Rainer
Browse files

Add archive databases

parent cd7c5785
Hide whitespace changes
Inline Side-by-side
drslib/config.py
View file @ 59e78c6d
......@@ -82,7 +82,6 @@ CONFIG['RED_DIR'] = os.path.join(CONFIG['BASE_RED_DIR'],CONFIG['DATE'])
CONFIG['RED_CALIB'] = os.path.join(CONFIG['RED_DIR'],'CALIB')
CONFIG['RED_STR'] = os.path.join(CONFIG['RED_DIR'],'STR')
# Resources
CONFIG['RES_DIR'] = os.path.join(CONFIG['APP_DIR'], 'resources')
......@@ -94,7 +93,8 @@ CONFIG['STRAIGHT_OPT'] = 'I=1'
CONFIG['BASE_CALIB_DIR'] = os.path.join(CONFIG['APP_DIR'], 'calibrations')
CONFIG['CALIB_DIR'] = os.path.join(CONFIG['BASE_CALIB_DIR'], 'calfiles')
CONFIG['CALIB_DB_DIR'] = os.path.join(CONFIG['BASE_CALIB_DIR'], 'database')
CONFIG['NIGHT_DB_DIR'] = os.path.join(CONFIG['APP_DIR'], 'databases')
#CONFIG['NIGHT_DB_DIR'] = os.path.join(CONFIG['APP_DIR'], 'databases')
CONFIG['WEBUI_DB_DIR'] = os.path.join(CONFIG['APP_DIR'], 'webuidatabases')
# Output name
......@@ -155,8 +155,8 @@ CONFIG['MASTERFLAT'] = (' '.join((CONFIG['KEY_DRS'],'FLAT')),'Flat-field used fo
CONFIG['MASTERLAMP'] = (' '.join((CONFIG['KEY_DRS'],'LAMP')),'Calibration lamp used for the reduction')
# Wavelength calibration
#CONFIG['CAL_FUNC'] = {'Oliva':True, 'Poly3':False} # function used for wavelength calibration (Tino Oliva Polynomial or 3-degree polynomial)
CONFIG['CAL_FUNC'] = {'Oliva':False, 'Poly3':True}
CONFIG['CAL_FUNC'] = {'Oliva':True, 'Poly3':False} # function used for wavelength calibration (Tino Oliva Polynomial or 3-degree polynomial)
#CONFIG['CAL_FUNC'] = {'Oliva':False, 'Poly3':True}
CONFIG['WLFIT'] = (' '.join((CONFIG['KEY_DRS'],'CAL WLFUNC')),'Function used for wavelength calibration')
if CONFIG['CAL_FUNC']['Oliva']:
......@@ -188,14 +188,26 @@ for colname in CONFIG['DB_CALIB_COLS']:
CONFIG['DB_CALIB_DATATYPE']['01_data'] = 'TEXT unique'
# Night reduction database
CONFIG['DB_NIGHT'] = ''.join(('drs_',CONFIG['DATE'],'.db'))
CONFIG['DB_NIGHT_PATH'] = os.path.join(CONFIG['NIGHT_DB_DIR'], CONFIG['DB_NIGHT'])
CONFIG['DB_NIGHT_TBL'] = 'reduced'
CONFIG['DB_NIGHT_COLS'] = {'fpath': 'FPATH'}
CONFIG['DB_NIGHT_DATATYPE'] = {}
for colname in CONFIG['DB_CALIB_COLS']:
CONFIG['DB_NIGHT_DATATYPE'][colname] = 'TEXT'
CONFIG['DB_NIGHT_DATATYPE']['fpath'] = 'TEXT unique'
CONFIG['DB_ONLINE'] = ''.join(('db_',CONFIG['DATE'],'_online.db'))
CONFIG['DB_OFFLINE'] = ''.join(('db_',CONFIG['DATE'],'_offline.db'))
CONFIG['DB_ONLINE_PATH'] = os.path.join(CONFIG['WEBUI_DB_DIR'], CONFIG['DB_ONLINE'])
CONFIG['DB_OFFLINE_PATH'] = os.path.join(CONFIG['WEBUI_DB_DIR'], CONFIG['DB_OFFLINE'])
CONFIG['DB_2D_TBL'] = 'spec2dfiles'
CONFIG['DB_1D_TBL'] = 'spec1dfiles'
CONFIG['DB_2D_COLS'] = OrderedDict([('names', 'spec_2d_names'), ('stamp', 'process_timestamp')])
CONFIG['DB_1D_COLS'] = OrderedDict([('path', 'spec_1d_path'), ('stamp', 'process_timestamp'), ('id', 'spec_1d_id'), ('type', 'spec_1d_type'), ('stat', 'arch_stat'), ('name', 'objname'), ('slit', 'slitpos'), ('snry', 'snr1'), ('snrj', 'snr2'), ('snrh', 'snr3'), ('snrk', 'snr4')])
CONFIG['DB_2D_DATATYPE'] = {'names': 'TEXT unique', 'stamp': 'REAL'}
CONFIG['DB_1D_DATATYPE'] = {'path': 'TEXT unique', 'stamp': 'REAL', 'id': 'TEXT', 'type': 'TEXT', 'stat': 'INTEGER', 'name': 'TEXT', 'slit': 'TEXT', 'snry': 'REAL', 'snrj': 'REAL', 'snrh': 'REAL', 'snrk': 'REAL'}
CONFIG['DB_NIGHT'] = CONFIG['DB_ONLINE']
CONFIG['DB_NIGHT_PATH'] = CONFIG['DB_ONLINE_PATH']
#
# Extraction info
......@@ -271,4 +283,7 @@ if USRCONFIG['OFFLINE']:
CONFIG['RAMP_NAME'] = 'ramps' + CONFIG['DATE'] + '.db'
CONFIG['DB_RAMP'] = os.path.join(CONFIG['BASE_RAMP'],CONFIG['RAMP_NAME'])
CONFIG['DB_NIGHT'] = CONFIG['DB_OFFLINE']
CONFIG['DB_NIGHT_PATH'] = CONFIG['DB_OFFLINE_PATH']
drslib/db.py
View file @ 59e78c6d
......@@ -26,17 +26,22 @@ def create_db_calib():
def create_db_night():
try:
os.mkdir(CONFIG['NIGHT_DB_DIR'])
os.mkdir(CONFIG['WEBUI_DB_DIR'])
except OSError:
pass
dbcalib = sqlite3.connect(CONFIG['DB_NIGHT_PATH'])
cursor = dbcalib.cursor()
cursor.execute("CREATE TABLE IF NOT EXISTS " + \
CONFIG['DB_NIGHT_TBL'] + \
CONFIG['DB_2D_TBL'] + \
"(" + \
",".join([CONFIG['DB_NIGHT_COLS'][colname] + ' ' + CONFIG['DB_NIGHT_DATATYPE'][colname] for colname in CONFIG['DB_NIGHT_COLS']]) + \
",".join([CONFIG['DB_2D_COLS'][colname] + ' ' + CONFIG['DB_2D_DATATYPE'][colname] for colname in CONFIG['DB_2D_COLS']]) + \
')')
cursor.execute("CREATE TABLE IF NOT EXISTS " + \
CONFIG['DB_1D_TBL'] + \
"(" + \
",".join([CONFIG['DB_1D_COLS'][colname] + ' ' + CONFIG['DB_1D_DATATYPE'][colname] for colname in CONFIG['DB_1D_COLS']]) + \
')')
dbcalib.commit()
return dbcalib
......@@ -195,23 +200,41 @@ def check_night(dbnight,rawfile):
cursor = dbnight.cursor()
try:
cursor.execute("SELECT * FROM {table} WHERE {fpath}=?"\
.format(table=CONFIG['DB_NIGHT_TBL'],fpath=CONFIG['DB_NIGHT_COLS']['fpath']),(rawfile,))
filename = cursor.fetchone()[0]
if filename:
return True
cursor.execute("SELECT {stamp} FROM {table} WHERE instr({fpath},?) > 0"\
.format(stamp=CONFIG['DB_2D_COLS']['stamp'],table=CONFIG['DB_2D_TBL'],fpath=CONFIG['DB_2D_COLS']['names']),(rawfile,))
stamp = cursor.fetchone()[0]
if stamp:
cursor.execute("SELECT * FROM {table} WHERE {stamp}=?"\
.format(table=CONFIG['DB_1D_TBL'],stamp=CONFIG['DB_1D_COLS']['stamp']),(stamp,))
reduced = cursor.fetchone()[0]
if reduced:
return True
else:
return False
else:
return False
except:
return False
def insert_dbnight(dbnight, reduced):
def insert_dbnight(dbnight, reduced, stamp):
cursor = dbnight.cursor()
cursor.execute("UPDATE {table} SET {fpath}=?, {fstamp}=? WHERE instr({fpath},?) > 0"\
.format(table=CONFIG['DB_2D_TBL'],fpath=CONFIG['DB_2D_COLS']['names'],fstamp=CONFIG['DB_2D_COLS']['stamp']),(reduced,stamp,reduced,))
cursor.execute("INSERT OR IGNORE INTO {table} ({fpath},{fstamp}) VALUES (?,?)"\
.format(table=CONFIG['DB_2D_TBL'],fpath=CONFIG['DB_2D_COLS']['names'],fstamp=CONFIG['DB_2D_COLS']['stamp']),(reduced,stamp,))
dbnight.commit()
def insert_dbreduced(dbnight, reduced, stamp):
cursor = dbnight.cursor()
cursor.execute("INSERT OR IGNORE INTO {table} ({fpath}) VALUES (?)"\
.format(table=CONFIG['DB_NIGHT_TBL'],fpath=CONFIG['DB_NIGHT_COLS']['fpath']),(reduced,))
cursor.execute("INSERT OR IGNORE INTO {table} ({path},{stamp},{rid},{ftype},{stat},{name},{slit},{snry},{snrj},{snrh},{snrk}) VALUES (?,?,?,?,?,?,?,?,?,?,?)"\
.format(table=CONFIG['DB_1D_TBL'],path=CONFIG['DB_1D_COLS']['path'], stamp=CONFIG['DB_1D_COLS']['stamp'], rid=CONFIG['DB_1D_COLS']['id'], ftype=CONFIG['DB_1D_COLS']['type'], stat=CONFIG['DB_1D_COLS']['stat'], name=CONFIG['DB_1D_COLS']['name'], slit=CONFIG['DB_1D_COLS']['slit'], snry=CONFIG['DB_1D_COLS']['snry'], snrj=CONFIG['DB_1D_COLS']['snrj'], snrh=CONFIG['DB_1D_COLS']['snrh'], snrk=CONFIG['DB_1D_COLS']['snrk']),(reduced['path'], stamp, reduced['id'], reduced['type'], 0, reduced['name'], reduced['slit'], reduced['snry'], reduced['snrj'], reduced['snrh'], reduced['snrk'], ))
dbnight.commit()
drslib/nodding.py
View file @ 59e78c6d
......@@ -41,10 +41,11 @@ from collections import OrderedDict
class GBNodding():
def __init__(self, nodding, group, dbconn):
def __init__(self, nodding, group, dbconn, dbnight):
self.nodding = nodding
self.group = group
self.dbconn = dbconn
self.dbnight = dbnight
self.quality = []
self.messages = []
self.nodlist = []
......@@ -168,6 +169,7 @@ class GBNodding():
"""
Straighten, divide by the masterflat, optimal extraction
"""
dbreduced = {}
if slit_pos == CONFIG['A_POS']:
slit = 'A'
......@@ -442,10 +444,14 @@ class GBNodding():
snr[snr==-np.inf] = 0
snr = np.nan_to_num(snr)
self.messages.append('Nodding %s: SNR[Y band, order=73, wl=1050 nm] = %s' % (str(slit),str(round(np.mean(snr[39]),2))),)
self.messages.append('Nodding %s: SNR[J band, order=61, wl=1250 nm] = %s' % (str(slit),str(round(np.mean(snr[29]),2))),)
self.messages.append('Nodding %s: SNR[H band, order=46, wl=1650 nm] = %s' % (str(slit),str(round(np.mean(snr[14]),2))),)
self.messages.append('Nodding %s: SNR[K band, order=35, wl=2200 nm] = %s' % (str(slit),str(round(np.mean(snr[3]),2))),)
snry = round(np.mean(snr[41]),2)
snrj = round(np.mean(snr[29]),2)
snrh = round(np.mean(snr[14]),2)
snrk = round(np.mean(snr[3]),2)
self.messages.append('Nodding %s: SNR[Y band, order=73, wl=1050 nm] = %s' % (str(slit),str(snry),))
self.messages.append('Nodding %s: SNR[J band, order=61, wl=1250 nm] = %s' % (str(slit),str(snrj),))
self.messages.append('Nodding %s: SNR[H band, order=46, wl=1650 nm] = %s' % (str(slit),str(snrh),))
self.messages.append('Nodding %s: SNR[K band, order=35, wl=2200 nm] = %s' % (str(slit),str(snrk),))
for o in xrange(CONFIG['N_ORD']):
......@@ -489,6 +495,8 @@ class GBNodding():
#print 's1d'
#print calname1d
try: obj_name = heaspe[CONFIG['KEYS']['OBJECT']]
except: obj_name = 'NONE'
if CONFIG['S1D']:
#s1d = varie.create_s1d(optSpectrum,snr,heaspe)
......@@ -505,20 +513,27 @@ class GBNodding():
calname1d = os.path.join(CONFIG['RED_DIR'],calname1d)
s1dfits.writeto(calname1d,clobber=True)
rid = varie.random_id(12)
dbreduced['s1d'] = {'slit':slit, 'path':calname1d, 'snry':snry, 'snrj':snrj, 'snrh':snrh, 'snrk':snrk, 'type':'s1d', 'name':obj_name, 'id':rid}
rid = varie.random_id(12)
dbreduced['ms1d'] = {'slit':slit, 'path':calname, 'snry':snry, 'snrj':snrj, 'snrh':snrh, 'snrk':snrk, 'type':'ms1d', 'name':obj_name, 'id':rid}
#t2 = time.time()
#print 's1d spectrum: %s s' % str(t2-t1)
if hea_ima[CONFIG['KEYS']['EXTMODE']] == CONFIG['EXTPAIR']:
return calname, fsnr, straight
return calname, fsnr, straight, dbreduced
elif 'grp' in fitsfile:
return calname, fsnr, straight
return calname, fsnr, straight, dbreduced
return calname, fsnr, False
return calname, fsnr, False, dbreduced
def combine(self,acalib,bcalib,fsnr):
dbreduced = {}
#abnome = acalib.replace('_A_e2ds.fits','_AB_e2ds.fits')
#abnome1d = acalib.replace('_A_e2ds.fits','_AB_s1d.fits')
#abnome = abnome.replace('_Agrp_e2ds.fits','_ABgrp_e2ds.fits')
......@@ -640,6 +655,15 @@ class GBNodding():
#results.writeto(abnome,clobber=True)
snry = round(np.mean(snr[41]),2)
snrj = round(np.mean(snr[29]),2)
snrh = round(np.mean(snr[14]),2)
snrk = round(np.mean(snr[3]),2)
self.messages.append('Nodding %s: SNR[Y band, order=73, wl=1050 nm] = %s' % ('AB',str(snry)),)
self.messages.append('Nodding %s: SNR[J band, order=61, wl=1250 nm] = %s' % ('AB',str(snrj)),)
self.messages.append('Nodding %s: SNR[H band, order=46, wl=1650 nm] = %s' % ('AB',str(snrh)),)
self.messages.append('Nodding %s: SNR[K band, order=35, wl=2200 nm] = %s' % ('AB',str(snrk)),)
orders=np.arange(CONFIG['N_ORD'])+32
c1 = fits.Column(name='ORDER', format='I', array=orders)
......@@ -652,6 +676,9 @@ class GBNodding():
tbhdu.writeto(abnome,clobber=True)
try: obj_name = heaspe[CONFIG['KEYS']['OBJECT']]
except: obj_name = 'NONE'
if CONFIG['S1D']:
#s1d = varie.create_s1d(abcalib,snr,heaspe)
......@@ -667,11 +694,12 @@ class GBNodding():
s1dfits.writeto(abnome1d,clobber=True)
rid = varie.random_id(12)
dbreduced['s1d'] = {'slit':'AB', 'path':abnome1d, 'snry':snry, 'snrj':snrj, 'snrh':snrh, 'snrk':snrk, 'type':'s1d', 'name':obj_name, 'id':rid}
rid = varie.random_id(12)
dbreduced['ms1d'] = {'slit':'AB', 'path':abnome, 'snry':snry, 'snrj':snrj, 'snrh':snrh, 'snrk':snrk, 'type':'ms1d', 'name':obj_name, 'id':rid}
self.messages.append('Nodding merged AB: SNR[Y band, order=73, wl=1050 nm] = %s' % (str(round(np.mean(snr[41]),2))),)
self.messages.append('Nodding merged AB: SNR[J band, order=61, wl=1250 nm] = %s' % (str(round(np.mean(snr[29]),2))),)
self.messages.append('Nodding merged AB: SNR[H band, order=46, wl=1650 nm] = %s' % (str(round(np.mean(snr[14]),2))),)
self.messages.append('Nodding merged AB: SNR[K band, order=35, wl=2200 nm] = %s' % (str(round(np.mean(snr[3]),2))),)
#calibrated = np.vstack((np.concatenate(np.flipud(awaves)),np.concatenate(np.flipud(abcalib))))
......@@ -680,7 +708,7 @@ class GBNodding():
#extract = abnome.replace('.fits','.txt')
#ascii.write(np.transpose(calibrated),extract)
return
return dbreduced
def group_avg(self):
......@@ -758,15 +786,38 @@ class GBNodding():
def pair_process(self):
reduced = ','.join(map(str,self.nodding))
stamp = time.time()
db.insert_dbnight(self.dbnight, reduced, stamp)
warnings.simplefilter('ignore', category=AstropyWarning)
if self.qualitycheck():
#t1 = time.time()
ab, heaA, heaB = self.createAB()
#t2 = time.time()
#print 'Bad pixels, create nodding: %s s' % str(t2-t1)
acalib, fsnr, straight = self.reduce(ab,CONFIG['A_POS'], heaA)
bcalib, fsnr, straight = self.reduce(ab,CONFIG['B_POS'], heaB)
self.combine(acalib,bcalib,fsnr)
acalib, fsnr, straight, dbreduced = self.reduce(ab,CONFIG['A_POS'], heaA)
try:
db.insert_dbreduced(self.dbnight, dbreduced['s1d'], stamp)
except:
pass
db.insert_dbreduced(self.dbnight, dbreduced['ms1d'], stamp)
bcalib, fsnr, straight, dbreduced = self.reduce(ab,CONFIG['B_POS'], heaB)
try:
db.insert_dbreduced(self.dbnight, dbreduced['s1d'], stamp)
except:
pass
db.insert_dbreduced(self.dbnight, dbreduced['ms1d'], stamp)
dbreduced = self.combine(acalib,bcalib,fsnr)
try:
db.insert_dbreduced(self.dbnight, dbreduced['s1d'], stamp)
except:
pass
db.insert_dbreduced(self.dbnight, dbreduced['ms1d'], stamp)
if straight:
os.remove(ab)
os.remove(straight)
......@@ -784,9 +835,9 @@ class GBNodding():
try:
if self.group['noddings'] and len(self.group['noddings'])>1 :
ab, heaA, heaB = self.group_avg()
acalib, fsnr, straight = self.reduce(ab,CONFIG['A_POS'], heaA)
bcalib, fsnr, straight = self.reduce(ab,CONFIG['B_POS'], heaB)
self.combine(acalib,bcalib,fsnr)
acalib, fsnr, straight, dbreduced = self.reduce(ab,CONFIG['A_POS'], heaA)
bcalib, fsnr, straight, dbreduced = self.reduce(ab,CONFIG['B_POS'], heaB)
dbreduced = self.combine(acalib,bcalib,fsnr)
acalib = None
bcalib = None
os.remove(ab)
......@@ -804,9 +855,9 @@ class GBNodding():
try:
if self.group['noddings'] and len(self.group['noddings'])>1:
ab, heaA, heaB = self.group_avg()
acalib, fsnr, straight = self.reduce(ab,CONFIG['A_POS'], heaA)
bcalib, fsnr, straight = self.reduce(ab,CONFIG['B_POS'], heaB)
self.combine(acalib,bcalib,fsnr)
acalib, fsnr, straight, dbreduced = self.reduce(ab,CONFIG['A_POS'], heaA)
bcalib, fsnr, straight, dbreduced = self.reduce(ab,CONFIG['B_POS'], heaB)
dbreduced = self.combine(acalib,bcalib,fsnr)
acalib = None
bcalib = None
os.remove(ab)
......
drslib/stare.py
View file @ 59e78c6d
......@@ -48,10 +48,11 @@ from collections import OrderedDict, Counter
class GBStare():
def __init__(self, stare, group, dbconn):
def __init__(self, stare, group, dbconn, dbnight):
self.stare = stare
self.group = group
self.dbconn = dbconn
self.dbnight = dbnight
self.quality = []
self.messages = []
self.starelist = []
......@@ -321,6 +322,8 @@ class GBStare():
"""
Straighten, divide by the masterflat, optimal extraction
"""
dbreduced = {}
# straighten
straight = fitsfile.replace('.fits','_str.fits')
......@@ -538,10 +541,16 @@ class GBStare():
snr[snr==-np.inf] = 0
snr = np.nan_to_num(snr)
self.messages.append('Stare image: SNR[Y band, order=73, wl=1050 nm] = %s' % (str(round(np.mean(snr[41]),2))),)
self.messages.append('Stare image: SNR[J band, order=61, wl=1250 nm] = %s' % (str(round(np.mean(snr[29]),2))),)
self.messages.append('Stare image: SNR[H band, order=46, wl=1650 nm] = %s' % (str(round(np.mean(snr[14]),2))),)
self.messages.append('Stare image: SNR[K band, order=35, wl=2200 nm] = %s' % (str(round(np.mean(snr[3]),2))),)
snry = round(np.mean(snr[41]),2)
snrj = round(np.mean(snr[29]),2)
snrh = round(np.mean(snr[14]),2)
snrk = round(np.mean(snr[3]),2)
self.messages.append('Stare image: SNR[Y band, order=73, wl=1050 nm] = %s' % (str(snry)),)
self.messages.append('Stare image: SNR[J band, order=61, wl=1250 nm] = %s' % (str(snrj)),)
self.messages.append('Stare image: SNR[H band, order=46, wl=1650 nm] = %s' % (str(snrh)),)
self.messages.append('Stare image: SNR[K band, order=35, wl=2200 nm] = %s' % (str(snrk)),)
for o in xrange(CONFIG['N_ORD']):
key_snr = ''.join((CONFIG['SNR'][0],str(o+32)))
......@@ -577,6 +586,9 @@ class GBStare():
calname = os.path.join(CONFIG['RED_DIR'],calname)
tbhdu.writeto(calname,clobber=True)
try: obj_name = heaspe[CONFIG['KEYS']['OBJECT']]
except: obj_name = 'NONE'
if CONFIG['S1D']:
#s1d = varie.create_s1d(optSpectrum,snr,heaspe)
......@@ -591,21 +603,39 @@ class GBStare():
s1dfits = fits.PrimaryHDU(s1d,header=heaspe)
calname1d = os.path.join(CONFIG['RED_DIR'],calname1d)
s1dfits.writeto(calname1d,clobber=True)
rid = varie.random_id(12)
dbreduced['s1d'] = {'slit':slit_pos, 'path':calname1d, 'snry':snry, 'snrj':snrj, 'snrh':snrh, 'snrk':snrk, 'type':'s1d', 'name':obj_name, 'id':rid}
rid = varie.random_id(12)
dbreduced['ms1d'] = {'slit':slit_pos, 'path':calname, 'snry':snry, 'snrj':snrj, 'snrh':snrh, 'snrk':snrk, 'type':'ms1d', 'name':obj_name, 'id':rid}
if imstr.header[CONFIG['KEYS']['EXTMODE']] == CONFIG['EXTPAIR']:
return calname, straight
return calname, straight, dbreduced
elif 'grp' in fitsfile:
return calname, straight
return calname, straight, dbreduced
return calname, False
return calname, False, dbreduced
def pair_process(self):
reduced = ','.join(map(str,self.stare))
stamp = time.time()
db.insert_dbnight(self.dbnight, reduced, stamp)
warnings.simplefilter('ignore', category=AstropyWarning)
if self.qualitycheck():
obj = self.createObj(False)
calib, straight = self.reduce(obj,CONFIG['C_POS'])
calib, straight, dbreduced = self.reduce(obj,CONFIG['C_POS'])
try:
db.insert_dbreduced(self.dbnight, dbreduced['s1d'], stamp)
except:
pass
db.insert_dbreduced(self.dbnight, dbreduced['ms1d'], stamp)
if straight:
os.remove(obj)
os.remove(straight)
......@@ -623,7 +653,7 @@ class GBStare():
#print self.stare
if self.qualitycheck():
obj = self.createObj(True)
calib, straight = self.reduce(obj,CONFIG['C_POS'])
calib, straight, dbreduced = self.reduce(obj,CONFIG['C_POS'])
if straight:
os.remove(obj)
os.remove(straight)
......@@ -637,7 +667,7 @@ class GBStare():
self.stare = self.group['stares']
if self.qualitycheck():
obj = self.createObj(True)
calib, straight = self.reduce(obj,CONFIG['C_POS'])
calib, straight, dbreduced = self.reduce(obj,CONFIG['C_POS'])
if straight:
os.remove(obj)
os.remove(straight)
......
drslib/varie.py
View file @ 59e78c6d
......@@ -14,6 +14,7 @@ Last modified: 2017-06-07
- check_keyraw/check_keywords: check for keyword existence
- berv: computation of barycentric velocity correction (to be updated)
- create_s1d: create s1d output
- random_id: create random string
"""
from drslib.config import CONFIG
......@@ -28,6 +29,7 @@ import numpy as np
import numpy.polynomial.polynomial as poly
import math
import warnings
import string, random
#import matplotlib.pyplot as plt
#from scipy import optimize, interpolate, signal
......@@ -1183,3 +1185,9 @@ def create_s1d(spectrum, header):
return s1d, start[CONFIG['N_ORD']-1]
#--------------------- Create random string id -------------------
def random_id(length):
return ''.join(random.SystemRandom().choice(string.lowercase+string.uppercase+string.digits) for i in range(length))
gofioDRS.py
View file @ 59e78c6d
......@@ -33,6 +33,11 @@ try:
maskc = ''.join(('GIANOB_MASKC_',CONFIG['DATE'],'.fits'))
CONFIG['MASK_C'] = os.path.join(CONFIG['RES_DIR'], maskc)
CONFIG['DB_OFFLINE'] = ''.join(('db_',CONFIG['DATE'],'_offline.db'))
CONFIG['DB_OFFLINE_PATH'] = os.path.join(CONFIG['WEBUI_DB_DIR'], CONFIG['DB_OFFLINE'])
CONFIG['DB_NIGHT'] = CONFIG['DB_OFFLINE']
CONFIG['DB_NIGHT_PATH'] = CONFIG['DB_OFFLINE_PATH']
except:
pass
......@@ -80,11 +85,20 @@ except OSError:
drslogger = DrsLogger()
# Databases online/offline
dbcalib = db.create_db_calib()
if not CONFIG['OFFLINE']:
if CONFIG['OFFLINE']:
try:
os.remove(CONFIG['DB_NIGHT_PATH'])
except:
pass
else:
dbramps = db.connect_db_ramp()
dbnight = db.create_db_night()
dbnight = db.create_db_night()
do_calib = CONFIG['DO_CALIB']
......@@ -148,9 +162,9 @@ def GianoBreduce(rawfits,lists,dbcalib,working,group,do_calib,stop):
darks.process()
for message in darks.messages:
drslogger.log(message)
if not CONFIG['OFFLINE']:
for reduced in rawlists[CONFIG['DARK']]:
db.insert_dbnight(dbnight, reduced)
#if not CONFIG['OFFLINE']:
# for reduced in rawlists[CONFIG['DARK']]:
# db.insert_dbnight(dbnight, reduced)
rawlists[CONFIG['DARK']][:] = []
group.clear()
drslogger.log('----------------------------------------------')
......@@ -162,9 +176,9 @@ def GianoBreduce(rawfits,lists,dbcalib,working,group,do_calib,stop):
flats.process()
for message in flats.messages:
drslogger.log(message)
if not CONFIG['OFFLINE']:
for reduced in rawlists[CONFIG['FLAT']]:
db.insert_dbnight(dbnight, reduced)
#if not CONFIG['OFFLINE']:
# for reduced in rawlists[CONFIG['FLAT']]:
# db.insert_dbnight(dbnight, reduced)
rawlists[CONFIG['FLAT']][:] = []
group.clear()
drslogger.log('----------------------------------------------')
......@@ -176,9 +190,9 @@ def GianoBreduce(rawfits,lists,dbcalib,working,group,do_calib,stop):
wls.une_process()
for message in wls.messages:
drslogger.log(message)
if not CONFIG['OFFLINE']:
for reduced in rawlists[CONFIG['WCAL_UNE']]:
db.insert_dbnight(dbnight, reduced)
#if not CONFIG['OFFLINE']:
# for reduced in rawlists[CONFIG['WCAL_UNE']]:
# db.insert_dbnight(dbnight, reduced)
rawlists[CONFIG['WCAL_UNE']][:] = []
group.clear()
drslogger.log('----------------------------------------------')
......@@ -190,9 +204,9 @@ def GianoBreduce(rawfits,lists,dbcalib,working,group,do_calib,stop):
fps.fp_process()
for message in fps.messages:
drslogger.log(message)
if not CONFIG['OFFLINE']:
for reduced in rawlists[CONFIG['WCAL_FP']]:
db.insert_dbnight(dbnight, reduced)
#if not CONFIG['OFFLINE']:
# for reduced in rawlists[CONFIG['WCAL_FP']]:
# db.insert_dbnight(dbnight, reduced)
rawlists[CONFIG['WCAL_FP']][:] = []
group.clear()
drslogger.log('----------------------------------------------')
......@@ -201,15 +215,15 @@ def GianoBreduce(rawfits,lists,dbcalib,working,group,do_calib,stop):
drslogger.log('----------------------------------------------')
drslogger.log(' *** NODDING PAIR *** ')
t1 = time.time()
nods = nodding.GBNodding(rawlists[CONFIG['SCIENCE']],group,dbcalib)
nods = nodding.GBNodding(rawlists[CONFIG['SCIENCE']],group,dbcalib,dbnight)
nods.pair_process()
for message in nods.messages:
drslogger.log(message)
t2 = time.time()
drslogger.log(' *** Reduction: %s s ***' % (str(round((t2-t1),2)), ))
if not CONFIG['OFFLINE']:
for reduced in rawlists[CONFIG['SCIENCE']]:
db.insert_dbnight(dbnight, reduced)
#if not CONFIG['OFFLINE']:
# for reduced in rawlists[CONFIG['SCIENCE']]:
# db.insert_dbnight(dbnight, reduced)
rawlists[CONFIG['SCIENCE']][:] = []
drslogger.log('----------------------------------------------')
......@@ -217,15 +231,15 @@ def GianoBreduce(rawfits,lists,dbcalib,working,group,do_calib,stop):
drslogger.log('----------------------------------------------')
drslogger.log(' *** STARE PAIR *** ')
t1 = time.time()
stares = stare.GBStare(rawlists[CONFIG['SCIENCE']],group,dbcalib)
stares = stare.GBStare(rawlists[CONFIG['SCIENCE']],group,dbcalib,dbnight)
stares.pair_process()
for message in stares.messages:
drslogger.log(message)
t2 = time.time()
drslogger.log(' *** Reduction: %s s ***' % (str(round((t2-t1),2)), ))
if not CONFIG['OFFLINE']:
for reduced in rawlists[CONFIG['SCIENCE']]:
db.insert_dbnight(dbnight, reduced)
#if not CONFIG['OFFLINE']:
# for reduced in rawlists[CONFIG['SCIENCE']]:
# db.insert_dbnight(dbnight, reduced)
rawlists[CONFIG['SCIENCE']][:] = []
drslogger.log('----------------------------------------------')
......@@ -234,15 +248,15 @@ def GianoBreduce(rawfits,lists,dbcalib,working,group,do_calib,stop):
drslogger.log('----------------------------------------------')