Commit 8a9821e2 authored by Monica Rainer's avatar Monica Rainer
Browse files

Move logger.py to drslib

parent 1a7e24fd
...@@ -8,8 +8,7 @@ varie ...@@ -8,8 +8,7 @@ varie
vradlib vradlib
vrad_telluriche vrad_telluriche
DA_FARE.txt DA_FARE.txt
GIARPS_GOFIO_v1.docx GOFIO_*.*
GIARPS_GOFIO_v1.odt
patchfile patchfile
setup.py setup.py
vrad_telluric.py vrad_telluric.py
......
"""
Remove files form calfiles that are not present in drs_calib.db
"""
import sqlite3, os, glob
from drslib.config import CONFIG
pathdb = CONFIG['DB_CALIB_PATH']
tablename = CONFIG['DB_CALIB_TBL']
pathcal = CONFIG['CALIB_DIR']
def connect_db_cal():
dbcal = sqlite3.connect(pathdb)
return dbcal
def column_names(dbcalib, tablename):
"""
Return the names of the columns in the database.
"""
dbcalib.row_factory = sqlite3.Row
cursor = dbcalib.execute("SELECT * FROM {table}"\
.format(table=tablename,))
row = cursor.fetchone()
names = row.keys()
return names
def check_db(dbcalib, tablename, columns, calname):
"""
Return True if the file does not exist in the database.
"""
cursor = dbcalib.cursor()
for col in columns:
#print col
try:
cursor.execute("SELECT * FROM {table} WHERE instr({fpath},?) > 0"\
.format(table=tablename,fpath=col),(calname,))
found = cursor.fetchone()[0]
if found:
return False
except:
pass
return True
if __name__ == "__main__":
# list the calibration files
names = os.path.join(pathcal,"*.fits")
cals = glob.glob(names)
cals.sort()
dbcal = connect_db_cal()
columns = column_names(dbcal, tablename)
while cals:
cal = os.path.basename(cals[0])
check = check_db(dbcal, tablename, columns, cal)
if check:
os.remove(cals[0])
cals.pop(0)
dbcal.close()
...@@ -65,21 +65,20 @@ CONFIG['OFFLINE'] = False ...@@ -65,21 +65,20 @@ CONFIG['OFFLINE'] = False
# Directories # Directories
# #
CONFIG['USR_DIR'] = os.environ.get('HOME')
CONFIG['TMP_DIR'] = tempfile.gettempdir() CONFIG['TMP_DIR'] = tempfile.gettempdir()
CONFIG['APP_DIR'] = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) CONFIG['APP_DIR'] = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
CONFIG['BASE_RAW'] = '/mnt/nfs/RAW' #CONFIG['BASE_RAW'] = '/mnt/nfs/RAW'
#CONFIG['BASE_RAW'] = '/home/monica/Documenti/gianob/gianobdrs/varie/commissioning_marzo/raw' CONFIG['BASE_RAW'] = '/home/monica/Documenti/gianob/gianobdrs/varie/commissioning_marzo/raw'
#CONFIG['BASE_RED_DIR'] = os.path.join(CONFIG['APP_DIR'],'reduced') CONFIG['BASE_RED_DIR'] = os.path.join(CONFIG['APP_DIR'],'reduced')
CONFIG['BASE_RED_DIR'] = '/mnt/nfs/REDUCED' #CONFIG['BASE_RED_DIR'] = '/mnt/nfs/REDUCED'
CONFIG['BASE_RAMP'] = '/home/avet/gianorp/databases' CONFIG['BASE_RAMP'] = '/home/avet/gianorp/databases'
CONFIG['RAW_DIR'] = os.path.join(CONFIG['BASE_RAW'],CONFIG['DATE']) CONFIG['RAW_DIR'] = os.path.join(CONFIG['BASE_RAW'],CONFIG['DATE'])
CONFIG['OFFLINE_DIR'] = False
CONFIG['RED_DIR'] = os.path.join(CONFIG['BASE_RED_DIR'],CONFIG['DATE']) CONFIG['RED_DIR'] = os.path.join(CONFIG['BASE_RED_DIR'],CONFIG['DATE'])
CONFIG['OFFLINE_DIR'] = False
CONFIG['RED_CALIB'] = os.path.join(CONFIG['RED_DIR'],'CALIB') CONFIG['RED_CALIB'] = os.path.join(CONFIG['RED_DIR'],'CALIB')
CONFIG['RED_STR'] = os.path.join(CONFIG['RED_DIR'],'STR') CONFIG['RED_STR'] = os.path.join(CONFIG['RED_DIR'],'STR')
...@@ -117,6 +116,7 @@ CONFIG['DARK'] = 'DARK' ...@@ -117,6 +116,7 @@ CONFIG['DARK'] = 'DARK'
CONFIG['FLAT'] = 'FLAT' CONFIG['FLAT'] = 'FLAT'
CONFIG['WCAL_UNE'] = 'WCAL_UNE' CONFIG['WCAL_UNE'] = 'WCAL_UNE'
CONFIG['WCAL_FP'] = 'WCAL_FP' CONFIG['WCAL_FP'] = 'WCAL_FP'
#CONFIG['WCAL_FP'] = 'WCAL_LFC'
CONFIG['SCIENCE'] = 'SCIENCE' CONFIG['SCIENCE'] = 'SCIENCE'
# CONFIG['NODSTARE'] keyword values to determine if stare or nodding # CONFIG['NODSTARE'] keyword values to determine if stare or nodding
CONFIG['NODVALUE'] = 'GIANO_sci_nodding' CONFIG['NODVALUE'] = 'GIANO_sci_nodding'
...@@ -163,7 +163,7 @@ CONFIG['WLFIT'] = (' '.join((CONFIG['KEY_DRS'],'CAL WLFUNC')),'Function used for ...@@ -163,7 +163,7 @@ CONFIG['WLFIT'] = (' '.join((CONFIG['KEY_DRS'],'CAL WLFUNC')),'Function used for
if CONFIG['CAL_FUNC']['Oliva']: if CONFIG['CAL_FUNC']['Oliva']:
CONFIG['WLFIT_FUNC'] = 'l0 + k1*(x-xc) + k2*(x-xc)**2 + k3*(x-xc)**3' CONFIG['WLFIT_FUNC'] = 'l0 + k1*(x-xc) + k2*(x-xc)**2 + k3*(x-xc)**3'
#CONFIG['WLCOEFFS'] = {'k1':(' '.join((CONFIG['KEY_DRS'],'CAL K1')),'k1 coefficient of the calibration function'),'k2':(' '.join((CONFIG['KEY_DRS'],'CAL K2')),'k2 coefficient of the calibration function'),'k3':(' '.join((CONFIG['KEY_DRS'],'CAL K3')),'k3 coefficient of the calibration function'),'l0':(' '.join((CONFIG['KEY_DRS'],'CAL L0')),'l0 coefficient of the calibration function'),'xc':(' '.join((CONFIG['KEY_DRS'],'CAL XC')),'xc coefficient of the calibration function'),'rms':(' '.join((CONFIG['KEY_DRS'],'CAL RMSE')),'[m/s] RMS error of the calibration')}
else: else:
CONFIG['WLFIT_FUNC'] = 'c0 + c1*x + c2*x**2 + c3*x**3' CONFIG['WLFIT_FUNC'] = 'c0 + c1*x + c2*x**2 + c3*x**3'
...@@ -183,8 +183,6 @@ CONFIG['DB_CALIB'] = 'drs_calib.db' ...@@ -183,8 +183,6 @@ CONFIG['DB_CALIB'] = 'drs_calib.db'
CONFIG['DB_CALIB_PATH'] = os.path.join(CONFIG['CALIB_DB_DIR'], CONFIG['DB_CALIB']) CONFIG['DB_CALIB_PATH'] = os.path.join(CONFIG['CALIB_DB_DIR'], CONFIG['DB_CALIB'])
CONFIG['DB_CALIB_TBL'] = 'calibrations' CONFIG['DB_CALIB_TBL'] = 'calibrations'
#CONFIG['DB_CALIB_COLS'] = OrderedDict([('01_data', 'date_calib'), ('dark10', 'masterdark10'), ('dark30', 'masterdark30'), ('dark60', 'masterdark60'), ('dark100', 'masterdark100'), ('flat', 'masterflat'), ('flatstr', 'masterflat_str'), ('flatnor', 'masterflat_nor'), ('une', 'wl_une'), ('une_str', 'wl_une_str'), ('une_calib', 'wl_une_calib'), ('fp', 'wl_fp'), ('fp_str', 'wl_fp_str'), ('fp_calib','wl_fp_calib')])
CONFIG['DB_CALIB_COLS'] = OrderedDict([('01_data', 'date_calib'), ('dark10', 'masterdark10'), ('dark30', 'masterdark30'), ('dark60', 'masterdark60'), ('dark100', 'masterdark100'), ('dark200', 'masterdark200'), ('dark300', 'masterdark300'), ('dark600', 'masterdark600'),('flat', 'masterflat'), ('flatstr', 'masterflat_str'), ('flatnor', 'masterflat_nor'), ('une', 'wl_une'), ('une_str', 'wl_une_str'), ('une_calib', 'wl_une_calib'), ('fp', 'wl_fp'), ('fp_str', 'wl_fp_str'), ('fp_calib','wl_fp_calib')]) CONFIG['DB_CALIB_COLS'] = OrderedDict([('01_data', 'date_calib'), ('dark10', 'masterdark10'), ('dark30', 'masterdark30'), ('dark60', 'masterdark60'), ('dark100', 'masterdark100'), ('dark200', 'masterdark200'), ('dark300', 'masterdark300'), ('dark600', 'masterdark600'),('flat', 'masterflat'), ('flatstr', 'masterflat_str'), ('flatnor', 'masterflat_nor'), ('une', 'wl_une'), ('une_str', 'wl_une_str'), ('une_calib', 'wl_une_calib'), ('fp', 'wl_fp'), ('fp_str', 'wl_fp_str'), ('fp_calib','wl_fp_calib')])
CONFIG['DB_CALIB_DATATYPE'] = OrderedDict() CONFIG['DB_CALIB_DATATYPE'] = OrderedDict()
...@@ -207,12 +205,10 @@ CONFIG['DB_2D_COLS'] = OrderedDict([('names', 'spec_2d_names'), ('stamp', 'proce ...@@ -207,12 +205,10 @@ CONFIG['DB_2D_COLS'] = OrderedDict([('names', 'spec_2d_names'), ('stamp', 'proce
CONFIG['DB_1D_COLS'] = OrderedDict([('path', 'spec_1d_path'), ('stamp', 'process_timestamp'), ('id', 'spec_1d_id'), ('type', 'spec_1d_type'), ('stat', 'arch_stat'), ('name', 'objname'), ('slit', 'slitpos'), ('snry', 'snr1'), ('snrj', 'snr2'), ('snrh', 'snr3'), ('snrk', 'snr4')]) CONFIG['DB_1D_COLS'] = OrderedDict([('path', 'spec_1d_path'), ('stamp', 'process_timestamp'), ('id', 'spec_1d_id'), ('type', 'spec_1d_type'), ('stat', 'arch_stat'), ('name', 'objname'), ('slit', 'slitpos'), ('snry', 'snr1'), ('snrj', 'snr2'), ('snrh', 'snr3'), ('snrk', 'snr4')])
CONFIG['DB_CAL_COLS'] = OrderedDict([('type', 'calib_type'), ('names', 'calib_names'), ('status', 'calib_status'), ('stamp', 'process_timestamp')]) CONFIG['DB_CAL_COLS'] = OrderedDict([('type', 'calib_type'), ('names', 'calib_names'), ('status', 'calib_status'), ('stamp', 'process_timestamp')])
CONFIG['DB_2D_DATATYPE'] = {'names': 'TEXT unique', 'stamp': 'REAL'} CONFIG['DB_2D_DATATYPE'] = {'names': 'TEXT unique', 'stamp': 'REAL'}
CONFIG['DB_1D_DATATYPE'] = {'path': 'TEXT unique', 'stamp': 'REAL', 'id': 'TEXT', 'type': 'TEXT', 'stat': 'INTEGER', 'name': 'TEXT', 'slit': 'TEXT', 'snry': 'REAL', 'snrj': 'REAL', 'snrh': 'REAL', 'snrk': 'REAL'} CONFIG['DB_1D_DATATYPE'] = {'path': 'TEXT unique', 'stamp': 'REAL', 'id': 'TEXT', 'type': 'TEXT', 'stat': 'INTEGER', 'name': 'TEXT', 'slit': 'TEXT', 'snry': 'REAL', 'snrj': 'REAL', 'snrh': 'REAL', 'snrk': 'REAL'}
CONFIG['DB_CAL_DATATYPE'] = {'type': 'TEXT', 'names': 'TEXT unique', 'status': 'TEXT', 'stamp': 'REAL'} CONFIG['DB_CAL_DATATYPE'] = {'type': 'TEXT', 'names': 'TEXT unique', 'status': 'TEXT', 'stamp': 'REAL'}
CONFIG['DB_NIGHT'] = CONFIG['DB_ONLINE'] CONFIG['DB_NIGHT'] = CONFIG['DB_ONLINE']
CONFIG['DB_NIGHT_PATH'] = CONFIG['DB_ONLINE_PATH'] CONFIG['DB_NIGHT_PATH'] = CONFIG['DB_ONLINE_PATH']
...@@ -248,13 +244,10 @@ CONFIG['DO_CALIB'] = {'dark':True, 'flat':True, 'une':True, 'fp':True, 'only_cal ...@@ -248,13 +244,10 @@ CONFIG['DO_CALIB'] = {'dark':True, 'flat':True, 'une':True, 'fp':True, 'only_cal
CONFIG['USE_FLAT'] = {'global':False, 'order':True, 'nor':False} # global: flat normalized by global mean value - order: flat normalized order by order by mean value - nor: flat normalized row by row by continuum fitting, removing the blaze function - No True value: no flat division CONFIG['USE_FLAT'] = {'global':False, 'order':True, 'nor':False} # global: flat normalized by global mean value - order: flat normalized order by order by mean value - nor: flat normalized row by row by continuum fitting, removing the blaze function - No True value: no flat division
CONFIG['FLAT_EXPT'] = 100.0
CONFIG['WLCAL_EXPT'] = 100.0
CONFIG['FP_EXPT'] = 60.0
CONFIG['DARKLIST'] = {10:[],30:[],60:[],100:[],200:[],300:[],600:[]} # exptime dei dark CONFIG['DARKLIST'] = {10:[],30:[],60:[],100:[],200:[],300:[],600:[]} # exptime dei dark
CONFIG['NDARK'] = len(CONFIG['DARKLIST']) # quanti tipi di dark (esempio: 10,30,60,100 sec = 4) meglio sovrastimare CONFIG['NDARK'] = len(CONFIG['DARKLIST']) # quanti tipi di dark (esempio: 10,30,60,100 sec = 4) meglio sovrastimare
CONFIG['WEXT'] = 5 # extraction half-window for wavelength calibration, it will extract 2*wext around the center of the order CONFIG['WEXT'] = 10 # extraction half-window for wavelength calibration, it will extract 2*wext around the center of the order
# #
# CCD info # CCD info
......
""" """
Last modified 2016-11-14
Written by Avet Harutyunyan for the old GIANO pipeline Written by Avet Harutyunyan for the old GIANO pipeline
Implemented in the GIANOB DRS by Monica Rainer Implemented in the GIANOB DRS by Monica Rainer
TO DO: pass colour formatting for warnings
""" """
import logging import logging
......
...@@ -64,8 +64,8 @@ class GBStare(): ...@@ -64,8 +64,8 @@ class GBStare():
""" """
Check image's quality: check the signal in a well-defined region (only Obj). Check image's quality: check the signal in a well-defined region (only Obj).
After this, check the number of Obj and Sky images and their exposure times. After this, check the number of Obj and Sky images and their exposure times.
Discard those with exposure times different, discard other images as needed Discard those with exposure times different, (discard other images as needed
to have the same number of Obj and Sky. to have the same number of Obj and Sky --> NOT ANYMORE).
""" """
expt_obj = [] expt_obj = []
......
...@@ -125,6 +125,7 @@ class GBWls(): ...@@ -125,6 +125,7 @@ class GBWls():
else: else:
wlname = hea[CONFIG['KEYS']['IMANAME']].replace('.fts','_FP.fits') wlname = hea[CONFIG['KEYS']['IMANAME']].replace('.fts','_FP.fits')
#wlname = hea[CONFIG['KEYS']['IMANAME']].replace('.fts','_LFC.fits')
#wlname = wlname.replace(' ', '').replace('-', '_') #wlname = wlname.replace(' ', '').replace('-', '_')
nome = os.path.join(CONFIG['CALIB_DIR'],wlname) nome = os.path.join(CONFIG['CALIB_DIR'],wlname)
......
...@@ -27,7 +27,8 @@ import os, glob, sys, time ...@@ -27,7 +27,8 @@ import os, glob, sys, time
from docopt import docopt from docopt import docopt
from drslib.config import CONFIG, read_usr_config from drslib.config import CONFIG, read_usr_config
from logger.logger import DrsLogger from drslib.logger import DrsLogger
#from logger.logger import DrsLogger
from drslib import db, rawfiles, darkframes, flatframes, wlframes, nodding, stare, varie from drslib import db, rawfiles, darkframes, flatframes, wlframes, nodding, stare, varie
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment