Move logger.py to drslib

.gitignore

@@ -8,8 +8,7 @@ varie
 vradlib
 vrad_telluriche
 DA_FARE.txt
-GIARPS_GOFIO_v1.docx
-GIARPS_GOFIO_v1.odt
+GOFIO_*.*
 patchfile
 setup.py
 vrad_telluric.py

drslib/clean_db.py

+"""
+Remove files form calfiles that are not present in drs_calib.db
+"""
+
+import sqlite3, os, glob
+from drslib.config import CONFIG
+
+pathdb = CONFIG['DB_CALIB_PATH'] 
+tablename = CONFIG['DB_CALIB_TBL']
+pathcal = CONFIG['CALIB_DIR']
+
+def connect_db_cal():
+    dbcal = sqlite3.connect(pathdb)
+    return dbcal
+
+def column_names(dbcalib, tablename):
+    """
+    Return the names of the columns in the database.
+    """
+    dbcalib.row_factory = sqlite3.Row
+    cursor = dbcalib.execute("SELECT * FROM {table}"\
+                .format(table=tablename,))
+    row = cursor.fetchone()
+    names = row.keys()
+    return names
+
+
+def check_db(dbcalib, tablename, columns, calname):
+    """
+    Return True if the file does not exist in the database.
+    """
+
+    cursor = dbcalib.cursor()
+
+    for col in columns:
+        #print col
+        try:
+            cursor.execute("SELECT * FROM {table} WHERE instr({fpath},?) > 0"\
+                .format(table=tablename,fpath=col),(calname,))
+            found = cursor.fetchone()[0]
+
+            if found:
+                return False
+        except:
+            pass
+
+    return True
+
+
+if __name__ == "__main__":
+# list the calibration files
+    names = os.path.join(pathcal,"*.fits")
+    cals = glob.glob(names)
+    cals.sort()
+
+    dbcal = connect_db_cal()
+    columns = column_names(dbcal, tablename)
+
+    while cals:
+        cal = os.path.basename(cals[0])
+        check = check_db(dbcal, tablename, columns, cal)
+
+        if check:
+            os.remove(cals[0])
+
+        cals.pop(0)
+    dbcal.close()
+

drslib/config.py

@@ -65,21 +65,20 @@ CONFIG['OFFLINE'] = False
 # Directories 
 #
 
-CONFIG['USR_DIR'] = os.environ.get('HOME')
 CONFIG['TMP_DIR'] = tempfile.gettempdir()
 CONFIG['APP_DIR'] = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
-CONFIG['BASE_RAW'] = '/mnt/nfs/RAW'
-#CONFIG['BASE_RAW'] = '/home/monica/Documenti/gianob/gianobdrs/varie/commissioning_marzo/raw'
-#CONFIG['BASE_RED_DIR'] = os.path.join(CONFIG['APP_DIR'],'reduced')
-CONFIG['BASE_RED_DIR'] = '/mnt/nfs/REDUCED'
+#CONFIG['BASE_RAW'] = '/mnt/nfs/RAW'
+CONFIG['BASE_RAW'] = '/home/monica/Documenti/gianob/gianobdrs/varie/commissioning_marzo/raw'
+CONFIG['BASE_RED_DIR'] = os.path.join(CONFIG['APP_DIR'],'reduced')
+#CONFIG['BASE_RED_DIR'] = '/mnt/nfs/REDUCED'
 
 CONFIG['BASE_RAMP'] = '/home/avet/gianorp/databases'
 
 
 CONFIG['RAW_DIR'] = os.path.join(CONFIG['BASE_RAW'],CONFIG['DATE'])
-CONFIG['OFFLINE_DIR'] = False
 CONFIG['RED_DIR'] = os.path.join(CONFIG['BASE_RED_DIR'],CONFIG['DATE'])
+CONFIG['OFFLINE_DIR'] = False
 CONFIG['RED_CALIB'] = os.path.join(CONFIG['RED_DIR'],'CALIB')
 CONFIG['RED_STR'] = os.path.join(CONFIG['RED_DIR'],'STR')
 
@@ -117,6 +116,7 @@ CONFIG['DARK'] = 'DARK'
 CONFIG['FLAT'] = 'FLAT'
 CONFIG['WCAL_UNE'] = 'WCAL_UNE'
 CONFIG['WCAL_FP'] = 'WCAL_FP'
+#CONFIG['WCAL_FP'] = 'WCAL_LFC'
 CONFIG['SCIENCE'] = 'SCIENCE'
 # CONFIG['NODSTARE'] keyword values to determine if stare or nodding
 CONFIG['NODVALUE'] = 'GIANO_sci_nodding'
@@ -163,7 +163,7 @@ CONFIG['WLFIT'] = (' '.join((CONFIG['KEY_DRS'],'CAL WLFUNC')),'Function used for
 
 if CONFIG['CAL_FUNC']['Oliva']:
     CONFIG['WLFIT_FUNC'] = 'l0 + k1*(x-xc) + k2*(x-xc)**2 + k3*(x-xc)**3'
-    #CONFIG['WLCOEFFS'] = {'k1':(' '.join((CONFIG['KEY_DRS'],'CAL K1')),'k1 coefficient of the calibration function'),'k2':(' '.join((CONFIG['KEY_DRS'],'CAL K2')),'k2 coefficient of the calibration function'),'k3':(' '.join((CONFIG['KEY_DRS'],'CAL K3')),'k3 coefficient of the calibration function'),'l0':(' '.join((CONFIG['KEY_DRS'],'CAL L0')),'l0 coefficient of the calibration function'),'xc':(' '.join((CONFIG['KEY_DRS'],'CAL XC')),'xc coefficient of the calibration function'),'rms':(' '.join((CONFIG['KEY_DRS'],'CAL RMSE')),'[m/s] RMS error of the calibration')}
+
 else:
     CONFIG['WLFIT_FUNC'] = 'c0 + c1*x + c2*x**2 + c3*x**3'
 
@@ -183,8 +183,6 @@ CONFIG['DB_CALIB'] = 'drs_calib.db'
 CONFIG['DB_CALIB_PATH'] = os.path.join(CONFIG['CALIB_DB_DIR'], CONFIG['DB_CALIB'])
 CONFIG['DB_CALIB_TBL'] = 'calibrations'
 
-#CONFIG['DB_CALIB_COLS'] = OrderedDict([('01_data', 'date_calib'), ('dark10', 'masterdark10'), ('dark30', 'masterdark30'), ('dark60', 'masterdark60'), ('dark100', 'masterdark100'), ('flat', 'masterflat'), ('flatstr', 'masterflat_str'), ('flatnor', 'masterflat_nor'), ('une', 'wl_une'), ('une_str', 'wl_une_str'), ('une_calib', 'wl_une_calib'), ('fp', 'wl_fp'), ('fp_str', 'wl_fp_str'), ('fp_calib','wl_fp_calib')])
-
 CONFIG['DB_CALIB_COLS'] = OrderedDict([('01_data', 'date_calib'), ('dark10', 'masterdark10'), ('dark30', 'masterdark30'), ('dark60', 'masterdark60'), ('dark100', 'masterdark100'),  ('dark200', 'masterdark200'), ('dark300', 'masterdark300'), ('dark600', 'masterdark600'),('flat', 'masterflat'), ('flatstr', 'masterflat_str'), ('flatnor', 'masterflat_nor'), ('une', 'wl_une'), ('une_str', 'wl_une_str'), ('une_calib', 'wl_une_calib'), ('fp', 'wl_fp'), ('fp_str', 'wl_fp_str'), ('fp_calib','wl_fp_calib')])
 
 CONFIG['DB_CALIB_DATATYPE'] = OrderedDict()
@@ -207,12 +205,10 @@ CONFIG['DB_2D_COLS'] = OrderedDict([('names', 'spec_2d_names'), ('stamp', 'proce
 CONFIG['DB_1D_COLS'] = OrderedDict([('path', 'spec_1d_path'), ('stamp', 'process_timestamp'), ('id', 'spec_1d_id'), ('type', 'spec_1d_type'), ('stat', 'arch_stat'), ('name', 'objname'), ('slit', 'slitpos'), ('snry', 'snr1'), ('snrj', 'snr2'), ('snrh', 'snr3'), ('snrk', 'snr4')])
 CONFIG['DB_CAL_COLS'] = OrderedDict([('type', 'calib_type'), ('names', 'calib_names'), ('status', 'calib_status'), ('stamp', 'process_timestamp')])
 
-
 CONFIG['DB_2D_DATATYPE'] = {'names': 'TEXT unique', 'stamp': 'REAL'}
 CONFIG['DB_1D_DATATYPE'] = {'path': 'TEXT unique', 'stamp': 'REAL', 'id': 'TEXT', 'type': 'TEXT', 'stat': 'INTEGER', 'name': 'TEXT', 'slit': 'TEXT', 'snry': 'REAL', 'snrj': 'REAL', 'snrh': 'REAL', 'snrk': 'REAL'}
 CONFIG['DB_CAL_DATATYPE'] = {'type': 'TEXT', 'names': 'TEXT unique', 'status': 'TEXT', 'stamp': 'REAL'}
 
-
 CONFIG['DB_NIGHT'] = CONFIG['DB_ONLINE']
 CONFIG['DB_NIGHT_PATH'] = CONFIG['DB_ONLINE_PATH']
 
@@ -248,13 +244,10 @@ CONFIG['DO_CALIB'] = {'dark':True, 'flat':True, 'une':True, 'fp':True, 'only_cal
 
 CONFIG['USE_FLAT'] = {'global':False, 'order':True, 'nor':False} # global: flat normalized by global mean value - order: flat normalized order by order by mean value - nor: flat normalized row by row by continuum fitting, removing the blaze function - No True value: no flat division
 
-CONFIG['FLAT_EXPT'] = 100.0
-CONFIG['WLCAL_EXPT'] = 100.0
-CONFIG['FP_EXPT'] = 60.0
 CONFIG['DARKLIST'] = {10:[],30:[],60:[],100:[],200:[],300:[],600:[]} # exptime dei dark
 CONFIG['NDARK'] = len(CONFIG['DARKLIST']) # quanti tipi di dark (esempio: 10,30,60,100 sec = 4) meglio sovrastimare
 
-CONFIG['WEXT'] = 5 # extraction half-window for wavelength calibration, it will extract 2*wext around the center of the order
+CONFIG['WEXT'] = 10 # extraction half-window for wavelength calibration, it will extract 2*wext around the center of the order
 
 #
 # CCD info

logger/logger.py → drslib/logger.py

 """
-Last modified 2016-11-14
 Written by Avet Harutyunyan for the old GIANO pipeline
 Implemented in the GIANOB DRS by Monica Rainer
-TO DO: pass colour formatting for warnings
 """
 
 import logging

drslib/stare.py

@@ -64,8 +64,8 @@ class GBStare():
         """
         Check image's quality: check the signal in a well-defined region (only Obj).
         After this, check the number of Obj and Sky images and their exposure times.
-        Discard those with exposure times different, discard other images as needed
-        to have the same number of Obj and Sky.
+        Discard those with exposure times different, (discard other images as needed
+        to have the same number of Obj and Sky --> NOT ANYMORE).
         """
 
         expt_obj = []

drslib/wlframes.py

@@ -125,6 +125,7 @@ class GBWls():
 
             else:
                 wlname = hea[CONFIG['KEYS']['IMANAME']].replace('.fts','_FP.fits')
+                #wlname = hea[CONFIG['KEYS']['IMANAME']].replace('.fts','_LFC.fits')
 
             #wlname = wlname.replace(' ', '').replace('-', '_')
             nome = os.path.join(CONFIG['CALIB_DIR'],wlname)

gofioDRS.py

@@ -27,7 +27,8 @@ import os, glob, sys, time
 from docopt import docopt
 
 from drslib.config import CONFIG, read_usr_config
-from logger.logger import DrsLogger
+from drslib.logger import DrsLogger
+#from logger.logger import DrsLogger
 
 from drslib import db, rawfiles, darkframes, flatframes, wlframes, nodding, stare, varie