Merge branch 'v1.2.0' into develop

.gitignore

 backup
-calibrations
+fringing
 git_howto.txt
-gofio_latest
-gofio_latest.tar
+gofioDRS.py
 reduced
 varie
 vradlib
 vrad_telluriche
 DA_FARE.txt
-GIARPS_GOFIO_v1.docx
-GIARPS_GOFIO_v1.odt
+GOFIO_*.*
 patchfile
 setup.py
 vrad_telluric.py
@@ -19,4 +17,7 @@ webuidatabases
 */*/*/*.pyc
 *~
 .~*
+gofio/calibrations
+gofio/resources/straight_giano_2D
+gofio/resources/straight_giano_2D_v1_1_old.f
 

README.md

@@ -7,8 +7,66 @@ Python packages:
  - NumPy v1.12
  - SciPy v0.19
  - watchdog v0.8.2
+ - docopt v0.6.1
+Other requirements:
+ - fortran77 compiler
+ - libcfitsio library
+
+Installation:
+ - preserve the directory tree:
+   gofio/
+     __init__.py
+     __version__.py
+     gofioDRS.py
+     drslib/
+       __init__.py
+       clean_db.py
+       config.py
+       darkframes.py
+       db.py
+       flatframes.py
+       gofio.py
+       gofio2ascii.py
+       logger.py
+       nodding.py
+       rawfiles.py
+       stare.py
+       varie.py
+       wlframes.py
+       berv/
+         __init__.py
+         astroTimeLegacy.py
+         baryvel.py
+     resources/
+       GIANOB_MASKC.fits
+       UNe_lines_GIANO_selected.txt
+       UNe_observed_lines_GIANOB_18nov2016.txt
+       badpix_mask.fits
+       straight_giano_2D_v1_1.f
+ - compile the fortran77 straight_giano_2D_v1_1.f file:
+   gfortran -o straight_giano_2D straight_giano_2D_v1_1.f -lcfitsio
+ - modify the configuration file drslib/config.py in order to set the
+   right directory path for raw and reduced files (CONFIG['BASE_RAW']
+   and CONFIG['BASE_RED_DIR'])
+ - run "python gofioDRS.py -h" for help
+
 
 Output:
- - intermediate: *_str.fits, 2D images with orders straightened and bad pixel removed
- - final output: *_ms1d.fits. The spectra are flat-fielded, extracted, calibrated, and the cosmic rays are removed. The echelle orders are not merged. The FITS table has 50 rows and 4 columns, and it stores each echelle order in a row. The columns are: the number of the echelle order (from 32 - the reddest - to 81 - the bluest), the wavelength in nanometers, the fluxes and the signal-to-noise ratios.
- - final output (optional): *_s1d.fits. The spectra are flat-fielded, extracted, calibrated, and the cosmic rays are removed. The echelle orders are merged. The barycentric correction is applied. These are monodimensional spectra with constant step in wavelength. 
+ - intermediate: *_str.fits, 2D images with orders straightened
+   and bad pixel removed
+ - final output: *_ms1d.fits. The spectra are flat-fielded, extracted,
+   calibrated, and the cosmic rays are removed.
+   The echelle orders are not merged.
+   The FITS table has 50 rows and 4 columns, and it stores each echelle
+   order in a row. The columns are: the number of the echelle order
+   (from 32 - the reddest - to 81 - the bluest), the wavelength in nanometers,
+   the fluxes and the signal-to-noise ratios.
+ - final output (optional): *_s1d.fits. The spectra are flat-fielded, extracted, 
+   calibrated, and the cosmic rays are removed. The echelle orders are merged.
+   The barycentric correction is applied. These are monodimensional spectra with
+   constant step in wavelength.
+
+The program gofio2ascii.py may be used to convert the outputs from FITS format
+to ASCII files. It requires as input a text list with the filepaths of the
+files to convert.
+

gofio/__init__.py

+from __version__ import __version__

gofio/__version__.py

+__version__ = '1.2.0'

gofio/drslib/clean_db.py

+"""
+Remove files form calfiles that are not present in drs_calib.db
+It must be run from the drslib directory with the command:
+python clean_db.py
+"""
+
+import sqlite3, os, glob
+from drslib.config import CONFIG
+
+pathdb = CONFIG['DB_CALIB_PATH'] 
+tablename = CONFIG['DB_CALIB_TBL']
+pathcal = CONFIG['CALIB_DIR']
+
+def connect_db_cal():
+    dbcal = sqlite3.connect(pathdb)
+    return dbcal
+
+def column_names(dbcalib, tablename):
+    """
+    Return the names of the columns in the database.
+    """
+    dbcalib.row_factory = sqlite3.Row
+    cursor = dbcalib.execute("SELECT * FROM {table}"\
+                .format(table=tablename,))
+    row = cursor.fetchone()
+    names = row.keys()
+    return names
+
+
+def check_db(dbcalib, tablename, columns, calname):
+    """
+    Return True if the file does not exist in the database.
+    """
+
+    cursor = dbcalib.cursor()
+
+    for col in columns:
+        #print col
+        try:
+            cursor.execute("SELECT * FROM {table} WHERE instr({fpath},?) > 0"\
+                .format(table=tablename,fpath=col),(calname,))
+            found = cursor.fetchone()[0]
+
+            if found:
+                return False
+        except:
+            pass
+
+    return True
+
+
+if __name__ == "__main__":
+# list the calibration files
+    names = os.path.join(pathcal,"*.fits")
+    cals = glob.glob(names)
+    cals.sort()
+
+    dbcal = connect_db_cal()
+    columns = column_names(dbcal, tablename)
+
+    while cals:
+        cal = os.path.basename(cals[0])
+        check = check_db(dbcal, tablename, columns, cal)
+
+        if check:
+            os.remove(cals[0])
+
+        cals.pop(0)
+    dbcal.close()
+

drslib/config.py → gofio/drslib/config.py

@@ -5,6 +5,7 @@ Implemented in the GIANOB DRS by Monica Rainer
 
 import os, tempfile, datetime
 from collections import OrderedDict
+from __version__ import __version__
 
 
 def getdate(date_time=None):
@@ -57,7 +58,8 @@ else:
 
 CONFIG = {}
 CONFIG['APPNAME'] = 'GOFIO'
-CONFIG['VERSION'] = 1.0
+#CONFIG['VERSION'] = 1.0
+CONFIG['VERSION'] = __version__
 CONFIG['DATE'] = getdate()
 CONFIG['OFFLINE'] = False
 
@@ -65,21 +67,20 @@ CONFIG['OFFLINE'] = False
 # Directories 
 #
 
-CONFIG['USR_DIR'] = os.environ.get('HOME')
 CONFIG['TMP_DIR'] = tempfile.gettempdir()
 CONFIG['APP_DIR'] = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
-CONFIG['BASE_RAW'] = '/mnt/nfs/RAW'
-#CONFIG['BASE_RAW'] = '/home/monica/Documenti/gianob/gianobdrs/varie/commissioning_marzo/raw'
-#CONFIG['BASE_RED_DIR'] = os.path.join(CONFIG['APP_DIR'],'reduced')
-CONFIG['BASE_RED_DIR'] = '/mnt/nfs/REDUCED'
+#CONFIG['BASE_RAW'] = '/mnt/nfs/RAW'
+CONFIG['BASE_RAW'] = '/home/monica/Documenti/gianob/gianobdrs/varie/commissioning_marzo/raw'
+CONFIG['BASE_RED_DIR'] = os.path.join(CONFIG['APP_DIR'],'reduced')
+#CONFIG['BASE_RED_DIR'] = '/mnt/nfs/REDUCED'
 
 CONFIG['BASE_RAMP'] = '/home/avet/gianorp/databases'
 
 
 CONFIG['RAW_DIR'] = os.path.join(CONFIG['BASE_RAW'],CONFIG['DATE'])
-CONFIG['OFFLINE_DIR'] = False
 CONFIG['RED_DIR'] = os.path.join(CONFIG['BASE_RED_DIR'],CONFIG['DATE'])
+CONFIG['OFFLINE_DIR'] = False
 CONFIG['RED_CALIB'] = os.path.join(CONFIG['RED_DIR'],'CALIB')
 CONFIG['RED_STR'] = os.path.join(CONFIG['RED_DIR'],'STR')
 
@@ -90,8 +91,8 @@ CONFIG['BADPIX_MASK']  = os.path.join(CONFIG['RES_DIR'], 'badpix_mask.fits')
 CONFIG['LOGGER'] = {'TERM':True,'FILE':True} # logger writes to terminal and/or to file
 CONFIG['LOG_FILE'] = os.path.join(CONFIG['RED_DIR'], 'drs.log')
 CONFIG['STRAIGHT'] = os.path.join(CONFIG['RES_DIR'],'straight_giano_2D')
-#CONFIG['STRAIGHT_OPT'] = ['I=1']
-CONFIG['STRAIGHT_OPT'] = ['I=1','DY=-5']
+CONFIG['STRAIGHT_OPT'] = ['I=1'] # the option DY is computed automatically
+#CONFIG['STRAIGHT_OPT'] = ['I=1','DY=-5'] # define DY only if you want to override the automatic procedure
 CONFIG['BASE_CALIB_DIR']    = os.path.join(CONFIG['APP_DIR'], 'calibrations')
 CONFIG['CALIB_DIR']    = os.path.join(CONFIG['BASE_CALIB_DIR'], 'calfiles')
 CONFIG['CALIB_DB_DIR']    = os.path.join(CONFIG['BASE_CALIB_DIR'], 'database')
@@ -117,6 +118,7 @@ CONFIG['DARK'] = 'DARK'
 CONFIG['FLAT'] = 'FLAT'
 CONFIG['WCAL_UNE'] = 'WCAL_UNE'
 CONFIG['WCAL_FP'] = 'WCAL_FP'
+#CONFIG['WCAL_FP'] = 'WCAL_LFC'
 CONFIG['SCIENCE'] = 'SCIENCE'
 # CONFIG['NODSTARE'] keyword values to determine if stare or nodding
 CONFIG['NODVALUE'] = 'GIANO_sci_nodding'
@@ -163,7 +165,7 @@ CONFIG['WLFIT'] = (' '.join((CONFIG['KEY_DRS'],'CAL WLFUNC')),'Function used for
 
 if CONFIG['CAL_FUNC']['Oliva']:
     CONFIG['WLFIT_FUNC'] = 'l0 + k1*(x-xc) + k2*(x-xc)**2 + k3*(x-xc)**3'
-    #CONFIG['WLCOEFFS'] = {'k1':(' '.join((CONFIG['KEY_DRS'],'CAL K1')),'k1 coefficient of the calibration function'),'k2':(' '.join((CONFIG['KEY_DRS'],'CAL K2')),'k2 coefficient of the calibration function'),'k3':(' '.join((CONFIG['KEY_DRS'],'CAL K3')),'k3 coefficient of the calibration function'),'l0':(' '.join((CONFIG['KEY_DRS'],'CAL L0')),'l0 coefficient of the calibration function'),'xc':(' '.join((CONFIG['KEY_DRS'],'CAL XC')),'xc coefficient of the calibration function'),'rms':(' '.join((CONFIG['KEY_DRS'],'CAL RMSE')),'[m/s] RMS error of the calibration')}
+
 else:
     CONFIG['WLFIT_FUNC'] = 'c0 + c1*x + c2*x**2 + c3*x**3'
 
@@ -183,14 +185,13 @@ CONFIG['DB_CALIB'] = 'drs_calib.db'
 CONFIG['DB_CALIB_PATH'] = os.path.join(CONFIG['CALIB_DB_DIR'], CONFIG['DB_CALIB'])
 CONFIG['DB_CALIB_TBL'] = 'calibrations'
 
-#CONFIG['DB_CALIB_COLS'] = OrderedDict([('01_data', 'date_calib'), ('dark10', 'masterdark10'), ('dark30', 'masterdark30'), ('dark60', 'masterdark60'), ('dark100', 'masterdark100'), ('flat', 'masterflat'), ('flatstr', 'masterflat_str'), ('flatnor', 'masterflat_nor'), ('une', 'wl_une'), ('une_str', 'wl_une_str'), ('une_calib', 'wl_une_calib'), ('fp', 'wl_fp'), ('fp_str', 'wl_fp_str'), ('fp_calib','wl_fp_calib')])
-
-CONFIG['DB_CALIB_COLS'] = OrderedDict([('01_data', 'date_calib'), ('dark10', 'masterdark10'), ('dark30', 'masterdark30'), ('dark60', 'masterdark60'), ('dark100', 'masterdark100'),  ('dark200', 'masterdark200'), ('dark300', 'masterdark300'), ('dark600', 'masterdark600'),('flat', 'masterflat'), ('flatstr', 'masterflat_str'), ('flatnor', 'masterflat_nor'), ('une', 'wl_une'), ('une_str', 'wl_une_str'), ('une_calib', 'wl_une_calib'), ('fp', 'wl_fp'), ('fp_str', 'wl_fp_str'), ('fp_calib','wl_fp_calib')])
+CONFIG['DB_CALIB_COLS'] = OrderedDict([('01_data', 'date_calib'), ('dark10', 'masterdark10'), ('dark30', 'masterdark30'), ('dark60', 'masterdark60'), ('dark100', 'masterdark100'),  ('dark200', 'masterdark200'), ('dark300', 'masterdark300'), ('dark600', 'masterdark600'),('flat', 'masterflat'), ('flatstr', 'masterflat_str'), ('flatnor', 'masterflat_nor'), ('shiftY', 'shift_Y') ,('une', 'wl_une'), ('une_str', 'wl_une_str'), ('une_calib', 'wl_une_calib'), ('fp', 'wl_fp'), ('fp_str', 'wl_fp_str'), ('fp_calib','wl_fp_calib')])
 
 CONFIG['DB_CALIB_DATATYPE'] = OrderedDict()
 for colname in CONFIG['DB_CALIB_COLS']:
     CONFIG['DB_CALIB_DATATYPE'][colname] = 'TEXT' 
 CONFIG['DB_CALIB_DATATYPE']['01_data'] = 'TEXT unique'
+CONFIG['DB_CALIB_DATATYPE']['shiftY'] = 'INTEGER'
 
 # Night reduction database
 
@@ -207,12 +208,10 @@ CONFIG['DB_2D_COLS'] = OrderedDict([('names', 'spec_2d_names'), ('stamp', 'proce
 CONFIG['DB_1D_COLS'] = OrderedDict([('path', 'spec_1d_path'), ('stamp', 'process_timestamp'), ('id', 'spec_1d_id'), ('type', 'spec_1d_type'), ('stat', 'arch_stat'), ('name', 'objname'), ('slit', 'slitpos'), ('snry', 'snr1'), ('snrj', 'snr2'), ('snrh', 'snr3'), ('snrk', 'snr4')])
 CONFIG['DB_CAL_COLS'] = OrderedDict([('type', 'calib_type'), ('names', 'calib_names'), ('status', 'calib_status'), ('stamp', 'process_timestamp')])
 
-
 CONFIG['DB_2D_DATATYPE'] = {'names': 'TEXT unique', 'stamp': 'REAL'}
 CONFIG['DB_1D_DATATYPE'] = {'path': 'TEXT unique', 'stamp': 'REAL', 'id': 'TEXT', 'type': 'TEXT', 'stat': 'INTEGER', 'name': 'TEXT', 'slit': 'TEXT', 'snry': 'REAL', 'snrj': 'REAL', 'snrh': 'REAL', 'snrk': 'REAL'}
 CONFIG['DB_CAL_DATATYPE'] = {'type': 'TEXT', 'names': 'TEXT unique', 'status': 'TEXT', 'stamp': 'REAL'}
 
-
 CONFIG['DB_NIGHT'] = CONFIG['DB_ONLINE']
 CONFIG['DB_NIGHT_PATH'] = CONFIG['DB_ONLINE_PATH']
 
@@ -224,6 +223,7 @@ CONFIG['MASK_C'] = os.path.join(CONFIG['RES_DIR'], 'GIANOB_MASKC.fits')
 CONFIG['A_POS'] = 8. # guess pixel position for A
 CONFIG['B_POS'] = 23. # guess pixel position for B
 CONFIG['C_POS'] = 15. # guess pixel position for C
+CONFIG['Y_POS'] = 4 # max. pixel difference between expected and real pixel position
 CONFIG['HWTM'] = 6. # hwtm limit for contamination between A and B
 CONFIG['S1D'] = True # flag for creation of s1d output (True/False)
 CONFIG['S1D_NORM'] = False # flag for normalization of s1d output (True/False)
@@ -248,16 +248,13 @@ CONFIG['DO_CALIB'] = {'dark':True, 'flat':True, 'une':True, 'fp':True, 'only_cal
 
 CONFIG['USE_FLAT'] = {'global':False, 'order':True, 'nor':False} # global: flat normalized by global mean value - order: flat normalized order by order by mean value - nor: flat normalized row by row by continuum fitting, removing the blaze function - No True value: no flat division
 
-CONFIG['FLAT_EXPT'] = 100.0
-CONFIG['WLCAL_EXPT'] = 100.0
-CONFIG['FP_EXPT'] = 60.0
 CONFIG['DARKLIST'] = {10:[],30:[],60:[],100:[],200:[],300:[],600:[]} # exptime dei dark
 CONFIG['NDARK'] = len(CONFIG['DARKLIST']) # quanti tipi di dark (esempio: 10,30,60,100 sec = 4) meglio sovrastimare
 
-CONFIG['WEXT'] = 5 # extraction half-window for wavelength calibration, it will extract 2*wext around the center of the order
+CONFIG['WEXT'] = 10 # extraction half-window for wavelength calibration, it will extract 2*wext around the center of the order
 
 #
-# CCD info
+# Detector info
 #
 
 CONFIG['RON'] = 5.0 # electrons
@@ -266,6 +263,7 @@ CONFIG['XCCD'] = 2048
 CONFIG['YCCD'] = 2048
 CONFIG['N_ORD'] = 50 # number of echelle orders
 CONFIG['W_ORD'] = 41 # width of straighten order in pixel
+CONFIG['SHIFT_Y'] = 20 # expected bottom position of flat signal on the lowest order and the left side of the detector. DO NOT CHANGE.
 
 #
 # Quality checks
@@ -297,4 +295,7 @@ if USRCONFIG['OFFLINE']:
     CONFIG['DB_NIGHT'] = CONFIG['DB_OFFLINE']
     CONFIG['DB_NIGHT_PATH'] = CONFIG['DB_OFFLINE_PATH']
 
+    maskc = ''.join(('GIANOB_MASKC_',CONFIG['DATE'],'.fits'))
+    CONFIG['MASK_C'] = os.path.join(CONFIG['RES_DIR'], maskc)
+
 

drslib/flatframes.py → gofio/drslib/flatframes.py

@@ -78,6 +78,7 @@ class GBFlats():
         using ron and gain, correct ron and gain --> ron_eff and gain_eff
         Write the result in the database.
         """
+
         if self.flatlist:
             exptime = self.flatlist[0].header[CONFIG['KEYS']['EXPTIME']]
             darkname = 'dark' + str(int(exptime))
@@ -117,7 +118,7 @@ class GBFlats():
 
             for flat in self.flatlist:
 
-                # subdtract the masterdak
+                # subdtract the masterdark
                 if use_dark:
                     subflat = ccdproc.subtract_dark(flat,mdark,exposure_time=CONFIG['KEYS']['EXPTIME'],exposure_unit=u.second)
                     self.messages.append('Masterdark subtracted.')
@@ -132,12 +133,16 @@ class GBFlats():
 
                 self.flatcorr.append(flat_corr)
 
-            combineflat = ccdproc.Combiner(self.flatcorr)
+            if len(self.flatcorr) > 1:
+
+                combineflat = ccdproc.Combiner(self.flatcorr)
 
-            # mask the pixels using ron/gain
-            #combineflat.sigma_clipping(func=np.ma.mean, dev_func=varie.stdcombine)
-            combineflat.sigma_clipping(func=np.ma.median, dev_func=varie.stdcombine)
-            mflat = combineflat.average_combine()
+                # mask the pixels using ron/gain
+                #combineflat.sigma_clipping(func=np.ma.mean, dev_func=varie.stdcombine)
+                combineflat.sigma_clipping(func=np.ma.median, dev_func=varie.stdcombine)
+                mflat = combineflat.average_combine()
+            else:
+                mflat = self.flatcorr[0]
 
             mflat.data = np.asarray(mflat.data, dtype='float32')
 
@@ -156,19 +161,44 @@ class GBFlats():
             masterflat = fits.HDUList([hdu])
             masterflat.writeto(nome,clobber=True)
 
-            mflat = None
-
             db.insert_dbfile(self.dbconn,'flat',nome)
             self.messages.append('Masterflat %s was created and inserted in the calibration database' % (str(os.path.basename(nome))))
 
+            shiftY = varie.shiftY(mflat.data)
+
+            db.insert_dbfile(self.dbconn,'shiftY',shiftY)
+
+            mflat = None
+
         else:
             self.messages.append('There are no available flats. The masterflat will be taken from the calibration database.')
             if db.check_dbfile(self.dbconn,'flat'):
                 try:
                     db.copy_dbfile(self.dbconn,'flat')
+                    try:
+                        db.copy_dbfile(self.dbconn,'shiftY')
+                        shift = db.extract_dbfile(self.dbconn,'shiftY')
+                        if not shift:
+                            cal_flat = db.extract_dbfile(self.dbconn,'flat')
+                            mflat = ccdproc.CCDData.read(cal_flat, unit=u.adu)
+                            shiftY = varie.shiftY(mflat.data)
+                            db.insert_dbfile(self.dbconn,'shiftY',shiftY)
+
+                    except:
+                        cal_flat = db.extract_dbfile(self.dbconn,'flat')
+                        mflat = ccdproc.CCDData.read(cal_flat, unit=u.adu)
+                        shiftY = varie.shiftY(mflat.data)
+                        db.insert_dbfile(self.dbconn,'shiftY',shiftY)
                 except:
                     self.messages.append('There are no flats in the calibration database.')
                     return False
+
+            if db.check_dbfile(self.dbconn,'shiftY'):
+                cal_flat = db.extract_dbfile(self.dbconn,'flat')
+                mflat = ccdproc.CCDData.read(cal_flat, unit=u.adu)
+                shiftY = varie.shiftY(mflat.data)
+                db.insert_dbfile(self.dbconn,'shiftY',shiftY)
+
         return True
 
 
@@ -185,9 +215,32 @@ class GBFlats():
         nor = flat.replace('.fits','_nor.fits')
         #extracted = flat.replace('.fits','_ext.fits')
 
-        #args = [CONFIG['STRAIGHT'],flat,straight,CONFIG['STRAIGHT_OPT']]
         args = [CONFIG['STRAIGHT'],flat,straight]
         args.extend(CONFIG['STRAIGHT_OPT'])
+        # search for shift defined in the straighten options in config.py
+        dy = True
+        for opt in CONFIG['STRAIGHT_OPT']:
+            try:
+                dy = opt.rindex('DY=')
+                ypos = int(opt[dy-2:])
+                shift = CONFIG['SHIFT_Y'] + ypos
+                dy = False
+            except:
+                pass
+
+        if dy:
+            shift = db.extract_dbfile(self.dbconn,'shiftY')
+            if not shift:
+                cal_flat = db.extract_dbfile(self.dbconn,'flat')
+                mflat = ccdproc.CCDData.read(cal_flat, unit=u.adu)
+                shift = varie.shiftY(mflat.data)
+                db.insert_dbfile(self.dbconn,'shiftY',shift)
+
+            shiftY = [''.join(('DY=',str(shift - CONFIG['SHIFT_Y'])))]
+            #print shiftY
+            args.extend(shiftY)
+
+        #print args
         subprocess.call(args)
 
         red_straight = os.path.join(CONFIG['RED_CALIB'],os.path.basename(straight))

gofio/drslib/gofio.py

+import os, sys, time
+
+from drslib.config import CONFIG, read_usr_config
+from drslib.logger import DrsLogger
+
+from drslib import db, rawfiles, darkframes, flatframes, wlframes, nodding, stare
+
+
+def gofio_init(docopt_args):
+
+    if docopt_args['--cfg']:
+        try:
+            USRCONFIG = read_usr_config(docopt_args['-g'])
+        except:
+            try:
+                USRCONFIG = read_usr_config(docopt_args['--cfg'])
+            except:
+                USRCONFIG = {}
+                USRCONFIG['OFFLINE'] = False
+
+
+        if USRCONFIG['OFFLINE']:
+            for key in USRCONFIG.keys():
+                CONFIG[key] = USRCONFIG[key]
+
+            CONFIG['RAW_DIR'] = os.path.join(CONFIG['BASE_RAW'],CONFIG['DATE'])
+            CONFIG['OFFLINE_DIR'] = os.path.join(CONFIG['BASE_RED_DIR'],CONFIG['DATE'])
+            CONFIG['RED_DIR'] = os.path.join(CONFIG['OFFLINE_DIR'],'offline')
+            CONFIG['RED_CALIB'] = os.path.join(CONFIG['RED_DIR'],'CALIB')
+            CONFIG['RED_STR'] = os.path.join(CONFIG['RED_DIR'],'STR')
+            CONFIG['LOG_FILE'] = os.path.join(CONFIG['RED_DIR'], 'drs.log')
+            CONFIG['RAMP_NAME'] = 'ramps' + CONFIG['DATE'] + '.db'
+            CONFIG['DB_RAMP'] = os.path.join(CONFIG['BASE_RAMP'],CONFIG['RAMP_NAME'])
+
+            CONFIG['DB_OFFLINE'] = ''.join(('db_',CONFIG['DATE'],'_offline.db'))
+            CONFIG['DB_OFFLINE_PATH'] = os.path.join(CONFIG['WEBUI_DB_DIR'], CONFIG['DB_OFFLINE'])
+            CONFIG['DB_NIGHT'] = CONFIG['DB_OFFLINE']
+            CONFIG['DB_NIGHT_PATH'] = CONFIG['DB_OFFLINE_PATH']
+
+            maskc = ''.join(('GIANOB_MASKC_',CONFIG['DATE'],'.fits'))
+            CONFIG['MASK_C'] = os.path.join(CONFIG['RES_DIR'], maskc)
+
+    if docopt_args['<date>']:
+        if docopt_args['--all_calib']:
+            docopt_args['--dark'] = True
+            docopt_args['--flat'] = True
+            docopt_args['--une'] = True
+            docopt_args['--fp'] = True
+
+        CONFIG['DATE'] = docopt_args['<date>']
+        CONFIG['OFFLINE'] = True
+        CONFIG['RAW_DIR'] = os.path.join(CONFIG['BASE_RAW'],CONFIG['DATE'])
+        CONFIG['OFFLINE_DIR'] = os.path.join(CONFIG['BASE_RED_DIR'],CONFIG['DATE'])
+        CONFIG['RED_DIR'] = os.path.join(CONFIG['OFFLINE_DIR'],'offline')
+        CONFIG['RED_CALIB'] = os.path.join(CONFIG['RED_DIR'],'CALIB')
+        CONFIG['RED_STR'] = os.path.join(CONFIG['RED_DIR'],'STR')
+        CONFIG['LOG_FILE'] = os.path.join(CONFIG['RED_DIR'], 'drs.log')
+        CONFIG['DO_CALIB'] = {'dark': docopt_args['--dark'], 'flat': docopt_args['--flat'], 'une': docopt_args['--une'], 'fp': docopt_args['--fp'], 'only_calib': docopt_args['--only_calib']}
+ 
+        if docopt_args['--use_flat'] == 'global':
+            CONFIG['USE_FLAT'] = {'global':True, 'order':False, 'nor':False}
+        elif docopt_args['--use_flat'] == 'order':
+            CONFIG['USE_FLAT'] = {'global':False, 'order':True, 'nor':False} 
+        elif docopt_args['--use_flat'] == 'nor':
+            CONFIG['USE_FLAT'] = {'global':False, 'order':False, 'nor':True}
+        else:
+            CONFIG['USE_FLAT'] = {'global':False, 'order':False, 'nor':False}
+
+        if docopt_args['--s1d'] == 'norm':
+            CONFIG['S1D'] = True
+            CONFIG['S1D_NORM'] = True
+        elif docopt_args['--s1d'] == 'no':
+            CONFIG['S1D'] = False
+            CONFIG['S1D_NORM'] = False
+        else:
+            CONFIG['S1D'] = True
+            CONFIG['S1D_NORM'] = False
+
+
+
+        maskc = ''.join(('GIANOB_MASKC_',CONFIG['DATE'],'.fits'))
+        CONFIG['MASK_C'] = os.path.join(CONFIG['RES_DIR'], maskc)
+
+        CONFIG['DB_OFFLINE'] = ''.join(('db_',CONFIG['DATE'],'_offline.db'))
+        CONFIG['DB_OFFLINE_PATH'] = os.path.join(CONFIG['WEBUI_DB_DIR'], CONFIG['DB_OFFLINE'])
+        CONFIG['DB_NIGHT'] = CONFIG['DB_OFFLINE']
+        CONFIG['DB_NIGHT_PATH'] = CONFIG['DB_OFFLINE_PATH']
+
+
+    # second argument: how to query the calibration database
+    # to be set if the the user wants to reduce the night using 
+    # another night's calibrations
+    if docopt_args['<calib_date>']:
+        CONFIG['DATE'] = docopt_args['<calib_date>']
+
+
+
+    # remove the mask with the order position (if it exists)
+    # it will be re-build with the correct flat-field
+    try:
+        os.remove(CONFIG['MASK_C'])
+    except:
+        pass
+
+    # Create the reduction directory
+    try:
+        os.mkdir(CONFIG['BASE_RED_DIR'])
+    except OSError:
+        pass
+
+    if CONFIG['OFFLINE']:
+        try:
+            os.mkdir(CONFIG['OFFLINE_DIR'])
+        except OSError:
+            pass
+
+    try:
+        os.mkdir(CONFIG['RED_DIR'])
+    except OSError:
+        pass
+
+    try:
+        os.mkdir(CONFIG['RED_CALIB'])
+        os.mkdir(CONFIG['RED_STR'])
+    except OSError:
+        pass
+
+
+    # Create the calibration directory tree
+    try:
+        os.mkdir(CONFIG['BASE_CALIB_DIR'])
+    except OSError:
+        pass
+
+    try:
+        os.mkdir(CONFIG['CALIB_DIR'])
+    except OSError:
+        pass
+
+
+    # initialize logger
+    drslogger = DrsLogger()
+
+    # Databases online/offline
+
+    dbcalib = db.create_db_calib()
+
+    if CONFIG['OFFLINE']:
+        dbramps = False
+        try:
+            os.remove(CONFIG['DB_NIGHT_PATH'])
+        except:
+            pass
+    else:
+        dbramps = db.connect_db_ramp()
+
+    dbnight = db.create_db_night()
+
+    if not os.path.exists(CONFIG['RAW_DIR']):
+        drslogger.log('The RAW directory %s does not exist. GOFIO will now close down.' % CONFIG['RAW_DIR'])
+        drslogger.log('Pipeline stopped.')
+        dbcalib.close()
+        if not CONFIG['OFFLINE']:
+            dbramps.close()
+            dbnight.close()