Change structure, add gofio.py

.gitignore

 backup
-calibrations
 git_howto.txt
-gofio_latest
-gofio_latest.tar
+gofioDRS.py
 reduced
 varie
 vradlib
@@ -18,4 +16,5 @@ webuidatabases
 */*/*/*.pyc
 *~
 .~*
+gofio/calibrations
 

README.md

@@ -7,6 +7,7 @@ Python packages:
  - NumPy v1.12
  - SciPy v0.19
  - watchdog v0.8.2
+ - docopt v0.6.1
 
 Output:
  - intermediate: *_str.fits, 2D images with orders straightened and bad pixel removed

drslib/config.py → gofio/drslib/config.py

@@ -220,6 +220,7 @@ CONFIG['MASK_C'] = os.path.join(CONFIG['RES_DIR'], 'GIANOB_MASKC.fits')
 CONFIG['A_POS'] = 8. # guess pixel position for A
 CONFIG['B_POS'] = 23. # guess pixel position for B
 CONFIG['C_POS'] = 15. # guess pixel position for C
+CONFIG['Y_POS'] = 2 # max. pixel difference between expected and real pixel position
 CONFIG['HWTM'] = 6. # hwtm limit for contamination between A and B
 CONFIG['S1D'] = True # flag for creation of s1d output (True/False)
 CONFIG['S1D_NORM'] = False # flag for normalization of s1d output (True/False)

gofio/drslib/gofio.py

+import os, sys, time
+
+from drslib.config import CONFIG, read_usr_config
+from drslib.logger import DrsLogger
+
+from drslib import db, rawfiles, darkframes, flatframes, wlframes, nodding, stare
+
+
+def gofio_init(docopt_args):
+
+    if docopt_args['--cfg']:
+        try:
+            USRCONFIG = read_usr_config(docopt_args['-g'])
+        except:
+            try:
+                USRCONFIG = read_usr_config(docopt_args['--cfg'])
+            except:
+                USRCONFIG = {}
+                USRCONFIG['OFFLINE'] = False
+
+
+        if USRCONFIG['OFFLINE']:
+            for key in USRCONFIG.keys():
+                CONFIG[key] = USRCONFIG[key]
+
+            CONFIG['RAW_DIR'] = os.path.join(CONFIG['BASE_RAW'],CONFIG['DATE'])
+            CONFIG['OFFLINE_DIR'] = os.path.join(CONFIG['BASE_RED_DIR'],CONFIG['DATE'])
+            CONFIG['RED_DIR'] = os.path.join(CONFIG['OFFLINE_DIR'],'offline')
+            CONFIG['RED_CALIB'] = os.path.join(CONFIG['RED_DIR'],'CALIB')
+            CONFIG['RED_STR'] = os.path.join(CONFIG['RED_DIR'],'STR')
+            CONFIG['LOG_FILE'] = os.path.join(CONFIG['RED_DIR'], 'drs.log')
+            CONFIG['RAMP_NAME'] = 'ramps' + CONFIG['DATE'] + '.db'
+            CONFIG['DB_RAMP'] = os.path.join(CONFIG['BASE_RAMP'],CONFIG['RAMP_NAME'])
+
+            CONFIG['DB_OFFLINE'] = ''.join(('db_',CONFIG['DATE'],'_offline.db'))
+            CONFIG['DB_OFFLINE_PATH'] = os.path.join(CONFIG['WEBUI_DB_DIR'], CONFIG['DB_OFFLINE'])
+            CONFIG['DB_NIGHT'] = CONFIG['DB_OFFLINE']
+            CONFIG['DB_NIGHT_PATH'] = CONFIG['DB_OFFLINE_PATH']
+
+            maskc = ''.join(('GIANOB_MASKC_',CONFIG['DATE'],'.fits'))
+            CONFIG['MASK_C'] = os.path.join(CONFIG['RES_DIR'], maskc)
+
+    if docopt_args['<date>']:
+        CONFIG['DATE'] = docopt_args['<date>']
+        CONFIG['OFFLINE'] = True
+        CONFIG['RAW_DIR'] = os.path.join(CONFIG['BASE_RAW'],CONFIG['DATE'])
+        CONFIG['OFFLINE_DIR'] = os.path.join(CONFIG['BASE_RED_DIR'],CONFIG['DATE'])
+        CONFIG['RED_DIR'] = os.path.join(CONFIG['OFFLINE_DIR'],'offline')
+        CONFIG['RED_CALIB'] = os.path.join(CONFIG['RED_DIR'],'CALIB')
+        CONFIG['RED_STR'] = os.path.join(CONFIG['RED_DIR'],'STR')
+        CONFIG['LOG_FILE'] = os.path.join(CONFIG['RED_DIR'], 'drs.log')
+        CONFIG['DO_CALIB'] = {'dark': docopt_args['--dark'], 'flat': docopt_args['--flat'], 'une': docopt_args['--une'], 'fp': docopt_args['--fp'], 'only_calib': docopt_args['--only_calib']}
+ 
+        if docopt_args['--use_flat'] == 'global':
+            CONFIG['USE_FLAT'] = {'global':True, 'order':False, 'nor':False}
+        elif docopt_args['--use_flat'] == 'order':
+            CONFIG['USE_FLAT'] = {'global':False, 'order':True, 'nor':False} 
+        elif docopt_args['--use_flat'] == 'nor':
+            CONFIG['USE_FLAT'] = {'global':False, 'order':False, 'nor':True}
+        else:
+            CONFIG['USE_FLAT'] = {'global':False, 'order':False, 'nor':False}
+
+        maskc = ''.join(('GIANOB_MASKC_',CONFIG['DATE'],'.fits'))
+        CONFIG['MASK_C'] = os.path.join(CONFIG['RES_DIR'], maskc)
+
+        CONFIG['DB_OFFLINE'] = ''.join(('db_',CONFIG['DATE'],'_offline.db'))
+        CONFIG['DB_OFFLINE_PATH'] = os.path.join(CONFIG['WEBUI_DB_DIR'], CONFIG['DB_OFFLINE'])
+        CONFIG['DB_NIGHT'] = CONFIG['DB_OFFLINE']
+        CONFIG['DB_NIGHT_PATH'] = CONFIG['DB_OFFLINE_PATH']
+
+
+    # second argument: how to query the calibration database
+    # to be set if the the user wants to reduce the night using 
+    # another night's calibrations
+    if docopt_args['<calib_date>']:
+        CONFIG['DATE'] = docopt_args['<calib_date>']
+
+
+
+    # remove the mask with the order position (if it exists)
+    # it will be re-build with the correct flat-field
+    try:
+        os.remove(CONFIG['MASK_C'])
+    except:
+        pass
+
+    # Create the reduction directory
+    try:
+        os.mkdir(CONFIG['BASE_RED_DIR'])
+    except OSError:
+        pass
+
+    if CONFIG['OFFLINE']:
+        try:
+            os.mkdir(CONFIG['OFFLINE_DIR'])
+        except OSError:
+            pass
+
+    try:
+        os.mkdir(CONFIG['RED_DIR'])
+    except OSError:
+        pass
+
+    try:
+        os.mkdir(CONFIG['RED_CALIB'])
+        os.mkdir(CONFIG['RED_STR'])
+    except OSError:
+        pass
+
+
+    # Create the calibration directory tree
+    try:
+        os.mkdir(CONFIG['BASE_CALIB_DIR'])
+    except OSError:
+        pass
+
+    try:
+        os.mkdir(CONFIG['CALIB_DIR'])
+    except OSError:
+        pass
+
+
+    # initialize logger
+    drslogger = DrsLogger()
+
+    # Databases online/offline
+
+    dbcalib = db.create_db_calib()
+
+    if CONFIG['OFFLINE']:
+        dbramps = False
+        try:
+            os.remove(CONFIG['DB_NIGHT_PATH'])
+        except:
+            pass
+    else:
+        dbramps = db.connect_db_ramp()
+
+    dbnight = db.create_db_night()
+
+    if not os.path.exists(CONFIG['RAW_DIR']):
+        drslogger.log('The RAW directory %s does not exist. GOFIO will now close down.' % CONFIG['RAW_DIR'])
+        drslogger.log('Pipeline stopped.')
+        dbcalib.close()
+        if not CONFIG['OFFLINE']:
+            dbramps.close()
+            dbnight.close()
+        sys.exit(0)
+
+    rawlists = {CONFIG['DARK']:[],CONFIG['FLAT']:[],CONFIG['WCAL_UNE']:[],CONFIG['WCAL_FP']:[],CONFIG['SCIENCE']:[]}
+
+    # store the information on what the pipeline is working on (dark, flat, ...) and which science group/pair
+    working = 0
+    group = {}
+    stop = False
+
+    # --------------------------------
+    #     End initializing GOFIO
+    # --------------------------------
+
+    return CONFIG, rawlists, dbcalib, working, group, stop, drslogger, dbnight, dbramps
+
+
+
+# data reduction
+def GianoBreduce(rawfits,rawlists,dbcalib,working,group,do_calib,stop,drslogger, dbnight):
+
+    if stop:
+        try:
+            raw = rawfiles.GBCheckRawFiles(None,None,None,None,None)
+            group = raw.group
+            working = raw.working
+            if working == CONFIG['SCIENCE']:
+                if group['nodstare'] == CONFIG['NODVALUE'] and group['pair']:
+                    rawlists[CONFIG['SCIENCE']].pop()
+                if group:
+                    drslogger.log('Reducing pending frames.')
+                    print 'Press Ctrl-C again to stop the pipeline now.'
+                    raw.command[group['nodstare']] = 'incomplete_group'
+            elif working:
+                drslogger.log('Reducing pending %s frames.' % working)
+                print 'Press Ctrl-C again to stop the pipeline now.'
+                raw.command[working] = True
+            else:
+                return
+
+        except KeyboardInterrupt:
+            drslogger.log('Pipeline stopped.')
+            sys.exit(0)
+
+# check raw file and insert in the correct list
+
+    else:
+        drslogger.log('Checking the FITS file ' + str(os.path.basename(rawfits)))
+        raw = rawfiles.GBCheckRawFiles(rawfits,rawlists,working,group,do_calib)
+        raw.check_raw_file()
+        working = raw.working
+        group = raw.group
+        go = raw.go
+        #do_calib = raw.do_calib
+
+    for message in raw.messages:
+        drslogger.log(message)
+    #print group
+# see if the previous function passed a command and execute it
+    if raw.command[CONFIG['DARK']]:
+        drslogger.log('----------------------------------------------')
+        drslogger.log(' *** DARK FRAMES *** ')
+        stamp = time.time()
+        darks = darkframes.GBDarks(rawlists[CONFIG['DARK']],dbcalib)
+        darks.process()
+        for message in darks.messages:
+            drslogger.log(message)
+
+        db.insert_calib(dbnight, 'DARK', darks.darks, darks.quality, stamp)
+
+        rawlists[CONFIG['DARK']][:] = []
+        group.clear()
+        drslogger.log('----------------------------------------------')
+
+    elif raw.command[CONFIG['FLAT']]:
+        drslogger.log('----------------------------------------------')
+        drslogger.log(' *** FLAT-FIELD FRAMES *** ')
+        stamp = time.time()
+        flats = flatframes.GBFlats(rawlists[CONFIG['FLAT']],dbcalib)
+        flats.process()
+        for message in flats.messages:
+            drslogger.log(message)
+
+        db.insert_calib(dbnight, 'FLAT', flats.flats, flats.quality, stamp)
+
+        rawlists[CONFIG['FLAT']][:] = []
+        group.clear()
+        drslogger.log('----------------------------------------------')
+
+    elif raw.command[CONFIG['WCAL_UNE']]:
+        drslogger.log('----------------------------------------------')
+        drslogger.log(' *** UNE LAMP FRAMES *** ')
+        stamp = time.time()
+        wls = wlframes.GBWls(rawlists[CONFIG['WCAL_UNE']],dbcalib)
+        wls.une_process()
+        for message in wls.messages:
+            drslogger.log(message)
+
+        db.insert_calib(dbnight, 'WCAL_UNE', wls.wls, wls.quality, stamp)
+
+        rawlists[CONFIG['WCAL_UNE']][:] = []
+        group.clear()
+        drslogger.log('----------------------------------------------')
+
+    elif raw.command[CONFIG['WCAL_FP']]:
+        drslogger.log('----------------------------------------------')
+        drslogger.log(' *** FP LAMP FRAMES *** ')
+        stamp = time.time()
+        fps = wlframes.GBWls(rawlists[CONFIG['WCAL_FP']],dbcalib)
+        fps.fp_process()
+        for message in fps.messages:
+            drslogger.log(message)
+
+        db.insert_calib(dbnight, 'WCAL_FP', fps.wls, fps.quality, stamp)
+
+        rawlists[CONFIG['WCAL_FP']][:] = []
+        group.clear()
+        drslogger.log('----------------------------------------------')
+
+    elif raw.command[CONFIG['NODVALUE']] == 'pair':
+        drslogger.log('----------------------------------------------')
+        drslogger.log(' *** NODDING PAIR *** ')
+        t1 = time.time()
+        nods = nodding.GBNodding(rawlists[CONFIG['SCIENCE']],group,dbcalib,dbnight)
+        nods.pair_process()
+        for message in nods.messages:
+            drslogger.log(message)
+        t2 = time.time()
+        drslogger.log('  *** Reduction: %s s ***' % (str(round((t2-t1),2)), ))
+
+        rawlists[CONFIG['SCIENCE']][:] = []
+        drslogger.log('----------------------------------------------')
+
+    elif raw.command[CONFIG['STAREVALUE']] == 'pair':
+        drslogger.log('----------------------------------------------')
+        drslogger.log(' *** STARE PAIR *** ')
+        t1 = time.time()
+        stares = stare.GBStare(rawlists[CONFIG['SCIENCE']],group,dbcalib,dbnight)
+        stares.pair_process()
+        for message in stares.messages:
+            drslogger.log(message)
+        t2 = time.time()
+        drslogger.log('  *** Reduction: %s s ***' % (str(round((t2-t1),2)), ))
+
+        rawlists[CONFIG['SCIENCE']][:] = []
+        drslogger.log('----------------------------------------------')
+
+
+    elif raw.command[CONFIG['NODVALUE']] == 'group':
+        drslogger.log('----------------------------------------------')
+        drslogger.log(' *** NODDING GROUP *** ')
+        t1 = time.time()
+        nods = nodding.GBNodding(rawlists[CONFIG['SCIENCE']],group,dbcalib,dbnight)
+        nods.group_process()
+        for message in nods.messages:
+            drslogger.log(message)
+        t2 = time.time()
+        drslogger.log('  *** Reduction: %s s ***' % (str(round((t2-t1),2)), ))
+
+        rawlists[CONFIG['SCIENCE']][:] = []
+        drslogger.log('----------------------------------------------')
+
+    elif raw.command[CONFIG['STAREVALUE']] == 'group':
+        drslogger.log('----------------------------------------------')
+        drslogger.log(' *** STARE GROUP *** ')
+        t1 = time.time()
+        stares = stare.GBStare(rawlists[CONFIG['SCIENCE']],group,dbcalib,dbnight)
+        stares.group_process()
+        for message in stares.messages:
+            drslogger.log(message)
+        t2 = time.time()
+        drslogger.log('  *** Reduction: %s s ***' % (str(round((t2-t1),2)), ))
+
+        rawlists[CONFIG['SCIENCE']][:] = []
+        drslogger.log('----------------------------------------------')
+
+    elif raw.command[CONFIG['NODVALUE']] == 'incomplete_group':
+        drslogger.log('----------------------------------------------')
+        drslogger.log(' *** NODDING INCOMPLETE GROUP *** ')
+        t1 = time.time()
+        nods = nodding.GBNodding(rawlists[CONFIG['SCIENCE']],group,dbcalib,dbnight)
+        nods.ingroup_process()
+        for message in nods.messages:
+            drslogger.log(message)
+        t2 = time.time()   
+        drslogger.log('  *** Reduction: %s s ***' % (str(round((t2-t1),2)), ))
+
+        rawlists[CONFIG['SCIENCE']][:] = []
+        drslogger.log('----------------------------------------------')
+
+    elif raw.command[CONFIG['STAREVALUE']] == 'incomplete_group':
+        drslogger.log('----------------------------------------------')
+        drslogger.log(' *** STARE INCOMPLETE GROUP *** ')
+        t1 = time.time()
+        stares = stare.GBStare(rawlists[CONFIG['SCIENCE']],group,dbcalib,dbnight)
+        stares.ingroup_process()
+        for message in stares.messages:
+            drslogger.log(message) 
+        t2 = time.time()   
+        drslogger.log('  *** Reduction: %s s ***' % (str(round((t2-t1),2)), ))
+
+        rawlists[CONFIG['SCIENCE']][:] = []
+        drslogger.log('----------------------------------------------')
+    else:
+        pass
+
+    if stop:
+        return
+
+    #print group
+    return working, group, go, do_calib, rawlists
+

drslib/nodding.py → gofio/drslib/nodding.py

 """
-Last modified: 2017-03-07
-
 Reduction of the single AB nodding:
 - check the image quality (signal in well defined region)
 - check that the exposure times of A and B are the same

drslib/rawfiles.py → gofio/drslib/rawfiles.py

 """
-Last modified: 2017-03-08
-
 The FITS files are divided in darks, flats, UNes, FPs and science
 using the header keywords (defined in config.py).
 If all the images of one group have been taken, the command to
 process them is passed to the pipeline.
 If a different kind of image arrives, the previous ones are processed
 even if the group is incomplete.
-WARNING: the dark files are 4 times the expected number, because for
-each observations the ramp-processor creates 4 images with different
-exposure times (10, 30, 60 and 100 seconds).
+WARNING: the dark files are n times the expected number, because for
+each observations the ramp-processor creates n images with different
+exposure times (10, 30, 60, 100 seconds, and so on).
 """
 
 from drslib.config import CONFIG

drslib/stare.py → gofio/drslib/stare.py

 """
-Last modified: 2017-03-07
-
 Reduction of the single Obj-Sky group:
 - check the Obj image quality (signal in well defined region)
 - check that the exposure times of Obj and Sky are the same

drslib/varie.py → gofio/drslib/varie.py

 """
-Last modified: 2017-06-07
-
 - badpix: bad pixels removals
 - stdcombine: weights for flat and dark combiner
 - optExtract: optimal extraction
@@ -186,7 +184,7 @@ def optExtract(data,gain,ron,slit_pos,ordine):
 
     hwtm = math.sqrt(2*math.log(10)) * abs(sigmagauss) # half-width at tenth-maximum
 
-    if abs(x0 - slit_pos) > 2:
+    if abs(x0 - slit_pos) > CONFIG['Y_POS']:
         x0 = slit_pos
 
     # if bad seeing, there can be overlapping between A and B