Commit 2881c959 authored by Monica Rainer's avatar Monica Rainer
Browse files

Merge branch 'release/v1.2.0'

parents 0a182f07 2de7aff1
backup
calibrations
fringing
git_howto.txt
gofio_latest
gofio_latest.tar
gofioDRS.py
reduced
varie
vradlib
vrad_telluriche
DA_FARE.txt
GIARPS_GOFIO_v1.docx
GIARPS_GOFIO_v1.odt
GOFIO_*.*
patchfile
setup.py
vrad_telluric.py
......@@ -19,4 +17,7 @@ webuidatabases
*/*/*/*.pyc
*~
.~*
gofio/calibrations
gofio/resources/straight_giano_2D
gofio/resources/straight_giano_2D_v1_1_old.f
......@@ -7,8 +7,66 @@ Python packages:
- NumPy v1.12
- SciPy v0.19
- watchdog v0.8.2
- docopt v0.6.1
Other requirements:
- fortran77 compiler
- libcfitsio library
Installation:
- preserve the directory tree:
gofio/
__init__.py
__version__.py
gofioDRS.py
drslib/
__init__.py
clean_db.py
config.py
darkframes.py
db.py
flatframes.py
gofio.py
gofio2ascii.py
logger.py
nodding.py
rawfiles.py
stare.py
varie.py
wlframes.py
berv/
__init__.py
astroTimeLegacy.py
baryvel.py
resources/
GIANOB_MASKC.fits
UNe_lines_GIANO_selected.txt
UNe_observed_lines_GIANOB_18nov2016.txt
badpix_mask.fits
straight_giano_2D_v1_1.f
- compile the fortran77 straight_giano_2D_v1_1.f file:
gfortran -o straight_giano_2D straight_giano_2D_v1_1.f -lcfitsio
- modify the configuration file drslib/config.py in order to set the
right directory path for raw and reduced files (CONFIG['BASE_RAW']
and CONFIG['BASE_RED_DIR'])
- run "python gofioDRS.py -h" for help
Output:
- intermediate: *_str.fits, 2D images with orders straightened and bad pixel removed
- final output: *_ms1d.fits. The spectra are flat-fielded, extracted, calibrated, and the cosmic rays are removed. The echelle orders are not merged. The FITS table has 50 rows and 4 columns, and it stores each echelle order in a row. The columns are: the number of the echelle order (from 32 - the reddest - to 81 - the bluest), the wavelength in nanometers, the fluxes and the signal-to-noise ratios.
- final output (optional): *_s1d.fits. The spectra are flat-fielded, extracted, calibrated, and the cosmic rays are removed. The echelle orders are merged. The barycentric correction is applied. These are monodimensional spectra with constant step in wavelength.
- intermediate: *_str.fits, 2D images with orders straightened
and bad pixel removed
- final output: *_ms1d.fits. The spectra are flat-fielded, extracted,
calibrated, and the cosmic rays are removed.
The echelle orders are not merged.
The FITS table has 50 rows and 4 columns, and it stores each echelle
order in a row. The columns are: the number of the echelle order
(from 32 - the reddest - to 81 - the bluest), the wavelength in nanometers,
the fluxes and the signal-to-noise ratios.
- final output (optional): *_s1d.fits. The spectra are flat-fielded, extracted,
calibrated, and the cosmic rays are removed. The echelle orders are merged.
The barycentric correction is applied. These are monodimensional spectra with
constant step in wavelength.
The program gofio2ascii.py may be used to convert the outputs from FITS format
to ASCII files. It requires as input a text list with the filepaths of the
files to convert.
from __version__ import __version__
__version__ = '1.2.0'
"""
Remove files form calfiles that are not present in drs_calib.db
It must be run from the drslib directory with the command:
python clean_db.py
"""
import sqlite3, os, glob
from drslib.config import CONFIG
pathdb = CONFIG['DB_CALIB_PATH']
tablename = CONFIG['DB_CALIB_TBL']
pathcal = CONFIG['CALIB_DIR']
def connect_db_cal():
dbcal = sqlite3.connect(pathdb)
return dbcal
def column_names(dbcalib, tablename):
"""
Return the names of the columns in the database.
"""
dbcalib.row_factory = sqlite3.Row
cursor = dbcalib.execute("SELECT * FROM {table}"\
.format(table=tablename,))
row = cursor.fetchone()
names = row.keys()
return names
def check_db(dbcalib, tablename, columns, calname):
"""
Return True if the file does not exist in the database.
"""
cursor = dbcalib.cursor()
for col in columns:
#print col
try:
cursor.execute("SELECT * FROM {table} WHERE instr({fpath},?) > 0"\
.format(table=tablename,fpath=col),(calname,))
found = cursor.fetchone()[0]
if found:
return False
except:
pass
return True
if __name__ == "__main__":
# list the calibration files
names = os.path.join(pathcal,"*.fits")
cals = glob.glob(names)
cals.sort()
dbcal = connect_db_cal()
columns = column_names(dbcal, tablename)
while cals:
cal = os.path.basename(cals[0])
check = check_db(dbcal, tablename, columns, cal)
if check:
os.remove(cals[0])
cals.pop(0)
dbcal.close()
......@@ -5,6 +5,7 @@ Implemented in the GIANOB DRS by Monica Rainer
import os, tempfile, datetime
from collections import OrderedDict
from __version__ import __version__
def getdate(date_time=None):
......@@ -57,7 +58,8 @@ else:
CONFIG = {}
CONFIG['APPNAME'] = 'GOFIO'
CONFIG['VERSION'] = 1.0
#CONFIG['VERSION'] = 1.0
CONFIG['VERSION'] = __version__
CONFIG['DATE'] = getdate()
CONFIG['OFFLINE'] = False
......@@ -65,21 +67,20 @@ CONFIG['OFFLINE'] = False
# Directories
#
CONFIG['USR_DIR'] = os.environ.get('HOME')
CONFIG['TMP_DIR'] = tempfile.gettempdir()
CONFIG['APP_DIR'] = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
CONFIG['BASE_RAW'] = '/mnt/nfs/RAW'
#CONFIG['BASE_RAW'] = '/home/monica/Documenti/gianob/gianobdrs/varie/commissioning_marzo/raw'
#CONFIG['BASE_RED_DIR'] = os.path.join(CONFIG['APP_DIR'],'reduced')
CONFIG['BASE_RED_DIR'] = '/mnt/nfs/REDUCED'
#CONFIG['BASE_RAW'] = '/mnt/nfs/RAW'
CONFIG['BASE_RAW'] = '/home/monica/Documenti/gianob/gianobdrs/varie/commissioning_marzo/raw'
CONFIG['BASE_RED_DIR'] = os.path.join(CONFIG['APP_DIR'],'reduced')
#CONFIG['BASE_RED_DIR'] = '/mnt/nfs/REDUCED'
CONFIG['BASE_RAMP'] = '/home/avet/gianorp/databases'
CONFIG['RAW_DIR'] = os.path.join(CONFIG['BASE_RAW'],CONFIG['DATE'])
CONFIG['OFFLINE_DIR'] = False
CONFIG['RED_DIR'] = os.path.join(CONFIG['BASE_RED_DIR'],CONFIG['DATE'])
CONFIG['OFFLINE_DIR'] = False
CONFIG['RED_CALIB'] = os.path.join(CONFIG['RED_DIR'],'CALIB')
CONFIG['RED_STR'] = os.path.join(CONFIG['RED_DIR'],'STR')
......@@ -90,8 +91,8 @@ CONFIG['BADPIX_MASK'] = os.path.join(CONFIG['RES_DIR'], 'badpix_mask.fits')
CONFIG['LOGGER'] = {'TERM':True,'FILE':True} # logger writes to terminal and/or to file
CONFIG['LOG_FILE'] = os.path.join(CONFIG['RED_DIR'], 'drs.log')
CONFIG['STRAIGHT'] = os.path.join(CONFIG['RES_DIR'],'straight_giano_2D')
#CONFIG['STRAIGHT_OPT'] = ['I=1']
CONFIG['STRAIGHT_OPT'] = ['I=1','DY=-5']
CONFIG['STRAIGHT_OPT'] = ['I=1'] # the option DY is computed automatically
#CONFIG['STRAIGHT_OPT'] = ['I=1','DY=-5'] # define DY only if you want to override the automatic procedure
CONFIG['BASE_CALIB_DIR'] = os.path.join(CONFIG['APP_DIR'], 'calibrations')
CONFIG['CALIB_DIR'] = os.path.join(CONFIG['BASE_CALIB_DIR'], 'calfiles')
CONFIG['CALIB_DB_DIR'] = os.path.join(CONFIG['BASE_CALIB_DIR'], 'database')
......@@ -117,6 +118,7 @@ CONFIG['DARK'] = 'DARK'
CONFIG['FLAT'] = 'FLAT'
CONFIG['WCAL_UNE'] = 'WCAL_UNE'
CONFIG['WCAL_FP'] = 'WCAL_FP'
#CONFIG['WCAL_FP'] = 'WCAL_LFC'
CONFIG['SCIENCE'] = 'SCIENCE'
# CONFIG['NODSTARE'] keyword values to determine if stare or nodding
CONFIG['NODVALUE'] = 'GIANO_sci_nodding'
......@@ -163,7 +165,7 @@ CONFIG['WLFIT'] = (' '.join((CONFIG['KEY_DRS'],'CAL WLFUNC')),'Function used for
if CONFIG['CAL_FUNC']['Oliva']:
CONFIG['WLFIT_FUNC'] = 'l0 + k1*(x-xc) + k2*(x-xc)**2 + k3*(x-xc)**3'
#CONFIG['WLCOEFFS'] = {'k1':(' '.join((CONFIG['KEY_DRS'],'CAL K1')),'k1 coefficient of the calibration function'),'k2':(' '.join((CONFIG['KEY_DRS'],'CAL K2')),'k2 coefficient of the calibration function'),'k3':(' '.join((CONFIG['KEY_DRS'],'CAL K3')),'k3 coefficient of the calibration function'),'l0':(' '.join((CONFIG['KEY_DRS'],'CAL L0')),'l0 coefficient of the calibration function'),'xc':(' '.join((CONFIG['KEY_DRS'],'CAL XC')),'xc coefficient of the calibration function'),'rms':(' '.join((CONFIG['KEY_DRS'],'CAL RMSE')),'[m/s] RMS error of the calibration')}
else:
CONFIG['WLFIT_FUNC'] = 'c0 + c1*x + c2*x**2 + c3*x**3'
......@@ -183,14 +185,13 @@ CONFIG['DB_CALIB'] = 'drs_calib.db'
CONFIG['DB_CALIB_PATH'] = os.path.join(CONFIG['CALIB_DB_DIR'], CONFIG['DB_CALIB'])
CONFIG['DB_CALIB_TBL'] = 'calibrations'
#CONFIG['DB_CALIB_COLS'] = OrderedDict([('01_data', 'date_calib'), ('dark10', 'masterdark10'), ('dark30', 'masterdark30'), ('dark60', 'masterdark60'), ('dark100', 'masterdark100'), ('flat', 'masterflat'), ('flatstr', 'masterflat_str'), ('flatnor', 'masterflat_nor'), ('une', 'wl_une'), ('une_str', 'wl_une_str'), ('une_calib', 'wl_une_calib'), ('fp', 'wl_fp'), ('fp_str', 'wl_fp_str'), ('fp_calib','wl_fp_calib')])
CONFIG['DB_CALIB_COLS'] = OrderedDict([('01_data', 'date_calib'), ('dark10', 'masterdark10'), ('dark30', 'masterdark30'), ('dark60', 'masterdark60'), ('dark100', 'masterdark100'), ('dark200', 'masterdark200'), ('dark300', 'masterdark300'), ('dark600', 'masterdark600'),('flat', 'masterflat'), ('flatstr', 'masterflat_str'), ('flatnor', 'masterflat_nor'), ('une', 'wl_une'), ('une_str', 'wl_une_str'), ('une_calib', 'wl_une_calib'), ('fp', 'wl_fp'), ('fp_str', 'wl_fp_str'), ('fp_calib','wl_fp_calib')])
CONFIG['DB_CALIB_COLS'] = OrderedDict([('01_data', 'date_calib'), ('dark10', 'masterdark10'), ('dark30', 'masterdark30'), ('dark60', 'masterdark60'), ('dark100', 'masterdark100'), ('dark200', 'masterdark200'), ('dark300', 'masterdark300'), ('dark600', 'masterdark600'),('flat', 'masterflat'), ('flatstr', 'masterflat_str'), ('flatnor', 'masterflat_nor'), ('shiftY', 'shift_Y') ,('une', 'wl_une'), ('une_str', 'wl_une_str'), ('une_calib', 'wl_une_calib'), ('fp', 'wl_fp'), ('fp_str', 'wl_fp_str'), ('fp_calib','wl_fp_calib')])
CONFIG['DB_CALIB_DATATYPE'] = OrderedDict()
for colname in CONFIG['DB_CALIB_COLS']:
CONFIG['DB_CALIB_DATATYPE'][colname] = 'TEXT'
CONFIG['DB_CALIB_DATATYPE']['01_data'] = 'TEXT unique'
CONFIG['DB_CALIB_DATATYPE']['shiftY'] = 'INTEGER'
# Night reduction database
......@@ -207,12 +208,10 @@ CONFIG['DB_2D_COLS'] = OrderedDict([('names', 'spec_2d_names'), ('stamp', 'proce
CONFIG['DB_1D_COLS'] = OrderedDict([('path', 'spec_1d_path'), ('stamp', 'process_timestamp'), ('id', 'spec_1d_id'), ('type', 'spec_1d_type'), ('stat', 'arch_stat'), ('name', 'objname'), ('slit', 'slitpos'), ('snry', 'snr1'), ('snrj', 'snr2'), ('snrh', 'snr3'), ('snrk', 'snr4')])
CONFIG['DB_CAL_COLS'] = OrderedDict([('type', 'calib_type'), ('names', 'calib_names'), ('status', 'calib_status'), ('stamp', 'process_timestamp')])
CONFIG['DB_2D_DATATYPE'] = {'names': 'TEXT unique', 'stamp': 'REAL'}
CONFIG['DB_1D_DATATYPE'] = {'path': 'TEXT unique', 'stamp': 'REAL', 'id': 'TEXT', 'type': 'TEXT', 'stat': 'INTEGER', 'name': 'TEXT', 'slit': 'TEXT', 'snry': 'REAL', 'snrj': 'REAL', 'snrh': 'REAL', 'snrk': 'REAL'}
CONFIG['DB_CAL_DATATYPE'] = {'type': 'TEXT', 'names': 'TEXT unique', 'status': 'TEXT', 'stamp': 'REAL'}
CONFIG['DB_NIGHT'] = CONFIG['DB_ONLINE']
CONFIG['DB_NIGHT_PATH'] = CONFIG['DB_ONLINE_PATH']
......@@ -224,6 +223,7 @@ CONFIG['MASK_C'] = os.path.join(CONFIG['RES_DIR'], 'GIANOB_MASKC.fits')
CONFIG['A_POS'] = 8. # guess pixel position for A
CONFIG['B_POS'] = 23. # guess pixel position for B
CONFIG['C_POS'] = 15. # guess pixel position for C
CONFIG['Y_POS'] = 4 # max. pixel difference between expected and real pixel position
CONFIG['HWTM'] = 6. # hwtm limit for contamination between A and B
CONFIG['S1D'] = True # flag for creation of s1d output (True/False)
CONFIG['S1D_NORM'] = False # flag for normalization of s1d output (True/False)
......@@ -248,16 +248,13 @@ CONFIG['DO_CALIB'] = {'dark':True, 'flat':True, 'une':True, 'fp':True, 'only_cal
CONFIG['USE_FLAT'] = {'global':False, 'order':True, 'nor':False} # global: flat normalized by global mean value - order: flat normalized order by order by mean value - nor: flat normalized row by row by continuum fitting, removing the blaze function - No True value: no flat division
CONFIG['FLAT_EXPT'] = 100.0
CONFIG['WLCAL_EXPT'] = 100.0
CONFIG['FP_EXPT'] = 60.0
CONFIG['DARKLIST'] = {10:[],30:[],60:[],100:[],200:[],300:[],600:[]} # exptime dei dark
CONFIG['NDARK'] = len(CONFIG['DARKLIST']) # quanti tipi di dark (esempio: 10,30,60,100 sec = 4) meglio sovrastimare
CONFIG['WEXT'] = 5 # extraction half-window for wavelength calibration, it will extract 2*wext around the center of the order
CONFIG['WEXT'] = 10 # extraction half-window for wavelength calibration, it will extract 2*wext around the center of the order
#
# CCD info
# Detector info
#
CONFIG['RON'] = 5.0 # electrons
......@@ -266,6 +263,7 @@ CONFIG['XCCD'] = 2048
CONFIG['YCCD'] = 2048
CONFIG['N_ORD'] = 50 # number of echelle orders
CONFIG['W_ORD'] = 41 # width of straighten order in pixel
CONFIG['SHIFT_Y'] = 20 # expected bottom position of flat signal on the lowest order and the left side of the detector. DO NOT CHANGE.
#
# Quality checks
......@@ -297,4 +295,7 @@ if USRCONFIG['OFFLINE']:
CONFIG['DB_NIGHT'] = CONFIG['DB_OFFLINE']
CONFIG['DB_NIGHT_PATH'] = CONFIG['DB_OFFLINE_PATH']
maskc = ''.join(('GIANOB_MASKC_',CONFIG['DATE'],'.fits'))
CONFIG['MASK_C'] = os.path.join(CONFIG['RES_DIR'], maskc)
......@@ -78,6 +78,7 @@ class GBFlats():
using ron and gain, correct ron and gain --> ron_eff and gain_eff
Write the result in the database.
"""
if self.flatlist:
exptime = self.flatlist[0].header[CONFIG['KEYS']['EXPTIME']]
darkname = 'dark' + str(int(exptime))
......@@ -117,7 +118,7 @@ class GBFlats():
for flat in self.flatlist:
# subdtract the masterdak
# subdtract the masterdark
if use_dark:
subflat = ccdproc.subtract_dark(flat,mdark,exposure_time=CONFIG['KEYS']['EXPTIME'],exposure_unit=u.second)
self.messages.append('Masterdark subtracted.')
......@@ -132,12 +133,16 @@ class GBFlats():
self.flatcorr.append(flat_corr)
combineflat = ccdproc.Combiner(self.flatcorr)
if len(self.flatcorr) > 1:
combineflat = ccdproc.Combiner(self.flatcorr)
# mask the pixels using ron/gain
#combineflat.sigma_clipping(func=np.ma.mean, dev_func=varie.stdcombine)
combineflat.sigma_clipping(func=np.ma.median, dev_func=varie.stdcombine)
mflat = combineflat.average_combine()
# mask the pixels using ron/gain
#combineflat.sigma_clipping(func=np.ma.mean, dev_func=varie.stdcombine)
combineflat.sigma_clipping(func=np.ma.median, dev_func=varie.stdcombine)
mflat = combineflat.average_combine()
else:
mflat = self.flatcorr[0]
mflat.data = np.asarray(mflat.data, dtype='float32')
......@@ -156,19 +161,44 @@ class GBFlats():
masterflat = fits.HDUList([hdu])
masterflat.writeto(nome,clobber=True)
mflat = None
db.insert_dbfile(self.dbconn,'flat',nome)
self.messages.append('Masterflat %s was created and inserted in the calibration database' % (str(os.path.basename(nome))))
shiftY = varie.shiftY(mflat.data)
db.insert_dbfile(self.dbconn,'shiftY',shiftY)
mflat = None
else:
self.messages.append('There are no available flats. The masterflat will be taken from the calibration database.')
if db.check_dbfile(self.dbconn,'flat'):
try:
db.copy_dbfile(self.dbconn,'flat')
try:
db.copy_dbfile(self.dbconn,'shiftY')
shift = db.extract_dbfile(self.dbconn,'shiftY')
if not shift:
cal_flat = db.extract_dbfile(self.dbconn,'flat')
mflat = ccdproc.CCDData.read(cal_flat, unit=u.adu)
shiftY = varie.shiftY(mflat.data)
db.insert_dbfile(self.dbconn,'shiftY',shiftY)
except:
cal_flat = db.extract_dbfile(self.dbconn,'flat')
mflat = ccdproc.CCDData.read(cal_flat, unit=u.adu)
shiftY = varie.shiftY(mflat.data)
db.insert_dbfile(self.dbconn,'shiftY',shiftY)
except:
self.messages.append('There are no flats in the calibration database.')
return False
if db.check_dbfile(self.dbconn,'shiftY'):
cal_flat = db.extract_dbfile(self.dbconn,'flat')
mflat = ccdproc.CCDData.read(cal_flat, unit=u.adu)
shiftY = varie.shiftY(mflat.data)
db.insert_dbfile(self.dbconn,'shiftY',shiftY)
return True
......@@ -185,9 +215,32 @@ class GBFlats():
nor = flat.replace('.fits','_nor.fits')
#extracted = flat.replace('.fits','_ext.fits')
#args = [CONFIG['STRAIGHT'],flat,straight,CONFIG['STRAIGHT_OPT']]
args = [CONFIG['STRAIGHT'],flat,straight]
args.extend(CONFIG['STRAIGHT_OPT'])
# search for shift defined in the straighten options in config.py
dy = True
for opt in CONFIG['STRAIGHT_OPT']:
try:
dy = opt.rindex('DY=')
ypos = int(opt[dy-2:])
shift = CONFIG['SHIFT_Y'] + ypos
dy = False
except:
pass
if dy:
shift = db.extract_dbfile(self.dbconn,'shiftY')
if not shift:
cal_flat = db.extract_dbfile(self.dbconn,'flat')
mflat = ccdproc.CCDData.read(cal_flat, unit=u.adu)
shift = varie.shiftY(mflat.data)
db.insert_dbfile(self.dbconn,'shiftY',shift)
shiftY = [''.join(('DY=',str(shift - CONFIG['SHIFT_Y'])))]
#print shiftY
args.extend(shiftY)
#print args
subprocess.call(args)
red_straight = os.path.join(CONFIG['RED_CALIB'],os.path.basename(straight))
......
import os, sys, time
from drslib.config import CONFIG, read_usr_config
from drslib.logger import DrsLogger
from drslib import db, rawfiles, darkframes, flatframes, wlframes, nodding, stare
def gofio_init(docopt_args):
if docopt_args['--cfg']:
try:
USRCONFIG = read_usr_config(docopt_args['-g'])
except:
try:
USRCONFIG = read_usr_config(docopt_args['--cfg'])
except:
USRCONFIG = {}
USRCONFIG['OFFLINE'] = False
if USRCONFIG['OFFLINE']:
for key in USRCONFIG.keys():
CONFIG[key] = USRCONFIG[key]
CONFIG['RAW_DIR'] = os.path.join(CONFIG['BASE_RAW'],CONFIG['DATE'])
CONFIG['OFFLINE_DIR'] = os.path.join(CONFIG['BASE_RED_DIR'],CONFIG['DATE'])
CONFIG['RED_DIR'] = os.path.join(CONFIG['OFFLINE_DIR'],'offline')
CONFIG['RED_CALIB'] = os.path.join(CONFIG['RED_DIR'],'CALIB')
CONFIG['RED_STR'] = os.path.join(CONFIG['RED_DIR'],'STR')
CONFIG['LOG_FILE'] = os.path.join(CONFIG['RED_DIR'], 'drs.log')
CONFIG['RAMP_NAME'] = 'ramps' + CONFIG['DATE'] + '.db'
CONFIG['DB_RAMP'] = os.path.join(CONFIG['BASE_RAMP'],CONFIG['RAMP_NAME'])
CONFIG['DB_OFFLINE'] = ''.join(('db_',CONFIG['DATE'],'_offline.db'))
CONFIG['DB_OFFLINE_PATH'] = os.path.join(CONFIG['WEBUI_DB_DIR'], CONFIG['DB_OFFLINE'])
CONFIG['DB_NIGHT'] = CONFIG['DB_OFFLINE']
CONFIG['DB_NIGHT_PATH'] = CONFIG['DB_OFFLINE_PATH']
maskc = ''.join(('GIANOB_MASKC_',CONFIG['DATE'],'.fits'))
CONFIG['MASK_C'] = os.path.join(CONFIG['RES_DIR'], maskc)
if docopt_args['<date>']:
if docopt_args['--all_calib']:
docopt_args['--dark'] = True
docopt_args['--flat'] = True
docopt_args['--une'] = True
docopt_args['--fp'] = True
CONFIG['DATE'] = docopt_args['<date>']
CONFIG['OFFLINE'] = True
CONFIG['RAW_DIR'] = os.path.join(CONFIG['BASE_RAW'],CONFIG['DATE'])
CONFIG['OFFLINE_DIR'] = os.path.join(CONFIG['BASE_RED_DIR'],CONFIG['DATE'])
CONFIG['RED_DIR'] = os.path.join(CONFIG['OFFLINE_DIR'],'offline')
CONFIG['RED_CALIB'] = os.path.join(CONFIG['RED_DIR'],'CALIB')
CONFIG['RED_STR'] = os.path.join(CONFIG['RED_DIR'],'STR')
CONFIG['LOG_FILE'] = os.path.join(CONFIG['RED_DIR'], 'drs.log')
CONFIG['DO_CALIB'] = {'dark': docopt_args['--dark'], 'flat': docopt_args['--flat'], 'une': docopt_args['--une'], 'fp': docopt_args['--fp'], 'only_calib': docopt_args['--only_calib']}
if docopt_args['--use_flat'] == 'global':
CONFIG['USE_FLAT'] = {'global':True, 'order':False, 'nor':False}
elif docopt_args['--use_flat'] == 'order':
CONFIG['USE_FLAT'] = {'global':False, 'order':True, 'nor':False}
elif docopt_args['--use_flat'] == 'nor':
CONFIG['USE_FLAT'] = {'global':False, 'order':False, 'nor':True}
else:
CONFIG['USE_FLAT'] = {'global':False, 'order':False, 'nor':False}
if docopt_args['--s1d'] == 'norm':
CONFIG['S1D'] = True
CONFIG['S1D_NORM'] = True
elif docopt_args['--s1d'] == 'no':
CONFIG['S1D'] = False
CONFIG['S1D_NORM'] = False
else:
CONFIG['S1D'] = True
CONFIG['S1D_NORM'] = False
maskc = ''.join(('GIANOB_MASKC_',CONFIG['DATE'],'.fits'))
CONFIG['MASK_C'] = os.path.join(CONFIG['RES_DIR'], maskc)
CONFIG['DB_OFFLINE'] = ''.join(('db_',CONFIG['DATE'],'_offline.db'))
CONFIG['DB_OFFLINE_PATH'] = os.path.join(CONFIG['WEBUI_DB_DIR'], CONFIG['DB_OFFLINE'])
CONFIG['DB_NIGHT'] = CONFIG['DB_OFFLINE']
CONFIG['DB_NIGHT_PATH'] = CONFIG['DB_OFFLINE_PATH']
# second argument: how to query the calibration database
# to be set if the the user wants to reduce the night using
# another night's calibrations
if docopt_args['<calib_date>']:
CONFIG['DATE'] = docopt_args['<calib_date>']
# remove the mask with the order position (if it exists)
# it will be re-build with the correct flat-field
try:
os.remove(CONFIG['MASK_C'])
except:
pass
# Create the reduction directory
try:
os.mkdir(CONFIG['BASE_RED_DIR'])
except OSError:
pass
if CONFIG['OFFLINE']:
try:
os.mkdir(CONFIG['OFFLINE_DIR'])
except OSError:
pass
try:
os.mkdir(CONFIG['RED_DIR'])
except OSError:
pass
try:
os.mkdir(CONFIG['RED_CALIB'])
os.mkdir(CONFIG['RED_STR'])
except OSError:
pass
# Create the calibration directory tree
try:
os.mkdir(CONFIG['BASE_CALIB_DIR'])
except OSError:
pass
try:
os.mkdir(CONFIG['CALIB_DIR'])
except OSError:
pass
# initialize logger
drslogger = DrsLogger()
# Databases online/offline
dbcalib = db.create_db_calib()
if CONFIG['OFFLINE']:
dbramps = False
try:
os.remove(CONFIG['DB_NIGHT_PATH'])
except:
pass
else:
dbramps = db.connect_db_ramp()
dbnight = db.create_db_night()
if not os.path.exists(CONFIG['RAW_DIR']):
drslogger.log('The RAW directory %s does not exist. GOFIO will now close down.' % CONFIG['RAW_DIR'])
drslogger.log('Pipeline stopped.')
dbcalib.close()
if not CONFIG['OFFLINE']:
dbramps.close()
dbnight.close()
sys.exit(0)
rawlists = {CONFIG['DARK']:[],CONFIG['FLAT']:[],CONFIG['WCAL_UNE']:[],CONFIG['WCAL_FP']:[],CONFIG['SCIENCE']:[]}
# store the information on what the pipeline is working on (dark, flat, ...) and which science group/pair
working = 0
group = {}
stop = False