wlframes.py 16 KB
Newer Older
Monica Rainer's avatar
Monica Rainer committed
1
2
3
4
5
6
7
8
9
10
11
"""
Last modified: 2017-03-07

Reduction of the UNe lamps:
- check the exposure time???
- check the flux ratio in chosen areas???
- subtract the masterdark
- remove bad pixel using the bad pixel mask
- if more than one lamps, combine them and adjust ron and gain
- save the masterlamp in the calibration database
- straigthen the masterlamp
12
- extract the central 10 pixels of each order for quality check??
Monica Rainer's avatar
Monica Rainer committed
13
14
15
16
17
18
19
20
21
22
23
24
25
- UNe wavelength calibration (TO DO: FP)
- TO DO: UNe line diagnostics (checking the ratio between specific emission lines)
"""

from astropy import units as u
from astropy.io import fits
import warnings
from astropy.utils.exceptions import AstropyWarning


import ccdproc
from drslib.config import CONFIG
from drslib import db, varie
Andrea Bignamini's avatar
Andrea Bignamini committed
26
from drslib import metadata
Monica Rainer's avatar
Monica Rainer committed
27
28
29
30
31
32
33
34
35
36
37
38
import numpy as np
import math, os, subprocess, shutil


class GBWls():
    def __init__(self, wls, dbconn):
        self.wls = wls
        self.dbconn = dbconn
        self.quality = []
        self.messages = []
        self.wllist = []
        self.wlcorr = []
Monica Rainer's avatar
Monica Rainer committed
39
        self.mlamp = ''
Monica Rainer's avatar
Monica Rainer committed
40
41
42
43
44


    def qualitycheck(self,unefp):
        """
        Quality check for UNe and FP lamps still to be defined.
Monica Rainer's avatar
Monica Rainer committed
45
46
        --> It will be done after the default wavelength calibration,
        here everything is defined as ok.
Monica Rainer's avatar
Monica Rainer committed
47
48
49
50
51
52
        """
        for frame in self.wls:

            wl = ccdproc.CCDData.read(frame, unit=u.adu)

            self.wllist.append(wl)
53
            self.quality.append('OK')
Monica Rainer's avatar
Monica Rainer committed
54
55
56
57
58
        return

    def masterlamp(self,unefp):

        if self.wllist:
59
            exptime = float(self.wllist[0].header[CONFIG['KEYS']['EXPTIME']])
Monica Rainer's avatar
Monica Rainer committed
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
            darkname = 'dark' + str(int(exptime))
            use_dark = True

            try:
                masterdark = db.extract_dbfile(self.dbconn,darkname)
            except:
                masterdark = False

            if not masterdark:
                self.messages.append('No masterdark found for this night, it will be taken from the calibration database.')

                try:
                    db.copy_dbfile(self.dbconn,darkname)
                    masterdark = db.extract_dbfile(self.dbconn,darkname)
                except:
                    for key in CONFIG['DARKLIST']:
                        try:
                            darkname = 'dark' + str(int(key))
                            db.copy_dbfile(self.dbconn,darkname)
                            masterdark = db.extract_dbfile(self.dbconn,darkname)
                            self.messages.append('No masterdark in the calibration database with the same exposure time as the lamp. The %s sec masterdark will be used instead' % (str(int(key))))
                            break
                        except:
                            self.messages.append('There are no masterdark in the calibration database. The masterdark will not be used.')
                            use_dark = False

            if use_dark:
                mdark = ccdproc.CCDData.read(masterdark, unit=u.adu)
            badpix = ccdproc.CCDData.read(CONFIG['BADPIX_MASK'], unit=u.adu)
            bad_mask=badpix.data
            inverse_mask=np.logical_not(bad_mask)


            for wl in self.wllist:

                # subdtract the masterdak
                if use_dark:
                    subwl = ccdproc.subtract_dark(wl,mdark,exposure_time=CONFIG['KEYS']['EXPTIME'],exposure_unit=u.second)
                    self.messages.append('Masterdark subtracted.')
                else:
                    subwl = wl

                # mask the lamp using the badpix mask
                wl_corrected = varie.badpix(subwl.data,bad_mask,inverse_mask)
                self.messages.append('Bad pixel correction done.')

                wl_corr = ccdproc.CCDData(wl_corrected,unit=u.adu)

                self.wlcorr.append(wl_corr)

Monica Rainer's avatar
Monica Rainer committed
110
            if len(self.wlcorr)>1:
Monica Rainer's avatar
Monica Rainer committed
111

Monica Rainer's avatar
Monica Rainer committed
112
113
114
115
116
117
118
119
120
121
122
                combinewl = ccdproc.Combiner(self.wlcorr)


                # mask the pixels using ron/gain
                #combinewl.sigma_clipping(func=np.ma.mean, dev_func=varie.stdcombine)
                combinewl.sigma_clipping(func=np.ma.median, dev_func=varie.stdcombine)

                #combineflat.sigma_clipping(func=np.ma.mean)
                mwl = combinewl.average_combine()
            else:
                mwl = self.wlcorr[0]
Monica Rainer's avatar
Monica Rainer committed
123
124
125
126
127
128
129
130
131

            mwl.data = np.asarray(mwl.data, dtype='float32')

            hea = self.wllist[0].header
            if unefp == 'une':
                wlname = hea[CONFIG['KEYS']['IMANAME']].replace('.fts','_UNE.fits')

            else:
                wlname = hea[CONFIG['KEYS']['IMANAME']].replace('.fts','_FP.fits')
Monica Rainer's avatar
Monica Rainer committed
132
                #wlname = hea[CONFIG['KEYS']['IMANAME']].replace('.fts','_LFC.fits')
Monica Rainer's avatar
Monica Rainer committed
133

134
            #wlname = wlname.replace(' ', '').replace('-', '_')
Monica Rainer's avatar
Monica Rainer committed
135
            self.mlamp = os.path.join(CONFIG['CALIB_DIR'],wlname)
Monica Rainer's avatar
Monica Rainer committed
136
137
138
139
140
141
142

            mwl.header = hea
            mwl.header[CONFIG['KEYS']['FILENAME']] = wlname
            mwl.header[CONFIG['KEYS']['NCOMBINE']] = len(self.wllist)
            mwl.header[CONFIG['RON_EFF'][0]] = (math.sqrt(len(self.wllist))*CONFIG['RON'],CONFIG['RON_EFF'][1])
            mwl.header[CONFIG['GAIN_EFF'][0]] = (len(self.wllist)*CONFIG['GAIN'],CONFIG['GAIN_EFF'][1])

143
144
            mwl.header[CONFIG['DRS_VERSION'][0]] = (CONFIG['VERSION'], CONFIG['DRS_VERSION'][1])

Andrea Bignamini's avatar
Andrea Bignamini committed
145
146
147
            # Add metadata to header
            mwl.header = metadata.add_metadata(mwl.header)

Monica Rainer's avatar
Monica Rainer committed
148
149
            hdu = fits.PrimaryHDU(data=mwl.data,header=mwl.header)
            masterwl = fits.HDUList([hdu])
Monica Rainer's avatar
Monica Rainer committed
150
            masterwl.writeto(self.mlamp,clobber=True)
Monica Rainer's avatar
Monica Rainer committed
151
            mwl = None
Monica Rainer's avatar
Monica Rainer committed
152
153
154

            #db.insert_dbfile(self.dbconn,unefp,self.mlamp)
            #self.messages.append('%s masterlamp %s was created and inserted in the calibration database.' % (unefp,str(os.path.basename(self.mlamp))))
Monica Rainer's avatar
Monica Rainer committed
155
156
157


        else:
Monica Rainer's avatar
Monica Rainer committed
158
            self.messages.append('There are no useful %s lamps. The reference lamp will be taken from the calibration database.' % (unefp))
Monica Rainer's avatar
Monica Rainer committed
159
160
161
162

            if db.check_dbfile(self.dbconn,unefp):
                try:
                    db.copy_dbfile(self.dbconn,unefp)
Monica Rainer's avatar
Monica Rainer committed
163
                    self.mlamp = db.extract_dbfile(self.dbconn,unefp)
Monica Rainer's avatar
Monica Rainer committed
164
165
166
                except:
                    self.messages.append('There are no %s lamps in the calibration database.' %(unefp))
                    return False
167
168
169
170
171
172

            # Since masterlamp has been copied from calibration database
            # also str and extr will be copied
            self.copy_straighten(unefp)
            return False

Monica Rainer's avatar
Monica Rainer committed
173
174
175
        return True


176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
    def copy_straighten(self,unefp):
        """
        The masterlamp has been copied from calibration database
        This function also copy str and extr file from calibration database
        """

        # Get masterlamp name from calibdb
        # lamp is a fullpath name
        lamp = self.mlamp

        # Generate str and extr fullpath name from calibdb
        straight = lamp.replace('.fits','_str.fits')
        extracted = lamp.replace('.fits','_ext.fits')

        # Insert str and extr in calibdb for this night
        strname = '_'.join((unefp,'str'))
        db.insert_dbfile(self.dbconn,strname,straight)

        calname = '_'.join((unefp,'calib'))
        db.insert_dbfile(self.dbconn,calname,extracted)

        # Copy straight and extracted to RED_CALIB directory
        red_straight = os.path.join(CONFIG['RED_CALIB'],os.path.basename(straight))
        shutil.copyfile(straight,red_straight)
        red_extr = os.path.join(CONFIG['RED_CALIB'],os.path.basename(extracted))
        shutil.copyfile(extracted,red_extr)

        return


Monica Rainer's avatar
Monica Rainer committed
206
207
    def extract(self,unefp):
        # straighten the lamp's orders
Monica Rainer's avatar
Monica Rainer committed
208
209
        # lamp = db.extract_dbfile(self.dbconn,unefp)
        lamp = self.mlamp
Monica Rainer's avatar
Monica Rainer committed
210
211
212
213
214

        #print lamp
        straight = lamp.replace('.fits','_str.fits')
        extracted = lamp.replace('.fits','_ext.fits')

215
216
217
218
219
        args = [CONFIG['STRAIGHT'],lamp,straight]
        args.extend(CONFIG['STRAIGHT_OPT'])
        # search for shift defined in the straighten options in config.py
        dy = True
        for opt in CONFIG['STRAIGHT_OPT']:
220
            try:
221
222
223
224
                dy = opt.rindex('DY=')
                ypos = int(opt[dy-2:])
                shift = CONFIG['SHIFT_Y'] + ypos
                dy = False
225
            except:
226
227
228
                pass

        if dy:
229
230
231
232
233
            try:
                shift = db.extract_dbfile(self.dbconn,'shiftY')
            except:
                try:
                    cal_flat = db.extract_dbfile(self.dbconn,'flat')
234
235
236
                    mflat = ccdproc.CCDData.read(cal_flat, unit=u.adu)
                    shift = varie.shiftY(mflat.data)
                    db.insert_dbfile(self.dbconn,'shiftY',shift)
Monica Rainer's avatar
Monica Rainer committed
237
                except:
238
239
                    db.copy_dbfile(self.dbconn,'shiftY')
                    shift = db.extract_dbfile(self.dbconn,'shiftY')
Monica Rainer's avatar
Monica Rainer committed
240
241
            if not shift:
                shift = CONFIG['SHIFT_Y']
242

243
244
245
            shiftY = [''.join(('DY=',str(shift - CONFIG['SHIFT_Y'])))]
            #print shiftY
            args.extend(shiftY)
246

Monica Rainer's avatar
Monica Rainer committed
247
248
249
250
        subprocess.call(args)

        #self.messages.append('%s: orders straightened.' % (str(os.path.basename(lamp))))

Monica Rainer's avatar
Monica Rainer committed
251
252
253
        #strname = '_'.join((unefp,'str'))
        #db.insert_dbfile(self.dbconn,strname,straight)
        #self.messages.append('Straightened %s masterlamp %s was created and inserted in the calibration database.' % (unefp,str(os.path.basename(straight))))
Monica Rainer's avatar
Monica Rainer committed
254
255
256

        wl = ccdproc.CCDData.read(straight, unit=u.adu)

Andrea Bignamini's avatar
Andrea Bignamini committed
257
258
259
260
261
262
263
264
265
266
        # Update FILENAME in header then
        # add metadata to header and save straight file
        wl.header[CONFIG['KEYS']['FILENAME']] = os.path.basename(straight)
        wl.header = metadata.add_metadata(wl.header)

        hdu = fits.PrimaryHDU(data=wl.data, header=wl.header)
        str_fits = fits.HDUList([hdu])
        str_fits.writeto(straight, overwrite=True)


Monica Rainer's avatar
Monica Rainer committed
267
268
269
        wldata = wl.data
        wl.header[CONFIG['WLFIT'][0]] = (CONFIG['WLFIT_FUNC'],CONFIG['WLFIT'][1])

Monica Rainer's avatar
Monica Rainer committed
270
        ordini = np.zeros((CONFIG['N_ORD'],CONFIG['XCCD']))
Monica Rainer's avatar
Monica Rainer committed
271
272
273
274
275
276
277

        # read the lines to use in the wavelength calibration
        select_lines, all_lines = varie.UNe_linelist() 

        # extract the central 10 pixel of each order
        # wavelength calibration, to check the lamp and
        # to be used as default if later ad hoc calibration fails
Monica Rainer's avatar
Monica Rainer committed
278
279
        for x in xrange(CONFIG['N_ORD']):
            row = x*CONFIG['W_ORD'] + 19
Monica Rainer's avatar
Monica Rainer committed
280
 
Monica Rainer's avatar
Monica Rainer committed
281
            ordini[x] = np.sum(wldata[row-CONFIG['WEXT']:row+CONFIG['WEXT']],axis=0)
Monica Rainer's avatar
Monica Rainer committed
282
            if unefp == 'une':
283
                calib_failed, coeffs, comments = varie.UNe_calibrate(ordini[x],x+32,select_lines[x+32],all_lines[x+32],True,True)
Monica Rainer's avatar
Monica Rainer committed
284
285
286
                for comment in comments:
                    self.messages.append(comment)

287
288
                keyfail = ''.join((CONFIG['CAL_FAILED'][0],str(x+32)))

Monica Rainer's avatar
Monica Rainer committed
289
                if calib_failed:
Monica Rainer's avatar
Monica Rainer committed
290
291
292
293
294
295

                    if self.quality[0] == 'OK':
                        self.quality = ['FAILED' for q in self.quality]
                        #db.remove_dbfile(CONFIG['DATE'],'UNE')
                        #return

296
                    wl.header[keyfail] = (False,CONFIG['CAL_FAILED'][1])
Monica Rainer's avatar
Monica Rainer committed
297
298
299
                    self.messages.append(' *** WARNING ***')
                    self.messages.append('The default wavelength calibration for the order %s will be taken from the database and as such it will not be optimal for the night.' % str(x+32))
                    try:
300
                        wcalib = db.extract_dbfile(self.dbconn,'une_calib')
Monica Rainer's avatar
Monica Rainer committed
301
                    except:
Monica Rainer's avatar
   
Monica Rainer committed
302
                        #wcalib = False
Monica Rainer's avatar
Monica Rainer committed
303

Monica Rainer's avatar
   
Monica Rainer committed
304
305
306
307
308
309
310
                    #if not wcalib:
                        #self.messages.append('No UNe calibration found for this night, it will be taken from the calibration database.')
                        try:
                            db.copy_dbfile(self.dbconn,'une_calib')
                            wcalib = db.extract_dbfile(self.dbconn,'une_calib')
                        except:
                            wcalib = False
Monica Rainer's avatar
Monica Rainer committed
311
312
313
314
315
316

                    try:
                        wlc = ccdproc.CCDData.read(wcalib, unit=u.adu)
                        wl.header = wlc.header
                    except:
                        self.messages.append(' *** WARNING ***')
Monica Rainer's avatar
   
Monica Rainer committed
317
                        self.messages.append('There is no default wavelength calibration in the database, please observe a new lamp, otherwise the spectra will not be reduced.')
Monica Rainer's avatar
Monica Rainer committed
318
319

                else:
320
                    wl.header[keyfail] = (True,CONFIG['CAL_FAILED'][1])
Monica Rainer's avatar
Monica Rainer committed
321
322
323
324
325
326
327
                    for key in coeffs:
                        keyword = ''.join((CONFIG['WLCOEFFS'][key][0],str(x+32)))
                        wl.header[keyword] = (coeffs[key],CONFIG['WLCOEFFS'][key][1])

            else:
                # calibrate FP using UNe
                try:
Monica Rainer's avatar
Monica Rainer committed
328
                    wcalib = db.extract_dbfile(self.dbconn,'une_calib')
Monica Rainer's avatar
Monica Rainer committed
329
330
331
332
333
334
                except:
                    wcalib = False

                if not wcalib:
                    self.messages.append('No UNe calibration found for this night, it will be taken from the calibration database.')
                    try:
Monica Rainer's avatar
Monica Rainer committed
335
336
                        db.copy_dbfile(self.dbconn,'une_calib')
                        wcalib = db.extract_dbfile(self.dbconn,'une_calib')
Monica Rainer's avatar
Monica Rainer committed
337
338
339
340
341
                    except:
                        return

                #print wcalib
                wlc = ccdproc.CCDData.read(wcalib, unit=u.adu)
342
343
344
345
346
347
348

                for key in CONFIG['WLCOEFFS']:
                    keyword = ''.join((CONFIG['WLCOEFFS'][key][0],str(x+32)))
                    try:
                        wl.header[keyword] = (wlc.header[keyword],CONFIG['WLCOEFFS'][key][1])
                    except:
                        pass
Monica Rainer's avatar
Monica Rainer committed
349
350
351
352
353
354


        # save the extracted lamp as FITS file
        wlextract = ccdproc.CCDData(ordini, unit=u.adu)
        wlextract.data = np.asarray(wlextract.data, dtype='float32')
        wlextract.header = wl.header
355
        wlextract.header[CONFIG['KEY_WEXT'][0]] = (CONFIG['WEXT']*2,CONFIG['KEY_WEXT'][1])
356

Andrea Bignamini's avatar
Andrea Bignamini committed
357
358
359
360
361
362
        # Update FILENAME in header then
        # add metadata to header and save extracted file
        wlextract.header[CONFIG['KEYS']['FILENAME']] = os.path.basename(extracted)
        wlextract.header = metadata.add_metadata(wlextract.header)

        hdu = fits.PrimaryHDU(data=wlextract.data, header=wlextract.header)
Monica Rainer's avatar
Monica Rainer committed
363
        wlextr = fits.HDUList([hdu])
Andrea Bignamini's avatar
Andrea Bignamini committed
364
        wlextr.writeto(extracted, overwrite=True)
Monica Rainer's avatar
Monica Rainer committed
365

Andrea Bignamini's avatar
Andrea Bignamini committed
366
367
368
        # Copy straight and extracted to RED_CALIB directory
        red_straight = os.path.join(CONFIG['RED_CALIB'],os.path.basename(straight))
        shutil.copyfile(straight,red_straight)
Monica Rainer's avatar
Monica Rainer committed
369
370
371
        red_extr = os.path.join(CONFIG['RED_CALIB'],os.path.basename(extracted))
        shutil.copyfile(extracted,red_extr)

Monica Rainer's avatar
Monica Rainer committed
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
        if self.quality[0] == 'OK':
            db.insert_dbfile(self.dbconn,unefp,self.mlamp)
            self.messages.append('%s masterlamp %s was created and inserted in the calibration database.' % (unefp,str(os.path.basename(self.mlamp))))

            strname = '_'.join((unefp,'str'))
            db.insert_dbfile(self.dbconn,strname,straight)
            self.messages.append('Straightened %s masterlamp %s was created and inserted in the calibration database.' % (unefp,str(os.path.basename(straight))))

            calname = '_'.join((unefp,'calib'))
            db.insert_dbfile(self.dbconn,calname,extracted)
            self.messages.append('%s wavelength calibration %s was created and inserted in the calibration database.' % (unefp,str(os.path.basename(extracted))))

        else:
            os.remove(self.mlamp)
            os.remove(straight)
            os.remove(extracted)
Monica Rainer's avatar
   
Monica Rainer committed
388
389
390
            try: db.remove_dbfile1(self.mlamp)
            except: pass
            self.messages.append('%s frame %s failed the quality check and it will not be used.' % (unefp,str(os.path.basename(extracted))))
Monica Rainer's avatar
Monica Rainer committed
391

Monica Rainer's avatar
Monica Rainer committed
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416

        wlextract = None
        ordini = None

        return


    def une_process(self):
        warnings.simplefilter('ignore', category=AstropyWarning)
        self.qualitycheck('une')
        if self.masterlamp('une'):
            self.extract('une')
        #self.wls[:] = []
        return

    def fp_process(self):
        warnings.simplefilter('ignore', category=AstropyWarning)
        self.qualitycheck('fp')
        if self.masterlamp('fp'):
            self.extract('fp')
        #self.wls[:] = []
        return