Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Monica Rainer
gofio
Commits
59e78c6d
Commit
59e78c6d
authored
Oct 29, 2017
by
Monica Rainer
Browse files
Add archive databases
parent
cd7c5785
Changes
9
Hide whitespace changes
Inline
Side-by-side
drslib/config.py
View file @
59e78c6d
...
...
@@ -82,7 +82,6 @@ CONFIG['RED_DIR'] = os.path.join(CONFIG['BASE_RED_DIR'],CONFIG['DATE'])
CONFIG
[
'RED_CALIB'
]
=
os
.
path
.
join
(
CONFIG
[
'RED_DIR'
],
'CALIB'
)
CONFIG
[
'RED_STR'
]
=
os
.
path
.
join
(
CONFIG
[
'RED_DIR'
],
'STR'
)
# Resources
CONFIG
[
'RES_DIR'
]
=
os
.
path
.
join
(
CONFIG
[
'APP_DIR'
],
'resources'
)
...
...
@@ -94,7 +93,8 @@ CONFIG['STRAIGHT_OPT'] = 'I=1'
CONFIG
[
'BASE_CALIB_DIR'
]
=
os
.
path
.
join
(
CONFIG
[
'APP_DIR'
],
'calibrations'
)
CONFIG
[
'CALIB_DIR'
]
=
os
.
path
.
join
(
CONFIG
[
'BASE_CALIB_DIR'
],
'calfiles'
)
CONFIG
[
'CALIB_DB_DIR'
]
=
os
.
path
.
join
(
CONFIG
[
'BASE_CALIB_DIR'
],
'database'
)
CONFIG
[
'NIGHT_DB_DIR'
]
=
os
.
path
.
join
(
CONFIG
[
'APP_DIR'
],
'databases'
)
#CONFIG['NIGHT_DB_DIR'] = os.path.join(CONFIG['APP_DIR'], 'databases')
CONFIG
[
'WEBUI_DB_DIR'
]
=
os
.
path
.
join
(
CONFIG
[
'APP_DIR'
],
'webuidatabases'
)
# Output name
...
...
@@ -155,8 +155,8 @@ CONFIG['MASTERFLAT'] = (' '.join((CONFIG['KEY_DRS'],'FLAT')),'Flat-field used fo
CONFIG
[
'MASTERLAMP'
]
=
(
' '
.
join
((
CONFIG
[
'KEY_DRS'
],
'LAMP'
)),
'Calibration lamp used for the reduction'
)
# Wavelength calibration
#
CONFIG['CAL_FUNC'] = {'Oliva':True, 'Poly3':False} # function used for wavelength calibration (Tino Oliva Polynomial or 3-degree polynomial)
CONFIG
[
'CAL_FUNC'
]
=
{
'Oliva'
:
False
,
'Poly3'
:
True
}
CONFIG
[
'CAL_FUNC'
]
=
{
'Oliva'
:
True
,
'Poly3'
:
False
}
# function used for wavelength calibration (Tino Oliva Polynomial or 3-degree polynomial)
#
CONFIG['CAL_FUNC'] = {'Oliva':False, 'Poly3':True}
CONFIG
[
'WLFIT'
]
=
(
' '
.
join
((
CONFIG
[
'KEY_DRS'
],
'CAL WLFUNC'
)),
'Function used for wavelength calibration'
)
if
CONFIG
[
'CAL_FUNC'
][
'Oliva'
]:
...
...
@@ -188,14 +188,26 @@ for colname in CONFIG['DB_CALIB_COLS']:
CONFIG
[
'DB_CALIB_DATATYPE'
][
'01_data'
]
=
'TEXT unique'
# Night reduction database
CONFIG
[
'DB_NIGHT'
]
=
''
.
join
((
'drs_'
,
CONFIG
[
'DATE'
],
'.db'
))
CONFIG
[
'DB_NIGHT_PATH'
]
=
os
.
path
.
join
(
CONFIG
[
'NIGHT_DB_DIR'
],
CONFIG
[
'DB_NIGHT'
])
CONFIG
[
'DB_NIGHT_TBL'
]
=
'reduced'
CONFIG
[
'DB_NIGHT_COLS'
]
=
{
'fpath'
:
'FPATH'
}
CONFIG
[
'DB_NIGHT_DATATYPE'
]
=
{}
for
colname
in
CONFIG
[
'DB_CALIB_COLS'
]:
CONFIG
[
'DB_NIGHT_DATATYPE'
][
colname
]
=
'TEXT'
CONFIG
[
'DB_NIGHT_DATATYPE'
][
'fpath'
]
=
'TEXT unique'
CONFIG
[
'DB_ONLINE'
]
=
''
.
join
((
'db_'
,
CONFIG
[
'DATE'
],
'_online.db'
))
CONFIG
[
'DB_OFFLINE'
]
=
''
.
join
((
'db_'
,
CONFIG
[
'DATE'
],
'_offline.db'
))
CONFIG
[
'DB_ONLINE_PATH'
]
=
os
.
path
.
join
(
CONFIG
[
'WEBUI_DB_DIR'
],
CONFIG
[
'DB_ONLINE'
])
CONFIG
[
'DB_OFFLINE_PATH'
]
=
os
.
path
.
join
(
CONFIG
[
'WEBUI_DB_DIR'
],
CONFIG
[
'DB_OFFLINE'
])
CONFIG
[
'DB_2D_TBL'
]
=
'spec2dfiles'
CONFIG
[
'DB_1D_TBL'
]
=
'spec1dfiles'
CONFIG
[
'DB_2D_COLS'
]
=
OrderedDict
([(
'names'
,
'spec_2d_names'
),
(
'stamp'
,
'process_timestamp'
)])
CONFIG
[
'DB_1D_COLS'
]
=
OrderedDict
([(
'path'
,
'spec_1d_path'
),
(
'stamp'
,
'process_timestamp'
),
(
'id'
,
'spec_1d_id'
),
(
'type'
,
'spec_1d_type'
),
(
'stat'
,
'arch_stat'
),
(
'name'
,
'objname'
),
(
'slit'
,
'slitpos'
),
(
'snry'
,
'snr1'
),
(
'snrj'
,
'snr2'
),
(
'snrh'
,
'snr3'
),
(
'snrk'
,
'snr4'
)])
CONFIG
[
'DB_2D_DATATYPE'
]
=
{
'names'
:
'TEXT unique'
,
'stamp'
:
'REAL'
}
CONFIG
[
'DB_1D_DATATYPE'
]
=
{
'path'
:
'TEXT unique'
,
'stamp'
:
'REAL'
,
'id'
:
'TEXT'
,
'type'
:
'TEXT'
,
'stat'
:
'INTEGER'
,
'name'
:
'TEXT'
,
'slit'
:
'TEXT'
,
'snry'
:
'REAL'
,
'snrj'
:
'REAL'
,
'snrh'
:
'REAL'
,
'snrk'
:
'REAL'
}
CONFIG
[
'DB_NIGHT'
]
=
CONFIG
[
'DB_ONLINE'
]
CONFIG
[
'DB_NIGHT_PATH'
]
=
CONFIG
[
'DB_ONLINE_PATH'
]
#
# Extraction info
...
...
@@ -271,4 +283,7 @@ if USRCONFIG['OFFLINE']:
CONFIG
[
'RAMP_NAME'
]
=
'ramps'
+
CONFIG
[
'DATE'
]
+
'.db'
CONFIG
[
'DB_RAMP'
]
=
os
.
path
.
join
(
CONFIG
[
'BASE_RAMP'
],
CONFIG
[
'RAMP_NAME'
])
CONFIG
[
'DB_NIGHT'
]
=
CONFIG
[
'DB_OFFLINE'
]
CONFIG
[
'DB_NIGHT_PATH'
]
=
CONFIG
[
'DB_OFFLINE_PATH'
]
drslib/db.py
View file @
59e78c6d
...
...
@@ -26,17 +26,22 @@ def create_db_calib():
def
create_db_night
():
try
:
os
.
mkdir
(
CONFIG
[
'
NIGHT
_DB_DIR'
])
os
.
mkdir
(
CONFIG
[
'
WEBUI
_DB_DIR'
])
except
OSError
:
pass
dbcalib
=
sqlite3
.
connect
(
CONFIG
[
'DB_NIGHT_PATH'
])
cursor
=
dbcalib
.
cursor
()
cursor
.
execute
(
"CREATE TABLE IF NOT EXISTS "
+
\
CONFIG
[
'DB_
NIGHT
_TBL'
]
+
\
CONFIG
[
'DB_
2D
_TBL'
]
+
\
"("
+
\
","
.
join
([
CONFIG
[
'DB_
NIGHT
_COLS'
][
colname
]
+
' '
+
CONFIG
[
'DB_
NIGHT
_DATATYPE'
][
colname
]
for
colname
in
CONFIG
[
'DB_
NIGHT
_COLS'
]])
+
\
","
.
join
([
CONFIG
[
'DB_
2D
_COLS'
][
colname
]
+
' '
+
CONFIG
[
'DB_
2D
_DATATYPE'
][
colname
]
for
colname
in
CONFIG
[
'DB_
2D
_COLS'
]])
+
\
')'
)
cursor
.
execute
(
"CREATE TABLE IF NOT EXISTS "
+
\
CONFIG
[
'DB_1D_TBL'
]
+
\
"("
+
\
","
.
join
([
CONFIG
[
'DB_1D_COLS'
][
colname
]
+
' '
+
CONFIG
[
'DB_1D_DATATYPE'
][
colname
]
for
colname
in
CONFIG
[
'DB_1D_COLS'
]])
+
\
')'
)
dbcalib
.
commit
()
return
dbcalib
...
...
@@ -195,23 +200,41 @@ def check_night(dbnight,rawfile):
cursor
=
dbnight
.
cursor
()
try
:
cursor
.
execute
(
"SELECT * FROM {table} WHERE {fpath}=?"
\
.
format
(
table
=
CONFIG
[
'DB_NIGHT_TBL'
],
fpath
=
CONFIG
[
'DB_NIGHT_COLS'
][
'fpath'
]),(
rawfile
,))
filename
=
cursor
.
fetchone
()[
0
]
if
filename
:
return
True
cursor
.
execute
(
"SELECT {stamp} FROM {table} WHERE instr({fpath},?) > 0"
\
.
format
(
stamp
=
CONFIG
[
'DB_2D_COLS'
][
'stamp'
],
table
=
CONFIG
[
'DB_2D_TBL'
],
fpath
=
CONFIG
[
'DB_2D_COLS'
][
'names'
]),(
rawfile
,))
stamp
=
cursor
.
fetchone
()[
0
]
if
stamp
:
cursor
.
execute
(
"SELECT * FROM {table} WHERE {stamp}=?"
\
.
format
(
table
=
CONFIG
[
'DB_1D_TBL'
],
stamp
=
CONFIG
[
'DB_1D_COLS'
][
'stamp'
]),(
stamp
,))
reduced
=
cursor
.
fetchone
()[
0
]
if
reduced
:
return
True
else
:
return
False
else
:
return
False
except
:
return
False
def
insert_dbnight
(
dbnight
,
reduced
):
def
insert_dbnight
(
dbnight
,
reduced
,
stamp
):
cursor
=
dbnight
.
cursor
()
cursor
.
execute
(
"UPDATE {table} SET {fpath}=?, {fstamp}=? WHERE instr({fpath},?) > 0"
\
.
format
(
table
=
CONFIG
[
'DB_2D_TBL'
],
fpath
=
CONFIG
[
'DB_2D_COLS'
][
'names'
],
fstamp
=
CONFIG
[
'DB_2D_COLS'
][
'stamp'
]),(
reduced
,
stamp
,
reduced
,))
cursor
.
execute
(
"INSERT OR IGNORE INTO {table} ({fpath},{fstamp}) VALUES (?,?)"
\
.
format
(
table
=
CONFIG
[
'DB_2D_TBL'
],
fpath
=
CONFIG
[
'DB_2D_COLS'
][
'names'
],
fstamp
=
CONFIG
[
'DB_2D_COLS'
][
'stamp'
]),(
reduced
,
stamp
,))
dbnight
.
commit
()
def
insert_dbreduced
(
dbnight
,
reduced
,
stamp
):
cursor
=
dbnight
.
cursor
()
cursor
.
execute
(
"INSERT OR IGNORE INTO {table} ({
fpath}) VALUES (
?)"
\
.
format
(
table
=
CONFIG
[
'DB_
NIGHT_TBL'
],
fpath
=
CONFIG
[
'DB_NIGHT_COLS'
][
'fpath'
]),(
reduced
,
))
cursor
.
execute
(
"INSERT OR IGNORE INTO {table} ({
path},{stamp},{rid},{ftype},{stat},{name},{slit},{snry},{snrj},{snrh},{snrk}) VALUES (?,?,?,?,?,?,?,?,?,?,
?)"
\
.
format
(
table
=
CONFIG
[
'DB_
1D_TBL'
],
path
=
CONFIG
[
'DB_1D_COLS'
][
'path'
],
stamp
=
CONFIG
[
'DB_1D_COLS'
][
'stamp'
],
rid
=
CONFIG
[
'DB_1D_COLS'
][
'id'
],
ftype
=
CONFIG
[
'DB_1D_COLS'
][
'type'
],
stat
=
CONFIG
[
'DB_1D_COLS'
][
'stat'
],
name
=
CONFIG
[
'DB_1D_COLS'
][
'name'
],
slit
=
CONFIG
[
'DB_1D_COLS'
][
'slit'
],
snry
=
CONFIG
[
'DB_1D_COLS'
][
'snry'
],
snrj
=
CONFIG
[
'DB_1D_COLS'
][
'snrj'
],
snrh
=
CONFIG
[
'DB_1D_COLS'
][
'snrh'
],
snrk
=
CONFIG
[
'DB_1D_COLS'
][
'snrk'
]),(
reduced
[
'path'
],
stamp
,
reduced
[
'id'
],
reduced
[
'type'
],
0
,
reduced
[
'name'
],
reduced
[
'slit'
],
reduced
[
'snry'
],
reduced
[
'snrj'
],
reduced
[
'snrh'
],
reduced
[
'snrk'
],
))
dbnight
.
commit
()
drslib/nodding.py
View file @
59e78c6d
...
...
@@ -41,10 +41,11 @@ from collections import OrderedDict
class
GBNodding
():
def
__init__
(
self
,
nodding
,
group
,
dbconn
):
def
__init__
(
self
,
nodding
,
group
,
dbconn
,
dbnight
):
self
.
nodding
=
nodding
self
.
group
=
group
self
.
dbconn
=
dbconn
self
.
dbnight
=
dbnight
self
.
quality
=
[]
self
.
messages
=
[]
self
.
nodlist
=
[]
...
...
@@ -168,6 +169,7 @@ class GBNodding():
"""
Straighten, divide by the masterflat, optimal extraction
"""
dbreduced
=
{}
if
slit_pos
==
CONFIG
[
'A_POS'
]:
slit
=
'A'
...
...
@@ -442,10 +444,14 @@ class GBNodding():
snr
[
snr
==-
np
.
inf
]
=
0
snr
=
np
.
nan_to_num
(
snr
)
self
.
messages
.
append
(
'Nodding %s: SNR[Y band, order=73, wl=1050 nm] = %s'
%
(
str
(
slit
),
str
(
round
(
np
.
mean
(
snr
[
39
]),
2
))),)
self
.
messages
.
append
(
'Nodding %s: SNR[J band, order=61, wl=1250 nm] = %s'
%
(
str
(
slit
),
str
(
round
(
np
.
mean
(
snr
[
29
]),
2
))),)
self
.
messages
.
append
(
'Nodding %s: SNR[H band, order=46, wl=1650 nm] = %s'
%
(
str
(
slit
),
str
(
round
(
np
.
mean
(
snr
[
14
]),
2
))),)
self
.
messages
.
append
(
'Nodding %s: SNR[K band, order=35, wl=2200 nm] = %s'
%
(
str
(
slit
),
str
(
round
(
np
.
mean
(
snr
[
3
]),
2
))),)
snry
=
round
(
np
.
mean
(
snr
[
41
]),
2
)
snrj
=
round
(
np
.
mean
(
snr
[
29
]),
2
)
snrh
=
round
(
np
.
mean
(
snr
[
14
]),
2
)
snrk
=
round
(
np
.
mean
(
snr
[
3
]),
2
)
self
.
messages
.
append
(
'Nodding %s: SNR[Y band, order=73, wl=1050 nm] = %s'
%
(
str
(
slit
),
str
(
snry
),))
self
.
messages
.
append
(
'Nodding %s: SNR[J band, order=61, wl=1250 nm] = %s'
%
(
str
(
slit
),
str
(
snrj
),))
self
.
messages
.
append
(
'Nodding %s: SNR[H band, order=46, wl=1650 nm] = %s'
%
(
str
(
slit
),
str
(
snrh
),))
self
.
messages
.
append
(
'Nodding %s: SNR[K band, order=35, wl=2200 nm] = %s'
%
(
str
(
slit
),
str
(
snrk
),))
for
o
in
xrange
(
CONFIG
[
'N_ORD'
]):
...
...
@@ -489,6 +495,8 @@ class GBNodding():
#print 's1d'
#print calname1d
try
:
obj_name
=
heaspe
[
CONFIG
[
'KEYS'
][
'OBJECT'
]]
except
:
obj_name
=
'NONE'
if
CONFIG
[
'S1D'
]:
#s1d = varie.create_s1d(optSpectrum,snr,heaspe)
...
...
@@ -505,20 +513,27 @@ class GBNodding():
calname1d
=
os
.
path
.
join
(
CONFIG
[
'RED_DIR'
],
calname1d
)
s1dfits
.
writeto
(
calname1d
,
clobber
=
True
)
rid
=
varie
.
random_id
(
12
)
dbreduced
[
's1d'
]
=
{
'slit'
:
slit
,
'path'
:
calname1d
,
'snry'
:
snry
,
'snrj'
:
snrj
,
'snrh'
:
snrh
,
'snrk'
:
snrk
,
'type'
:
's1d'
,
'name'
:
obj_name
,
'id'
:
rid
}
rid
=
varie
.
random_id
(
12
)
dbreduced
[
'ms1d'
]
=
{
'slit'
:
slit
,
'path'
:
calname
,
'snry'
:
snry
,
'snrj'
:
snrj
,
'snrh'
:
snrh
,
'snrk'
:
snrk
,
'type'
:
'ms1d'
,
'name'
:
obj_name
,
'id'
:
rid
}
#t2 = time.time()
#print 's1d spectrum: %s s' % str(t2-t1)
if
hea_ima
[
CONFIG
[
'KEYS'
][
'EXTMODE'
]]
==
CONFIG
[
'EXTPAIR'
]:
return
calname
,
fsnr
,
straight
return
calname
,
fsnr
,
straight
,
dbreduced
elif
'grp'
in
fitsfile
:
return
calname
,
fsnr
,
straight
return
calname
,
fsnr
,
straight
,
dbreduced
return
calname
,
fsnr
,
False
return
calname
,
fsnr
,
False
,
dbreduced
def
combine
(
self
,
acalib
,
bcalib
,
fsnr
):
dbreduced
=
{}
#abnome = acalib.replace('_A_e2ds.fits','_AB_e2ds.fits')
#abnome1d = acalib.replace('_A_e2ds.fits','_AB_s1d.fits')
#abnome = abnome.replace('_Agrp_e2ds.fits','_ABgrp_e2ds.fits')
...
...
@@ -640,6 +655,15 @@ class GBNodding():
#results.writeto(abnome,clobber=True)
snry
=
round
(
np
.
mean
(
snr
[
41
]),
2
)
snrj
=
round
(
np
.
mean
(
snr
[
29
]),
2
)
snrh
=
round
(
np
.
mean
(
snr
[
14
]),
2
)
snrk
=
round
(
np
.
mean
(
snr
[
3
]),
2
)
self
.
messages
.
append
(
'Nodding %s: SNR[Y band, order=73, wl=1050 nm] = %s'
%
(
'AB'
,
str
(
snry
)),)
self
.
messages
.
append
(
'Nodding %s: SNR[J band, order=61, wl=1250 nm] = %s'
%
(
'AB'
,
str
(
snrj
)),)
self
.
messages
.
append
(
'Nodding %s: SNR[H band, order=46, wl=1650 nm] = %s'
%
(
'AB'
,
str
(
snrh
)),)
self
.
messages
.
append
(
'Nodding %s: SNR[K band, order=35, wl=2200 nm] = %s'
%
(
'AB'
,
str
(
snrk
)),)
orders
=
np
.
arange
(
CONFIG
[
'N_ORD'
])
+
32
c1
=
fits
.
Column
(
name
=
'ORDER'
,
format
=
'I'
,
array
=
orders
)
...
...
@@ -652,6 +676,9 @@ class GBNodding():
tbhdu
.
writeto
(
abnome
,
clobber
=
True
)
try
:
obj_name
=
heaspe
[
CONFIG
[
'KEYS'
][
'OBJECT'
]]
except
:
obj_name
=
'NONE'
if
CONFIG
[
'S1D'
]:
#s1d = varie.create_s1d(abcalib,snr,heaspe)
...
...
@@ -667,11 +694,12 @@ class GBNodding():
s1dfits
.
writeto
(
abnome1d
,
clobber
=
True
)
rid
=
varie
.
random_id
(
12
)
dbreduced
[
's1d'
]
=
{
'slit'
:
'AB'
,
'path'
:
abnome1d
,
'snry'
:
snry
,
'snrj'
:
snrj
,
'snrh'
:
snrh
,
'snrk'
:
snrk
,
'type'
:
's1d'
,
'name'
:
obj_name
,
'id'
:
rid
}
rid
=
varie
.
random_id
(
12
)
dbreduced
[
'ms1d'
]
=
{
'slit'
:
'AB'
,
'path'
:
abnome
,
'snry'
:
snry
,
'snrj'
:
snrj
,
'snrh'
:
snrh
,
'snrk'
:
snrk
,
'type'
:
'ms1d'
,
'name'
:
obj_name
,
'id'
:
rid
}
self
.
messages
.
append
(
'Nodding merged AB: SNR[Y band, order=73, wl=1050 nm] = %s'
%
(
str
(
round
(
np
.
mean
(
snr
[
41
]),
2
))),)
self
.
messages
.
append
(
'Nodding merged AB: SNR[J band, order=61, wl=1250 nm] = %s'
%
(
str
(
round
(
np
.
mean
(
snr
[
29
]),
2
))),)
self
.
messages
.
append
(
'Nodding merged AB: SNR[H band, order=46, wl=1650 nm] = %s'
%
(
str
(
round
(
np
.
mean
(
snr
[
14
]),
2
))),)
self
.
messages
.
append
(
'Nodding merged AB: SNR[K band, order=35, wl=2200 nm] = %s'
%
(
str
(
round
(
np
.
mean
(
snr
[
3
]),
2
))),)
#calibrated = np.vstack((np.concatenate(np.flipud(awaves)),np.concatenate(np.flipud(abcalib))))
...
...
@@ -680,7 +708,7 @@ class GBNodding():
#extract = abnome.replace('.fits','.txt')
#ascii.write(np.transpose(calibrated),extract)
return
return
dbreduced
def
group_avg
(
self
):
...
...
@@ -758,15 +786,38 @@ class GBNodding():
def
pair_process
(
self
):
reduced
=
','
.
join
(
map
(
str
,
self
.
nodding
))
stamp
=
time
.
time
()
db
.
insert_dbnight
(
self
.
dbnight
,
reduced
,
stamp
)
warnings
.
simplefilter
(
'ignore'
,
category
=
AstropyWarning
)
if
self
.
qualitycheck
():
#t1 = time.time()
ab
,
heaA
,
heaB
=
self
.
createAB
()
#t2 = time.time()
#print 'Bad pixels, create nodding: %s s' % str(t2-t1)
acalib
,
fsnr
,
straight
=
self
.
reduce
(
ab
,
CONFIG
[
'A_POS'
],
heaA
)
bcalib
,
fsnr
,
straight
=
self
.
reduce
(
ab
,
CONFIG
[
'B_POS'
],
heaB
)
self
.
combine
(
acalib
,
bcalib
,
fsnr
)
acalib
,
fsnr
,
straight
,
dbreduced
=
self
.
reduce
(
ab
,
CONFIG
[
'A_POS'
],
heaA
)
try
:
db
.
insert_dbreduced
(
self
.
dbnight
,
dbreduced
[
's1d'
],
stamp
)
except
:
pass
db
.
insert_dbreduced
(
self
.
dbnight
,
dbreduced
[
'ms1d'
],
stamp
)
bcalib
,
fsnr
,
straight
,
dbreduced
=
self
.
reduce
(
ab
,
CONFIG
[
'B_POS'
],
heaB
)
try
:
db
.
insert_dbreduced
(
self
.
dbnight
,
dbreduced
[
's1d'
],
stamp
)
except
:
pass
db
.
insert_dbreduced
(
self
.
dbnight
,
dbreduced
[
'ms1d'
],
stamp
)
dbreduced
=
self
.
combine
(
acalib
,
bcalib
,
fsnr
)
try
:
db
.
insert_dbreduced
(
self
.
dbnight
,
dbreduced
[
's1d'
],
stamp
)
except
:
pass
db
.
insert_dbreduced
(
self
.
dbnight
,
dbreduced
[
'ms1d'
],
stamp
)
if
straight
:
os
.
remove
(
ab
)
os
.
remove
(
straight
)
...
...
@@ -784,9 +835,9 @@ class GBNodding():
try
:
if
self
.
group
[
'noddings'
]
and
len
(
self
.
group
[
'noddings'
])
>
1
:
ab
,
heaA
,
heaB
=
self
.
group_avg
()
acalib
,
fsnr
,
straight
=
self
.
reduce
(
ab
,
CONFIG
[
'A_POS'
],
heaA
)
bcalib
,
fsnr
,
straight
=
self
.
reduce
(
ab
,
CONFIG
[
'B_POS'
],
heaB
)
self
.
combine
(
acalib
,
bcalib
,
fsnr
)
acalib
,
fsnr
,
straight
,
dbreduced
=
self
.
reduce
(
ab
,
CONFIG
[
'A_POS'
],
heaA
)
bcalib
,
fsnr
,
straight
,
dbreduced
=
self
.
reduce
(
ab
,
CONFIG
[
'B_POS'
],
heaB
)
dbreduced
=
self
.
combine
(
acalib
,
bcalib
,
fsnr
)
acalib
=
None
bcalib
=
None
os
.
remove
(
ab
)
...
...
@@ -804,9 +855,9 @@ class GBNodding():
try
:
if
self
.
group
[
'noddings'
]
and
len
(
self
.
group
[
'noddings'
])
>
1
:
ab
,
heaA
,
heaB
=
self
.
group_avg
()
acalib
,
fsnr
,
straight
=
self
.
reduce
(
ab
,
CONFIG
[
'A_POS'
],
heaA
)
bcalib
,
fsnr
,
straight
=
self
.
reduce
(
ab
,
CONFIG
[
'B_POS'
],
heaB
)
self
.
combine
(
acalib
,
bcalib
,
fsnr
)
acalib
,
fsnr
,
straight
,
dbreduced
=
self
.
reduce
(
ab
,
CONFIG
[
'A_POS'
],
heaA
)
bcalib
,
fsnr
,
straight
,
dbreduced
=
self
.
reduce
(
ab
,
CONFIG
[
'B_POS'
],
heaB
)
dbreduced
=
self
.
combine
(
acalib
,
bcalib
,
fsnr
)
acalib
=
None
bcalib
=
None
os
.
remove
(
ab
)
...
...
drslib/stare.py
View file @
59e78c6d
...
...
@@ -48,10 +48,11 @@ from collections import OrderedDict, Counter
class
GBStare
():
def
__init__
(
self
,
stare
,
group
,
dbconn
):
def
__init__
(
self
,
stare
,
group
,
dbconn
,
dbnight
):
self
.
stare
=
stare
self
.
group
=
group
self
.
dbconn
=
dbconn
self
.
dbnight
=
dbnight
self
.
quality
=
[]
self
.
messages
=
[]
self
.
starelist
=
[]
...
...
@@ -321,6 +322,8 @@ class GBStare():
"""
Straighten, divide by the masterflat, optimal extraction
"""
dbreduced
=
{}
# straighten
straight
=
fitsfile
.
replace
(
'.fits'
,
'_str.fits'
)
...
...
@@ -538,10 +541,16 @@ class GBStare():
snr
[
snr
==-
np
.
inf
]
=
0
snr
=
np
.
nan_to_num
(
snr
)
self
.
messages
.
append
(
'Stare image: SNR[Y band, order=73, wl=1050 nm] = %s'
%
(
str
(
round
(
np
.
mean
(
snr
[
41
]),
2
))),)
self
.
messages
.
append
(
'Stare image: SNR[J band, order=61, wl=1250 nm] = %s'
%
(
str
(
round
(
np
.
mean
(
snr
[
29
]),
2
))),)
self
.
messages
.
append
(
'Stare image: SNR[H band, order=46, wl=1650 nm] = %s'
%
(
str
(
round
(
np
.
mean
(
snr
[
14
]),
2
))),)
self
.
messages
.
append
(
'Stare image: SNR[K band, order=35, wl=2200 nm] = %s'
%
(
str
(
round
(
np
.
mean
(
snr
[
3
]),
2
))),)
snry
=
round
(
np
.
mean
(
snr
[
41
]),
2
)
snrj
=
round
(
np
.
mean
(
snr
[
29
]),
2
)
snrh
=
round
(
np
.
mean
(
snr
[
14
]),
2
)
snrk
=
round
(
np
.
mean
(
snr
[
3
]),
2
)
self
.
messages
.
append
(
'Stare image: SNR[Y band, order=73, wl=1050 nm] = %s'
%
(
str
(
snry
)),)
self
.
messages
.
append
(
'Stare image: SNR[J band, order=61, wl=1250 nm] = %s'
%
(
str
(
snrj
)),)
self
.
messages
.
append
(
'Stare image: SNR[H band, order=46, wl=1650 nm] = %s'
%
(
str
(
snrh
)),)
self
.
messages
.
append
(
'Stare image: SNR[K band, order=35, wl=2200 nm] = %s'
%
(
str
(
snrk
)),)
for
o
in
xrange
(
CONFIG
[
'N_ORD'
]):
key_snr
=
''
.
join
((
CONFIG
[
'SNR'
][
0
],
str
(
o
+
32
)))
...
...
@@ -577,6 +586,9 @@ class GBStare():
calname
=
os
.
path
.
join
(
CONFIG
[
'RED_DIR'
],
calname
)
tbhdu
.
writeto
(
calname
,
clobber
=
True
)
try
:
obj_name
=
heaspe
[
CONFIG
[
'KEYS'
][
'OBJECT'
]]
except
:
obj_name
=
'NONE'
if
CONFIG
[
'S1D'
]:
#s1d = varie.create_s1d(optSpectrum,snr,heaspe)
...
...
@@ -591,21 +603,39 @@ class GBStare():
s1dfits
=
fits
.
PrimaryHDU
(
s1d
,
header
=
heaspe
)
calname1d
=
os
.
path
.
join
(
CONFIG
[
'RED_DIR'
],
calname1d
)
s1dfits
.
writeto
(
calname1d
,
clobber
=
True
)
rid
=
varie
.
random_id
(
12
)
dbreduced
[
's1d'
]
=
{
'slit'
:
slit_pos
,
'path'
:
calname1d
,
'snry'
:
snry
,
'snrj'
:
snrj
,
'snrh'
:
snrh
,
'snrk'
:
snrk
,
'type'
:
's1d'
,
'name'
:
obj_name
,
'id'
:
rid
}
rid
=
varie
.
random_id
(
12
)
dbreduced
[
'ms1d'
]
=
{
'slit'
:
slit_pos
,
'path'
:
calname
,
'snry'
:
snry
,
'snrj'
:
snrj
,
'snrh'
:
snrh
,
'snrk'
:
snrk
,
'type'
:
'ms1d'
,
'name'
:
obj_name
,
'id'
:
rid
}
if
imstr
.
header
[
CONFIG
[
'KEYS'
][
'EXTMODE'
]]
==
CONFIG
[
'EXTPAIR'
]:
return
calname
,
straight
return
calname
,
straight
,
dbreduced
elif
'grp'
in
fitsfile
:
return
calname
,
straight
return
calname
,
straight
,
dbreduced
return
calname
,
False
return
calname
,
False
,
dbreduced
def
pair_process
(
self
):
reduced
=
','
.
join
(
map
(
str
,
self
.
stare
))
stamp
=
time
.
time
()
db
.
insert_dbnight
(
self
.
dbnight
,
reduced
,
stamp
)
warnings
.
simplefilter
(
'ignore'
,
category
=
AstropyWarning
)
if
self
.
qualitycheck
():
obj
=
self
.
createObj
(
False
)
calib
,
straight
=
self
.
reduce
(
obj
,
CONFIG
[
'C_POS'
])
calib
,
straight
,
dbreduced
=
self
.
reduce
(
obj
,
CONFIG
[
'C_POS'
])
try
:
db
.
insert_dbreduced
(
self
.
dbnight
,
dbreduced
[
's1d'
],
stamp
)
except
:
pass
db
.
insert_dbreduced
(
self
.
dbnight
,
dbreduced
[
'ms1d'
],
stamp
)
if
straight
:
os
.
remove
(
obj
)
os
.
remove
(
straight
)
...
...
@@ -623,7 +653,7 @@ class GBStare():
#print self.stare
if
self
.
qualitycheck
():
obj
=
self
.
createObj
(
True
)
calib
,
straight
=
self
.
reduce
(
obj
,
CONFIG
[
'C_POS'
])
calib
,
straight
,
dbreduced
=
self
.
reduce
(
obj
,
CONFIG
[
'C_POS'
])
if
straight
:
os
.
remove
(
obj
)
os
.
remove
(
straight
)
...
...
@@ -637,7 +667,7 @@ class GBStare():
self
.
stare
=
self
.
group
[
'stares'
]
if
self
.
qualitycheck
():
obj
=
self
.
createObj
(
True
)
calib
,
straight
=
self
.
reduce
(
obj
,
CONFIG
[
'C_POS'
])
calib
,
straight
,
dbreduced
=
self
.
reduce
(
obj
,
CONFIG
[
'C_POS'
])
if
straight
:
os
.
remove
(
obj
)
os
.
remove
(
straight
)
...
...
drslib/varie.py
View file @
59e78c6d
...
...
@@ -14,6 +14,7 @@ Last modified: 2017-06-07
- check_keyraw/check_keywords: check for keyword existence
- berv: computation of barycentric velocity correction (to be updated)
- create_s1d: create s1d output
- random_id: create random string
"""
from
drslib.config
import
CONFIG
...
...
@@ -28,6 +29,7 @@ import numpy as np
import
numpy.polynomial.polynomial
as
poly
import
math
import
warnings
import
string
,
random
#import matplotlib.pyplot as plt
#from scipy import optimize, interpolate, signal
...
...
@@ -1183,3 +1185,9 @@ def create_s1d(spectrum, header):
return
s1d
,
start
[
CONFIG
[
'N_ORD'
]
-
1
]
#--------------------- Create random string id -------------------
def
random_id
(
length
):
return
''
.
join
(
random
.
SystemRandom
().
choice
(
string
.
lowercase
+
string
.
uppercase
+
string
.
digits
)
for
i
in
range
(
length
))
gofioDRS.py
View file @
59e78c6d
...
...
@@ -33,6 +33,11 @@ try:
maskc
=
''
.
join
((
'GIANOB_MASKC_'
,
CONFIG
[
'DATE'
],
'.fits'
))
CONFIG
[
'MASK_C'
]
=
os
.
path
.
join
(
CONFIG
[
'RES_DIR'
],
maskc
)
CONFIG
[
'DB_OFFLINE'
]
=
''
.
join
((
'db_'
,
CONFIG
[
'DATE'
],
'_offline.db'
))
CONFIG
[
'DB_OFFLINE_PATH'
]
=
os
.
path
.
join
(
CONFIG
[
'WEBUI_DB_DIR'
],
CONFIG
[
'DB_OFFLINE'
])
CONFIG
[
'DB_NIGHT'
]
=
CONFIG
[
'DB_OFFLINE'
]
CONFIG
[
'DB_NIGHT_PATH'
]
=
CONFIG
[
'DB_OFFLINE_PATH'
]
except
:
pass
...
...
@@ -80,11 +85,20 @@ except OSError:
drslogger
=
DrsLogger
()
# Databases online/offline
dbcalib
=
db
.
create_db_calib
()
if
not
CONFIG
[
'OFFLINE'
]:
if
CONFIG
[
'OFFLINE'
]:
try
:
os
.
remove
(
CONFIG
[
'DB_NIGHT_PATH'
])
except
:
pass
else
:
dbramps
=
db
.
connect_db_ramp
()
dbnight
=
db
.
create_db_night
()
dbnight
=
db
.
create_db_night
()
do_calib
=
CONFIG
[
'DO_CALIB'
]
...
...
@@ -148,9 +162,9 @@ def GianoBreduce(rawfits,lists,dbcalib,working,group,do_calib,stop):
darks
.
process
()
for
message
in
darks
.
messages
:
drslogger
.
log
(
message
)
if
not
CONFIG
[
'OFFLINE'
]:
for
reduced
in
rawlists
[
CONFIG
[
'DARK'
]]:
db
.
insert_dbnight
(
dbnight
,
reduced
)
#
if not CONFIG['OFFLINE']:
#
for reduced in rawlists[CONFIG['DARK']]:
#
db.insert_dbnight(dbnight, reduced)
rawlists
[
CONFIG
[
'DARK'
]][:]
=
[]
group
.
clear
()
drslogger
.
log
(
'----------------------------------------------'
)
...
...
@@ -162,9 +176,9 @@ def GianoBreduce(rawfits,lists,dbcalib,working,group,do_calib,stop):
flats
.
process
()
for
message
in
flats
.
messages
:
drslogger
.
log
(
message
)
if
not
CONFIG
[
'OFFLINE'
]:
for
reduced
in
rawlists
[
CONFIG
[
'FLAT'
]]:
db
.
insert_dbnight
(
dbnight
,
reduced
)
#
if not CONFIG['OFFLINE']:
#
for reduced in rawlists[CONFIG['FLAT']]:
#
db.insert_dbnight(dbnight, reduced)
rawlists
[
CONFIG
[
'FLAT'
]][:]
=
[]
group
.
clear
()
drslogger
.
log
(
'----------------------------------------------'
)
...
...
@@ -176,9 +190,9 @@ def GianoBreduce(rawfits,lists,dbcalib,working,group,do_calib,stop):
wls
.
une_process
()
for
message
in
wls
.
messages
:
drslogger
.
log
(
message
)
if
not
CONFIG
[
'OFFLINE'
]:
for
reduced
in
rawlists
[
CONFIG
[
'WCAL_UNE'
]]:
db
.
insert_dbnight
(
dbnight
,
reduced
)
#
if not CONFIG['OFFLINE']:
#
for reduced in rawlists[CONFIG['WCAL_UNE']]:
#
db.insert_dbnight(dbnight, reduced)
rawlists
[
CONFIG
[
'WCAL_UNE'
]][:]
=
[]
group
.
clear
()
drslogger
.
log
(
'----------------------------------------------'
)
...
...
@@ -190,9 +204,9 @@ def GianoBreduce(rawfits,lists,dbcalib,working,group,do_calib,stop):
fps
.
fp_process
()
for
message
in
fps
.
messages
:
drslogger
.
log
(
message
)
if
not
CONFIG
[
'OFFLINE'
]:
for
reduced
in
rawlists
[
CONFIG
[
'WCAL_FP'
]]:
db
.
insert_dbnight
(
dbnight
,
reduced
)
#
if not CONFIG['OFFLINE']:
#
for reduced in rawlists[CONFIG['WCAL_FP']]:
#
db.insert_dbnight(dbnight, reduced)
rawlists
[
CONFIG
[
'WCAL_FP'
]][:]
=
[]
group
.
clear
()
drslogger
.
log
(
'----------------------------------------------'
)
...
...
@@ -201,15 +215,15 @@ def GianoBreduce(rawfits,lists,dbcalib,working,group,do_calib,stop):
drslogger
.
log
(
'----------------------------------------------'
)
drslogger
.
log
(
' *** NODDING PAIR *** '
)
t1
=
time
.
time
()
nods
=
nodding
.
GBNodding
(
rawlists
[
CONFIG
[
'SCIENCE'
]],
group
,
dbcalib
)
nods
=
nodding
.
GBNodding
(
rawlists
[
CONFIG
[
'SCIENCE'
]],
group
,
dbcalib
,
dbnight
)
nods
.
pair_process
()
for
message
in
nods
.
messages
:
drslogger
.
log
(
message
)
t2
=
time
.
time
()
drslogger
.
log
(
' *** Reduction: %s s ***'
%
(
str
(
round
((
t2
-
t1
),
2
)),
))
if
not
CONFIG
[
'OFFLINE'
]:
for
reduced
in
rawlists
[
CONFIG
[
'SCIENCE'
]]:
db
.
insert_dbnight
(
dbnight
,
reduced
)
#
if not CONFIG['OFFLINE']:
#
for reduced in rawlists[CONFIG['SCIENCE']]:
#
db.insert_dbnight(dbnight, reduced)
rawlists
[
CONFIG
[
'SCIENCE'
]][:]
=
[]
drslogger
.
log
(
'----------------------------------------------'
)
...
...
@@ -217,15 +231,15 @@ def GianoBreduce(rawfits,lists,dbcalib,working,group,do_calib,stop):
drslogger
.
log
(
'----------------------------------------------'
)
drslogger
.
log
(
' *** STARE PAIR *** '
)
t1
=
time
.
time
()
stares
=
stare
.
GBStare
(
rawlists
[
CONFIG
[
'SCIENCE'
]],
group
,
dbcalib
)
stares
=
stare
.
GBStare
(
rawlists
[
CONFIG
[
'SCIENCE'
]],
group
,
dbcalib
,
dbnight
)
stares
.
pair_process
()
for
message
in
stares
.
messages
:
drslogger
.
log
(
message
)
t2
=
time
.
time
()
drslogger
.
log
(
' *** Reduction: %s s ***'
%
(
str
(
round
((
t2
-
t1
),
2
)),
))
if
not
CONFIG
[
'OFFLINE'
]:
for
reduced
in
rawlists
[
CONFIG
[
'SCIENCE'
]]:
db
.
insert_dbnight
(
dbnight
,
reduced
)
#
if not CONFIG['OFFLINE']:
#
for reduced in rawlists[CONFIG['SCIENCE']]:
#
db.insert_dbnight(dbnight, reduced)
rawlists
[
CONFIG
[
'SCIENCE'
]][:]
=
[]
drslogger
.
log
(
'----------------------------------------------'
)