Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Avet Harutyunyan
gofio
Commits
60a29b8a
Commit
60a29b8a
authored
Nov 22, 2017
by
Monica Rainer
Browse files
Change structure, add gofio.py
parent
8a9821e2
Changes
28
Hide whitespace changes
Inline
Side-by-side
.gitignore
View file @
60a29b8a
backup
calibrations
git_howto.txt
gofio_latest
gofio_latest.tar
gofioDRS.py
reduced
varie
vradlib
...
...
@@ -18,4 +16,5 @@ webuidatabases
*/*/*/*.pyc
*~
.~*
gofio/calibrations
README.md
View file @
60a29b8a
...
...
@@ -7,6 +7,7 @@ Python packages:
-
NumPy v1.12
-
SciPy v0.19
-
watchdog v0.8.2
-
docopt v0.6.1
Output:
-
intermediate:
*
_str.fits, 2D images with orders straightened and bad pixel removed
...
...
drslib
/__init__.py
→
gofio
/__init__.py
View file @
60a29b8a
File moved
drslib/
berv/
__init__.py
→
gofio/
drslib/__init__.py
View file @
60a29b8a
File moved
gofio/drslib/berv/__init__.py
0 → 100644
View file @
60a29b8a
drslib/berv/astroTimeLegacy.py
→
gofio/
drslib/berv/astroTimeLegacy.py
View file @
60a29b8a
File moved
drslib/berv/baryvel.py
→
gofio/
drslib/berv/baryvel.py
View file @
60a29b8a
File moved
drslib/clean_db.py
→
gofio/
drslib/clean_db.py
View file @
60a29b8a
File moved
drslib/config.py
→
gofio/
drslib/config.py
View file @
60a29b8a
...
...
@@ -220,6 +220,7 @@ CONFIG['MASK_C'] = os.path.join(CONFIG['RES_DIR'], 'GIANOB_MASKC.fits')
CONFIG
[
'A_POS'
]
=
8.
# guess pixel position for A
CONFIG
[
'B_POS'
]
=
23.
# guess pixel position for B
CONFIG
[
'C_POS'
]
=
15.
# guess pixel position for C
CONFIG
[
'Y_POS'
]
=
2
# max. pixel difference between expected and real pixel position
CONFIG
[
'HWTM'
]
=
6.
# hwtm limit for contamination between A and B
CONFIG
[
'S1D'
]
=
True
# flag for creation of s1d output (True/False)
CONFIG
[
'S1D_NORM'
]
=
False
# flag for normalization of s1d output (True/False)
...
...
drslib/darkframes.py
→
gofio/
drslib/darkframes.py
View file @
60a29b8a
File moved
drslib/db.py
→
gofio/
drslib/db.py
View file @
60a29b8a
File moved
drslib/flatframes.py
→
gofio/
drslib/flatframes.py
View file @
60a29b8a
File moved
gofio/drslib/gofio.py
0 → 100644
View file @
60a29b8a
import
os
,
sys
,
time
from
drslib.config
import
CONFIG
,
read_usr_config
from
drslib.logger
import
DrsLogger
from
drslib
import
db
,
rawfiles
,
darkframes
,
flatframes
,
wlframes
,
nodding
,
stare
def
gofio_init
(
docopt_args
):
if
docopt_args
[
'--cfg'
]:
try
:
USRCONFIG
=
read_usr_config
(
docopt_args
[
'-g'
])
except
:
try
:
USRCONFIG
=
read_usr_config
(
docopt_args
[
'--cfg'
])
except
:
USRCONFIG
=
{}
USRCONFIG
[
'OFFLINE'
]
=
False
if
USRCONFIG
[
'OFFLINE'
]:
for
key
in
USRCONFIG
.
keys
():
CONFIG
[
key
]
=
USRCONFIG
[
key
]
CONFIG
[
'RAW_DIR'
]
=
os
.
path
.
join
(
CONFIG
[
'BASE_RAW'
],
CONFIG
[
'DATE'
])
CONFIG
[
'OFFLINE_DIR'
]
=
os
.
path
.
join
(
CONFIG
[
'BASE_RED_DIR'
],
CONFIG
[
'DATE'
])
CONFIG
[
'RED_DIR'
]
=
os
.
path
.
join
(
CONFIG
[
'OFFLINE_DIR'
],
'offline'
)
CONFIG
[
'RED_CALIB'
]
=
os
.
path
.
join
(
CONFIG
[
'RED_DIR'
],
'CALIB'
)
CONFIG
[
'RED_STR'
]
=
os
.
path
.
join
(
CONFIG
[
'RED_DIR'
],
'STR'
)
CONFIG
[
'LOG_FILE'
]
=
os
.
path
.
join
(
CONFIG
[
'RED_DIR'
],
'drs.log'
)
CONFIG
[
'RAMP_NAME'
]
=
'ramps'
+
CONFIG
[
'DATE'
]
+
'.db'
CONFIG
[
'DB_RAMP'
]
=
os
.
path
.
join
(
CONFIG
[
'BASE_RAMP'
],
CONFIG
[
'RAMP_NAME'
])
CONFIG
[
'DB_OFFLINE'
]
=
''
.
join
((
'db_'
,
CONFIG
[
'DATE'
],
'_offline.db'
))
CONFIG
[
'DB_OFFLINE_PATH'
]
=
os
.
path
.
join
(
CONFIG
[
'WEBUI_DB_DIR'
],
CONFIG
[
'DB_OFFLINE'
])
CONFIG
[
'DB_NIGHT'
]
=
CONFIG
[
'DB_OFFLINE'
]
CONFIG
[
'DB_NIGHT_PATH'
]
=
CONFIG
[
'DB_OFFLINE_PATH'
]
maskc
=
''
.
join
((
'GIANOB_MASKC_'
,
CONFIG
[
'DATE'
],
'.fits'
))
CONFIG
[
'MASK_C'
]
=
os
.
path
.
join
(
CONFIG
[
'RES_DIR'
],
maskc
)
if
docopt_args
[
'<date>'
]:
CONFIG
[
'DATE'
]
=
docopt_args
[
'<date>'
]
CONFIG
[
'OFFLINE'
]
=
True
CONFIG
[
'RAW_DIR'
]
=
os
.
path
.
join
(
CONFIG
[
'BASE_RAW'
],
CONFIG
[
'DATE'
])
CONFIG
[
'OFFLINE_DIR'
]
=
os
.
path
.
join
(
CONFIG
[
'BASE_RED_DIR'
],
CONFIG
[
'DATE'
])
CONFIG
[
'RED_DIR'
]
=
os
.
path
.
join
(
CONFIG
[
'OFFLINE_DIR'
],
'offline'
)
CONFIG
[
'RED_CALIB'
]
=
os
.
path
.
join
(
CONFIG
[
'RED_DIR'
],
'CALIB'
)
CONFIG
[
'RED_STR'
]
=
os
.
path
.
join
(
CONFIG
[
'RED_DIR'
],
'STR'
)
CONFIG
[
'LOG_FILE'
]
=
os
.
path
.
join
(
CONFIG
[
'RED_DIR'
],
'drs.log'
)
CONFIG
[
'DO_CALIB'
]
=
{
'dark'
:
docopt_args
[
'--dark'
],
'flat'
:
docopt_args
[
'--flat'
],
'une'
:
docopt_args
[
'--une'
],
'fp'
:
docopt_args
[
'--fp'
],
'only_calib'
:
docopt_args
[
'--only_calib'
]}
if
docopt_args
[
'--use_flat'
]
==
'global'
:
CONFIG
[
'USE_FLAT'
]
=
{
'global'
:
True
,
'order'
:
False
,
'nor'
:
False
}
elif
docopt_args
[
'--use_flat'
]
==
'order'
:
CONFIG
[
'USE_FLAT'
]
=
{
'global'
:
False
,
'order'
:
True
,
'nor'
:
False
}
elif
docopt_args
[
'--use_flat'
]
==
'nor'
:
CONFIG
[
'USE_FLAT'
]
=
{
'global'
:
False
,
'order'
:
False
,
'nor'
:
True
}
else
:
CONFIG
[
'USE_FLAT'
]
=
{
'global'
:
False
,
'order'
:
False
,
'nor'
:
False
}
maskc
=
''
.
join
((
'GIANOB_MASKC_'
,
CONFIG
[
'DATE'
],
'.fits'
))
CONFIG
[
'MASK_C'
]
=
os
.
path
.
join
(
CONFIG
[
'RES_DIR'
],
maskc
)
CONFIG
[
'DB_OFFLINE'
]
=
''
.
join
((
'db_'
,
CONFIG
[
'DATE'
],
'_offline.db'
))
CONFIG
[
'DB_OFFLINE_PATH'
]
=
os
.
path
.
join
(
CONFIG
[
'WEBUI_DB_DIR'
],
CONFIG
[
'DB_OFFLINE'
])
CONFIG
[
'DB_NIGHT'
]
=
CONFIG
[
'DB_OFFLINE'
]
CONFIG
[
'DB_NIGHT_PATH'
]
=
CONFIG
[
'DB_OFFLINE_PATH'
]
# second argument: how to query the calibration database
# to be set if the the user wants to reduce the night using
# another night's calibrations
if
docopt_args
[
'<calib_date>'
]:
CONFIG
[
'DATE'
]
=
docopt_args
[
'<calib_date>'
]
# remove the mask with the order position (if it exists)
# it will be re-build with the correct flat-field
try
:
os
.
remove
(
CONFIG
[
'MASK_C'
])
except
:
pass
# Create the reduction directory
try
:
os
.
mkdir
(
CONFIG
[
'BASE_RED_DIR'
])
except
OSError
:
pass
if
CONFIG
[
'OFFLINE'
]:
try
:
os
.
mkdir
(
CONFIG
[
'OFFLINE_DIR'
])
except
OSError
:
pass
try
:
os
.
mkdir
(
CONFIG
[
'RED_DIR'
])
except
OSError
:
pass
try
:
os
.
mkdir
(
CONFIG
[
'RED_CALIB'
])
os
.
mkdir
(
CONFIG
[
'RED_STR'
])
except
OSError
:
pass
# Create the calibration directory tree
try
:
os
.
mkdir
(
CONFIG
[
'BASE_CALIB_DIR'
])
except
OSError
:
pass
try
:
os
.
mkdir
(
CONFIG
[
'CALIB_DIR'
])
except
OSError
:
pass
# initialize logger
drslogger
=
DrsLogger
()
# Databases online/offline
dbcalib
=
db
.
create_db_calib
()
if
CONFIG
[
'OFFLINE'
]:
dbramps
=
False
try
:
os
.
remove
(
CONFIG
[
'DB_NIGHT_PATH'
])
except
:
pass
else
:
dbramps
=
db
.
connect_db_ramp
()
dbnight
=
db
.
create_db_night
()
if
not
os
.
path
.
exists
(
CONFIG
[
'RAW_DIR'
]):
drslogger
.
log
(
'The RAW directory %s does not exist. GOFIO will now close down.'
%
CONFIG
[
'RAW_DIR'
])
drslogger
.
log
(
'Pipeline stopped.'
)
dbcalib
.
close
()
if
not
CONFIG
[
'OFFLINE'
]:
dbramps
.
close
()
dbnight
.
close
()
sys
.
exit
(
0
)
rawlists
=
{
CONFIG
[
'DARK'
]:[],
CONFIG
[
'FLAT'
]:[],
CONFIG
[
'WCAL_UNE'
]:[],
CONFIG
[
'WCAL_FP'
]:[],
CONFIG
[
'SCIENCE'
]:[]}
# store the information on what the pipeline is working on (dark, flat, ...) and which science group/pair
working
=
0
group
=
{}
stop
=
False
# --------------------------------
# End initializing GOFIO
# --------------------------------
return
CONFIG
,
rawlists
,
dbcalib
,
working
,
group
,
stop
,
drslogger
,
dbnight
,
dbramps
# data reduction
def
GianoBreduce
(
rawfits
,
rawlists
,
dbcalib
,
working
,
group
,
do_calib
,
stop
,
drslogger
,
dbnight
):
if
stop
:
try
:
raw
=
rawfiles
.
GBCheckRawFiles
(
None
,
None
,
None
,
None
,
None
)
group
=
raw
.
group
working
=
raw
.
working
if
working
==
CONFIG
[
'SCIENCE'
]:
if
group
[
'nodstare'
]
==
CONFIG
[
'NODVALUE'
]
and
group
[
'pair'
]:
rawlists
[
CONFIG
[
'SCIENCE'
]].
pop
()
if
group
:
drslogger
.
log
(
'Reducing pending frames.'
)
print
'Press Ctrl-C again to stop the pipeline now.'
raw
.
command
[
group
[
'nodstare'
]]
=
'incomplete_group'
elif
working
:
drslogger
.
log
(
'Reducing pending %s frames.'
%
working
)
print
'Press Ctrl-C again to stop the pipeline now.'
raw
.
command
[
working
]
=
True
else
:
return
except
KeyboardInterrupt
:
drslogger
.
log
(
'Pipeline stopped.'
)
sys
.
exit
(
0
)
# check raw file and insert in the correct list
else
:
drslogger
.
log
(
'Checking the FITS file '
+
str
(
os
.
path
.
basename
(
rawfits
)))
raw
=
rawfiles
.
GBCheckRawFiles
(
rawfits
,
rawlists
,
working
,
group
,
do_calib
)
raw
.
check_raw_file
()
working
=
raw
.
working
group
=
raw
.
group
go
=
raw
.
go
#do_calib = raw.do_calib
for
message
in
raw
.
messages
:
drslogger
.
log
(
message
)
#print group
# see if the previous function passed a command and execute it
if
raw
.
command
[
CONFIG
[
'DARK'
]]:
drslogger
.
log
(
'----------------------------------------------'
)
drslogger
.
log
(
' *** DARK FRAMES *** '
)
stamp
=
time
.
time
()
darks
=
darkframes
.
GBDarks
(
rawlists
[
CONFIG
[
'DARK'
]],
dbcalib
)
darks
.
process
()
for
message
in
darks
.
messages
:
drslogger
.
log
(
message
)
db
.
insert_calib
(
dbnight
,
'DARK'
,
darks
.
darks
,
darks
.
quality
,
stamp
)
rawlists
[
CONFIG
[
'DARK'
]][:]
=
[]
group
.
clear
()
drslogger
.
log
(
'----------------------------------------------'
)
elif
raw
.
command
[
CONFIG
[
'FLAT'
]]:
drslogger
.
log
(
'----------------------------------------------'
)
drslogger
.
log
(
' *** FLAT-FIELD FRAMES *** '
)
stamp
=
time
.
time
()
flats
=
flatframes
.
GBFlats
(
rawlists
[
CONFIG
[
'FLAT'
]],
dbcalib
)
flats
.
process
()
for
message
in
flats
.
messages
:
drslogger
.
log
(
message
)
db
.
insert_calib
(
dbnight
,
'FLAT'
,
flats
.
flats
,
flats
.
quality
,
stamp
)
rawlists
[
CONFIG
[
'FLAT'
]][:]
=
[]
group
.
clear
()
drslogger
.
log
(
'----------------------------------------------'
)
elif
raw
.
command
[
CONFIG
[
'WCAL_UNE'
]]:
drslogger
.
log
(
'----------------------------------------------'
)
drslogger
.
log
(
' *** UNE LAMP FRAMES *** '
)
stamp
=
time
.
time
()
wls
=
wlframes
.
GBWls
(
rawlists
[
CONFIG
[
'WCAL_UNE'
]],
dbcalib
)
wls
.
une_process
()
for
message
in
wls
.
messages
:
drslogger
.
log
(
message
)
db
.
insert_calib
(
dbnight
,
'WCAL_UNE'
,
wls
.
wls
,
wls
.
quality
,
stamp
)
rawlists
[
CONFIG
[
'WCAL_UNE'
]][:]
=
[]
group
.
clear
()
drslogger
.
log
(
'----------------------------------------------'
)
elif
raw
.
command
[
CONFIG
[
'WCAL_FP'
]]:
drslogger
.
log
(
'----------------------------------------------'
)
drslogger
.
log
(
' *** FP LAMP FRAMES *** '
)
stamp
=
time
.
time
()
fps
=
wlframes
.
GBWls
(
rawlists
[
CONFIG
[
'WCAL_FP'
]],
dbcalib
)
fps
.
fp_process
()
for
message
in
fps
.
messages
:
drslogger
.
log
(
message
)
db
.
insert_calib
(
dbnight
,
'WCAL_FP'
,
fps
.
wls
,
fps
.
quality
,
stamp
)
rawlists
[
CONFIG
[
'WCAL_FP'
]][:]
=
[]
group
.
clear
()
drslogger
.
log
(
'----------------------------------------------'
)
elif
raw
.
command
[
CONFIG
[
'NODVALUE'
]]
==
'pair'
:
drslogger
.
log
(
'----------------------------------------------'
)
drslogger
.
log
(
' *** NODDING PAIR *** '
)
t1
=
time
.
time
()
nods
=
nodding
.
GBNodding
(
rawlists
[
CONFIG
[
'SCIENCE'
]],
group
,
dbcalib
,
dbnight
)
nods
.
pair_process
()
for
message
in
nods
.
messages
:
drslogger
.
log
(
message
)
t2
=
time
.
time
()
drslogger
.
log
(
' *** Reduction: %s s ***'
%
(
str
(
round
((
t2
-
t1
),
2
)),
))
rawlists
[
CONFIG
[
'SCIENCE'
]][:]
=
[]
drslogger
.
log
(
'----------------------------------------------'
)
elif
raw
.
command
[
CONFIG
[
'STAREVALUE'
]]
==
'pair'
:
drslogger
.
log
(
'----------------------------------------------'
)
drslogger
.
log
(
' *** STARE PAIR *** '
)
t1
=
time
.
time
()
stares
=
stare
.
GBStare
(
rawlists
[
CONFIG
[
'SCIENCE'
]],
group
,
dbcalib
,
dbnight
)
stares
.
pair_process
()
for
message
in
stares
.
messages
:
drslogger
.
log
(
message
)
t2
=
time
.
time
()
drslogger
.
log
(
' *** Reduction: %s s ***'
%
(
str
(
round
((
t2
-
t1
),
2
)),
))
rawlists
[
CONFIG
[
'SCIENCE'
]][:]
=
[]
drslogger
.
log
(
'----------------------------------------------'
)
elif
raw
.
command
[
CONFIG
[
'NODVALUE'
]]
==
'group'
:
drslogger
.
log
(
'----------------------------------------------'
)
drslogger
.
log
(
' *** NODDING GROUP *** '
)
t1
=
time
.
time
()
nods
=
nodding
.
GBNodding
(
rawlists
[
CONFIG
[
'SCIENCE'
]],
group
,
dbcalib
,
dbnight
)
nods
.
group_process
()
for
message
in
nods
.
messages
:
drslogger
.
log
(
message
)
t2
=
time
.
time
()
drslogger
.
log
(
' *** Reduction: %s s ***'
%
(
str
(
round
((
t2
-
t1
),
2
)),
))
rawlists
[
CONFIG
[
'SCIENCE'
]][:]
=
[]
drslogger
.
log
(
'----------------------------------------------'
)
elif
raw
.
command
[
CONFIG
[
'STAREVALUE'
]]
==
'group'
:
drslogger
.
log
(
'----------------------------------------------'
)
drslogger
.
log
(
' *** STARE GROUP *** '
)
t1
=
time
.
time
()
stares
=
stare
.
GBStare
(
rawlists
[
CONFIG
[
'SCIENCE'
]],
group
,
dbcalib
,
dbnight
)
stares
.
group_process
()
for
message
in
stares
.
messages
:
drslogger
.
log
(
message
)
t2
=
time
.
time
()
drslogger
.
log
(
' *** Reduction: %s s ***'
%
(
str
(
round
((
t2
-
t1
),
2
)),
))
rawlists
[
CONFIG
[
'SCIENCE'
]][:]
=
[]
drslogger
.
log
(
'----------------------------------------------'
)
elif
raw
.
command
[
CONFIG
[
'NODVALUE'
]]
==
'incomplete_group'
:
drslogger
.
log
(
'----------------------------------------------'
)
drslogger
.
log
(
' *** NODDING INCOMPLETE GROUP *** '
)
t1
=
time
.
time
()
nods
=
nodding
.
GBNodding
(
rawlists
[
CONFIG
[
'SCIENCE'
]],
group
,
dbcalib
,
dbnight
)
nods
.
ingroup_process
()
for
message
in
nods
.
messages
:
drslogger
.
log
(
message
)
t2
=
time
.
time
()
drslogger
.
log
(
' *** Reduction: %s s ***'
%
(
str
(
round
((
t2
-
t1
),
2
)),
))
rawlists
[
CONFIG
[
'SCIENCE'
]][:]
=
[]
drslogger
.
log
(
'----------------------------------------------'
)
elif
raw
.
command
[
CONFIG
[
'STAREVALUE'
]]
==
'incomplete_group'
:
drslogger
.
log
(
'----------------------------------------------'
)
drslogger
.
log
(
' *** STARE INCOMPLETE GROUP *** '
)
t1
=
time
.
time
()
stares
=
stare
.
GBStare
(
rawlists
[
CONFIG
[
'SCIENCE'
]],
group
,
dbcalib
,
dbnight
)
stares
.
ingroup_process
()
for
message
in
stares
.
messages
:
drslogger
.
log
(
message
)
t2
=
time
.
time
()
drslogger
.
log
(
' *** Reduction: %s s ***'
%
(
str
(
round
((
t2
-
t1
),
2
)),
))
rawlists
[
CONFIG
[
'SCIENCE'
]][:]
=
[]
drslogger
.
log
(
'----------------------------------------------'
)
else
:
pass
if
stop
:
return
#print group
return
working
,
group
,
go
,
do_calib
,
rawlists
drslib/logger.py
→
gofio/
drslib/logger.py
View file @
60a29b8a
File moved
drslib/nodding.py
→
gofio/
drslib/nodding.py
View file @
60a29b8a
"""
Last modified: 2017-03-07
Reduction of the single AB nodding:
- check the image quality (signal in well defined region)
- check that the exposure times of A and B are the same
...
...
drslib/rawfiles.py
→
gofio/
drslib/rawfiles.py
View file @
60a29b8a
"""
Last modified: 2017-03-08
The FITS files are divided in darks, flats, UNes, FPs and science
using the header keywords (defined in config.py).
If all the images of one group have been taken, the command to
process them is passed to the pipeline.
If a different kind of image arrives, the previous ones are processed
even if the group is incomplete.
WARNING: the dark files are
4
times the expected number, because for
each observations the ramp-processor creates
4
images with different
exposure times (10, 30, 60
and
100 seconds).
WARNING: the dark files are
n
times the expected number, because for
each observations the ramp-processor creates
n
images with different
exposure times (10, 30, 60
,
100 seconds
, and so on
).
"""
from
drslib.config
import
CONFIG
...
...
drslib/stare.py
→
gofio/
drslib/stare.py
View file @
60a29b8a
"""
Last modified: 2017-03-07
Reduction of the single Obj-Sky group:
- check the Obj image quality (signal in well defined region)
- check that the exposure times of Obj and Sky are the same
...
...
drslib/varie.py
→
gofio/
drslib/varie.py
View file @
60a29b8a
"""
Last modified: 2017-06-07
- badpix: bad pixels removals
- stdcombine: weights for flat and dark combiner
- optExtract: optimal extraction
...
...
@@ -186,7 +184,7 @@ def optExtract(data,gain,ron,slit_pos,ordine):
hwtm
=
math
.
sqrt
(
2
*
math
.
log
(
10
))
*
abs
(
sigmagauss
)
# half-width at tenth-maximum
if
abs
(
x0
-
slit_pos
)
>
2
:
if
abs
(
x0
-
slit_pos
)
>
CONFIG
[
'Y_POS'
]
:
x0
=
slit_pos
# if bad seeing, there can be overlapping between A and B
...
...
drslib/wlframes.py
→
gofio/
drslib/wlframes.py
View file @
60a29b8a
File moved
resources/GIANOB_MASKC.fits
→
gofio/
resources/GIANOB_MASKC.fits
View file @
60a29b8a
File moved
Prev
1
2
Next
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment