diff --git a/.gitignore b/.gitignore index 78414a8..170c6fc 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ *__pycache__* /build *.egg -/dist \ No newline at end of file +/dist +/graphs diff --git a/kcwidrp/configs/framework.cfg b/kcwidrp/configs/framework.cfg index 4008d41..64f008f 100644 --- a/kcwidrp/configs/framework.cfg +++ b/kcwidrp/configs/framework.cfg @@ -101,7 +101,7 @@ http_defaultFile = "" # The queue_manager_hostname is used for clients to connect to the server so share the event queue. # The authentication code is arbitrary. # -want_multiprocessing = True +want_multiprocessing = False queue_manager_hostname = "localhost" queue_manager_portnr = 50101 queue_manager_auth_code = b"kcwi" diff --git a/kcwidrp/configs/rti.cfg b/kcwidrp/configs/rti.cfg index 8346968..2b603ea 100644 --- a/kcwidrp/configs/rti.cfg +++ b/kcwidrp/configs/rti.cfg @@ -8,7 +8,6 @@ rti_retry_time = 5 # How many seconds to wait before trying connection again rti_user = '' rti_pass = '' # RTI API parameters -rti_ingesttype = 'lev1' rti_reingest = False rti_testonly = False rti_dev = True \ No newline at end of file diff --git a/kcwidrp/core/kcwi_proctab.py b/kcwidrp/core/kcwi_proctab.py index efc76ec..fa5d8bf 100644 --- a/kcwidrp/core/kcwi_proctab.py +++ b/kcwidrp/core/kcwi_proctab.py @@ -135,7 +135,6 @@ def update_proctab(self, frame, suffix='raw', newtype=None, filename=""): filename] else: new_row = None - # print("Attempting to add %s" % str(new_row)) self.proctab.add_row(new_row) self.proctab = unique(self.proctab, keys=['CID', 'FRAMENO', 'TYPE'], keep='last') diff --git a/kcwidrp/pipelines/keck_rti_pipeline.py b/kcwidrp/pipelines/keck_rti_pipeline.py index d7fbf02..df8d5f6 100644 --- a/kcwidrp/pipelines/keck_rti_pipeline.py +++ b/kcwidrp/pipelines/keck_rti_pipeline.py @@ -1,13 +1,12 @@ """ -KCWI +KCWI pipeline for RTI operations at WMKO -@author: lrizzi, mbrodheim +@author: lrizzi, mbrodheim, jneill """ from keckdrpframework.pipelines.base_pipeline import BasePipeline from keckdrpframework.models.processing_context import ProcessingContext from kcwidrp.primitives.kcwi_file_primitives import * -from kcwidrp.core.kcwi_proctab import Proctab from datetime import datetime @@ -33,22 +32,28 @@ class Keck_RTI_Pipeline(BasePipeline): # BIAS PROCESSING "process_bias": ("ProcessBias", "bias_processing_started", + "bias_subtract_overscan"), + "bias_subtract_overscan": ("SubtractOverscan", + "subtract_overscan_started", + "bias_trim_overscan"), + "bias_trim_overscan": ("TrimOverscan", + "trim_overscan_started", # intb "bias_make_master"), "bias_make_master": ("MakeMasterBias", - "master_bias_started", + "master_bias_started", # mbias None), # DARK PROCESSING "process_dark": ("ProcessDark", "dark_processing_started", - "dark_subtract_bias"), - "dark_subtract_bias": ("SubtractBias", - "subtract_bias_started", "dark_subtract_overscan"), "dark_subtract_overscan": ("SubtractOverscan", "subtract_overscan_started", "dark_trim_overscan"), "dark_trim_overscan": ("TrimOverscan", "trim_overscan_started", + "dark_subtract_bias"), + "dark_subtract_bias": ("SubtractBias", + "subtract_bias_started", "dark_correct_gain"), "dark_correct_gain": ("CorrectGain", "gain_correction_started", @@ -63,10 +68,10 @@ class Keck_RTI_Pipeline(BasePipeline): "create_unc_started", "dark_rectify_image"), "dark_rectify_image": ("RectifyImage", - "rectification_started", + "rectification_started", # int "dark_make_master"), "dark_make_master": ("MakeMasterDark", - "master_dark_started", + "master_dark_started", # mdark None), # CONTBARS PROCESSING "process_contbars": ("ProcessContbars", @@ -82,13 +87,16 @@ class Keck_RTI_Pipeline(BasePipeline): "gain_correction_started", "contbar_rectify_image"), "contbar_rectify_image": ("RectifyImage", - "rectification_started", + "rectification_started", # int + "contbar_make_master"), + "contbar_make_master": ("MakeMasterContbars", + "master_contbar_started", # mcont "contbar_find_bars"), "contbar_find_bars": ("FindBars", "find_bars_started", "contbar_trace_bars"), "contbar_trace_bars": ("TraceBars", - "trace_bars_started", + "trace_bars_started", # trace None), # ARCS PROCESSING "process_arc": ("ProcessArc", @@ -102,12 +110,18 @@ class Keck_RTI_Pipeline(BasePipeline): "arcs_correct_gain"), "arcs_correct_gain": ("CorrectGain", "gain_correction_started", + "arcs_correct_defects"), + "arcs_correct_defects": ("CorrectDefects", + "defect_correction_started", "arcs_create_unc"), "arcs_create_unc": ("CreateUncertaintyImage", "create_unc_started", "arcs_rectify_image"), "arcs_rectify_image": ("RectifyImage", - "rectification_started", + "rectification_started", # int + "arcs_make_master"), + "arcs_make_master": ("MakeMasterArc", + "master_arcs_started", # marc "arcs_extract_arcs"), "arcs_extract_arcs": ("ExtractArcs", "extract_arcs_started", @@ -131,66 +145,78 @@ class Keck_RTI_Pipeline(BasePipeline): "solving_arcs_started", "arcs_solve_geom"), "arcs_solve_geom": ("SolveGeom", - "solving_geom_started", + "solving_geom_started", # geom "arcs_generate_maps"), "arcs_generate_maps": ("GenerateMaps", - "generating_maps_started", + "generating_maps_started", # maps "arc_make_cube"), "arc_make_cube": ("MakeCube", - "making_cube_started", + "making_cube_started", # icube "arc_make_cubeimage"), "arc_make_cubeimage": ("CubeImage", - "making_cubeimage_started", + "making_cubeimage_started", # icube_2d None), # FLAT PROCESSING "process_flat": ("ProcessFlat", "flat_processing_started", - "flat_subtract_bias"), - "flat_subtract_bias": ("SubtractBias", - "subtract_bias started", "flat_subtract_overscan"), "flat_subtract_overscan": ("SubtractOverscan", "subtract_overscan_started", "flat_trim_overscan"), "flat_trim_overscan": ("TrimOverscan", "trim_overscan_started", + "flat_subtract_bias"), + "flat_subtract_bias": ("SubtractBias", + "subtract_bias started", + "flat_correct_gain"), + "flat_correct_gain": ("CorrectGain", + "gain_correction_started", "flat_correct_defects"), "flat_correct_defects": ("CorrectDefects", "defect_correction_started", "flat_remove_crs"), "flat_remove_crs": ("RemoveCosmicRays", "remove_crs_started", + "flat_create_unc"), + "flat_create_unc": ("CreateUncertaintyImage", + "create_unc_started", "flat_rectify_image"), "flat_rectify_image": ("RectifyImage", - "rectification_started", + "rectification_started", # int "flat_subtract_dark"), "flat_subtract_dark": ("SubtractDark", "subtract_dark_started", "flat_subtract_scat"), "flat_subtract_scat": ("SubtractScatteredLight", - "scat_subtract_started", + "scat_subtract_started", # intd "flat_make_stack"), "flat_make_stack": ("StackFlats", - "stack_flats_started", + "stack_flats_started", # sflat "flat_make_master"), "flat_make_master": ("MakeMasterFlat", - "master_flat_started", + "master_flat_started", # mflat "flat_correct_illumination"), "flat_correct_illumination": ("CorrectIllumination", - "illumination_correction_started", + "illumination_correction_started", # intf + "flat_make_cube"), + "flat_make_cube": ("MakeCube", + "making_flat_cube_started", # icube None), # OBJECT PROCESSING "process_object": ("ProcessObject", "object_processing_started", - "object_subtract_bias"), - "object_subtract_bias": ("SubtractBias", - "subtract_bias started", + "object_flag_saturation"), + "object_flag_saturation": ("FlagSaturation", + "object_flag_saturation_started", "object_subtract_overscan"), "object_subtract_overscan": ("SubtractOverscan", "subtract_overscan_started", "object_trim_overscan"), "object_trim_overscan": ("TrimOverscan", "trim_overscan_started", + "object_subtract_bias"), + "object_subtract_bias": ("SubtractBias", + "subtract_bias started", "object_correct_gain"), "object_correct_gain": ("CorrectGain", "gain_correction_started", @@ -205,37 +231,40 @@ class Keck_RTI_Pipeline(BasePipeline): "create_unc_started", "object_rectify_image"), "object_rectify_image": ("RectifyImage", - "rectification_started", + "rectification_started", # int "object_subtract_dark"), "object_subtract_dark": ("SubtractDark", "subtract_dark started", "object_subtract_scat"), "object_subtract_scat": ("SubtractScatteredLight", - "scat_subtract_started", + "scat_subtract_started", # intd "object_correct_illum"), "object_correct_illum": ("CorrectIllumination", - "illumination_correction_started", + "illumination_correction_started", # intf + "object_make_master"), + "object_make_master": ("MakeMasterObject", + "master_object_started", # mobj "object_make_sky"), "object_make_sky": ("MakeMasterSky", "making_master_sky_started", "object_subtract_sky"), "object_subtract_sky": ("SubtractSky", - "subtracting_sky_started", + "subtracting_sky_started", # intk "object_make_cube"), "object_make_cube": ("MakeCube", - "making_cube_started", + "making_cube_started", # icube "object_wavelengthcorr"), "object_wavelengthcorr": ("WavelengthCorrections", - "wavelength_correction_started", + "wavelength_correction_started", # icubew "object_correct_dar"), "object_correct_dar": ("CorrectDar", - "correcting_dar_started", + "correcting_dar_started", # icubed "object_make_invsens"), "object_make_invsens": ("MakeInvsens", "make_invsens_started", "object_flux_calibrate"), "object_flux_calibrate": ("FluxCalibrate", - "flux_calibration_started", + "flux_calibration_started", # icubes "alert_rti"), "alert_rti": ("SendHTTP", "http_started", @@ -243,15 +272,15 @@ class Keck_RTI_Pipeline(BasePipeline): # NOD AND SHUFFLE OBJECT PROCESSING "process_nandshuff": ("ProcessObject", "nandshuff_processing_started", - "nandshuff_subtract_bias"), - "nandshuff_subtract_bias": ("SubtractBias", - "subtract_bias started", "nandshuff_subtract_overscan"), "nandshuff_subtract_overscan": ("SubtractOverscan", "subtract_overscan_started", "nandshuff_trim_overscan"), "nandshuff_trim_overscan": ("TrimOverscan", "trim_overscan_started", + "nandshuff_subtract_bias"), + "nandshuff_subtract_bias": ("SubtractBias", + "subtract_bias started", "nandshuff_correct_gain"), "nandshuff_correct_gain": ("CorrectGain", "gain_correction_started", @@ -266,22 +295,28 @@ class Keck_RTI_Pipeline(BasePipeline): "create_unc_started", "nandshuff_rectify_image"), "nandshuff_rectify_image": ("RectifyImage", - "rectification_started", + "rectification_started", # int "nandshuff_subtract_sky"), "nandshuff_subtract_sky": ("NandshuffSubtractSky", - "nandshuff_skysub_started", + "nandshuff_skysub_started", # intk "nandshuff_correct_illum"), "nandshuff_correct_illum": ("CorrectIllumination", - "illumination_correction_started", + "illumination_correction_started", # intf "nandshuff_make_cube"), "nandshuff_make_cube": ("MakeCube", - "making_cube_started", + "making_cube_started", # icube + "nandshuff_wavelengthcorr"), + "nandshuff_wavelengthcorr": ("WavelengthCorrections", + "wavelength_correction_started", # icubew "nandshuff_correct_dar"), "nandshuff_correct_dar": ("CorrectDar", - "correcting_dar_started", + "correcting_dar_started", # icubed "nandshuff_flux_calibrate"), "nandshuff_flux_calibrate": ("FluxCalibrate", - "flux_calibration_started", + "flux_calibration_started", # icubes + "alert_rti"), + "alert_rti": ("SendHTTP", + "http_started", None), "next_file_stop": ("ingest_file", "file_ingested", None) } @@ -296,31 +331,48 @@ def __init__(self, context: ProcessingContext): self.cnt = 0 def add_to_dataframe_only(self, action, context): + self.context.pipeline_logger.info("******* ADD to DATAFRAME ONLY: %s" % + action.args.name) return action.args def action_planner(self, action, context): try: - self.context.pipeline_logger.info("******* FILE TYPE DETERMINED AS %s" % - action.args.imtype) - except: - self.context.pipeline_logger.warn("******* FILE TYPE is NOT determined. No processing is possible.") + self.context.pipeline_logger.info( + "******* FILE TYPE DETERMINED AS %s" % action.args.imtype) + except (AttributeError, TypeError, ValueError): + self.context.pipeline_logger.warn( + "******* FILE TYPE is NOT determined. " + "No processing is possible.") return False # Record the time of ingestion for RTI metrics action.args.ingest_time = datetime.utcnow() groupid = action.args.groupid - self.context.pipeline_logger.info("******* GROUPID is %s " % action.args.groupid) - self.context.pipeline_logger.info("******* STATEID is %s (%s) " % (action.args.ccddata.header["STATENAM"], action.args.ccddata.header["STATEID"])) - + camera = action.args.ccddata.header['CAMERA'].upper() + self.context.pipeline_logger.info("******* GROUPID is %s " % + action.args.groupid) + self.context.pipeline_logger.info( + "******* STATEID is %s (%s) " % + (action.args.ccddata.header["STATENAM"], + action.args.ccddata.header["STATEID"])) + self.context.pipeline_logger.info("******* CAMERA is %s " % camera) if action.args.in_proctab: - self.context.pipeline_logger.warn("Already processed (already in proctab)") + if len(action.args.last_suffix) > 0: + self.context.pipeline_logger.warn( + "Already processed (already in proctab up to %s)" % + action.args.last_suffix) + else: + self.context.pipeline_logger.warn( + "Already processed (already in proctab)") if action.args.in_proctab and not context.config.instrument.clobber: self.context.pipeline_logger.warn("Pushing noop to queue") context.push_event("noop", action.args) elif "BIAS" in action.args.imtype: if action.args.ttime > 0: - self.context.pipeline_logger.warn(f"BIAS frame with exposure time = {action.args.ttime} > 0. Discarding.") + self.context.pipeline_logger.warn( + f"BIAS frame with exposure time = {action.args.ttime} " + f"> 0. Discarding.") return False bias_args = action.args bias_args.groupid = groupid @@ -339,7 +391,15 @@ def action_planner(self, action, context): dark_args.in_directory = "redux" context.push_event("process_dark", dark_args) elif "CONTBARS" in action.args.imtype: - context.push_event("process_contbars", action.args) + contbars_args = action.args + contbars_args.groupid = groupid + contbars_args.want_type = "CONTBARS" + contbars_args.new_type = "MCBARS" + contbars_args.min_files = \ + context.config.instrument.contbars_min_nframes + contbars_args.new_file_name = "master_contbars_%s.fits" % groupid + contbars_args.in_directory = "redux" + context.push_event("process_contbars", contbars_args) elif "FLATLAMP" in action.args.imtype: flat_args = action.args flat_args.groupid = groupid @@ -371,13 +431,24 @@ def action_planner(self, action, context): flat_args.in_directory = "redux" context.push_event("process_flat", flat_args) elif "ARCLAMP" in action.args.imtype: - context.push_event("process_arc", action.args) + arc_args = action.args + arc_args.groupid = groupid + arc_args.want_type = "ARCLAMP" + arc_args.new_type = "MARC" + arc_args.min_files = context.config.instrument.arc_min_nframes + arc_args.new_file_name = "master_arc_%s.fits" % groupid + arc_args.in_directory = "redux" + context.push_event("process_arc", arc_args) elif "OBJECT" in action.args.imtype: if action.args.nasmask and action.args.numopen > 1: context.push_event("process_nandshuff", action.args) else: object_args = action.args - object_args.new_type = "SKY" + # object_args.new_type = "SKY" + object_args.new_type = "MOBJ" + object_args.min_files = \ + context.config.instrument.object_min_nframes + object_args.in_directory = "redux" context.push_event("process_object", object_args) return True diff --git a/kcwidrp/primitives/CorrectGain.py b/kcwidrp/primitives/CorrectGain.py index 6211dec..9e9e769 100644 --- a/kcwidrp/primitives/CorrectGain.py +++ b/kcwidrp/primitives/CorrectGain.py @@ -31,7 +31,6 @@ def _perform(self): # Header keyword to update key = 'GAINCOR' keycom = 'Gain corrected?' - # print(self.action.args.ccddata.header) number_of_amplifiers = self.action.args.namps bsec, dsec, tsec, direc, amps, aoff = self.action.args.map_ccd namps = len(amps) diff --git a/kcwidrp/primitives/ExtractArcs.py b/kcwidrp/primitives/ExtractArcs.py index a3c8ea2..545c1a9 100644 --- a/kcwidrp/primitives/ExtractArcs.py +++ b/kcwidrp/primitives/ExtractArcs.py @@ -87,7 +87,8 @@ def _perform(self): self.action.args.contbar_image_number = self.context.trace['CBARSNO'] self.action.args.contbar_image = self.context.trace['CBARSFL'] self.action.args.arc_number = self.action.args.ccddata.header['FRAMENO'] - self.action.args.arc_image = self.action.args.ccddata.header['OFNAME'] + self.action.args.arc_image = self.action.args.name + # self.action.args.arc_image = self.action.args.ccddata.header['OFNAME'] self.action.args.source_control_points = trace['src'] self.action.args.destination_control_points = trace['dst'] diff --git a/kcwidrp/primitives/FindBars.py b/kcwidrp/primitives/FindBars.py index ce20da1..7587058 100644 --- a/kcwidrp/primitives/FindBars.py +++ b/kcwidrp/primitives/FindBars.py @@ -201,7 +201,8 @@ def _perform(self): self.action.args.contbar_image_number = \ self.action.args.ccddata.header['FRAMENO'] self.action.args.contbar_image = \ - self.action.args.ccddata.header['OFNAME'] + self.action.args.name + # self.action.args.ccddata.header['OFNAME'] log_string = FindBars.__module__ self.action.args.ccddata.header['HISTORY'] = log_string diff --git a/kcwidrp/primitives/FluxCalibrate.py b/kcwidrp/primitives/FluxCalibrate.py index ca00c76..de8e09e 100644 --- a/kcwidrp/primitives/FluxCalibrate.py +++ b/kcwidrp/primitives/FluxCalibrate.py @@ -95,7 +95,7 @@ def _perform(self): sz[0] != mcsz[0]: self.logger.warning("wavelength scales not identical, " "resampling standard") - print(w0, mcw0, dw, mcdw, wav[-1], mcwav[-1], sz[0], mcsz[0]) + self.logger.info(w0, mcw0, dw, mcdw, wav[-1], mcwav[-1], sz[0], mcsz[0]) mcint = interp1d(mcwav, mcal, kind='cubic', fill_value='extrapolate') mscal = mcint(wav) * 1.e16 / expt diff --git a/kcwidrp/primitives/MakeInvsens.py b/kcwidrp/primitives/MakeInvsens.py index 279eb7f..aa7fcf2 100644 --- a/kcwidrp/primitives/MakeInvsens.py +++ b/kcwidrp/primitives/MakeInvsens.py @@ -333,7 +333,7 @@ def _perform(self): wlm0 = wgoo0 wlm1 = wgoo1 # interactively set wavelength limits - if self.config.instrument.plot_level >= 1: + if self.config.instrument.plot_level > 1: print("CHECKING WAVELENGTH LIMITS") print("Current WL limits: %.1f - %.1f Angstroms " "(blue vertical lines)" % (wlm0, wlm1)) @@ -387,7 +387,8 @@ def _perform(self): local_lmfile = os.path.basename(stdfile).split('.fits')[0] + '.lmsk' if not os.path.exists(local_lmfile): # then in stds directory - lmfile = stdfile.split('.fits')[0] + '.lmsk' + # lmfile = stdfile.split('.fits')[0] + '.lmsk' + lmfile = strip_fname(stdfile) + '.lmsk' if not os.path.exists(lmfile): lmfile = None else: @@ -428,7 +429,7 @@ def _perform(self): else: print("bad line: %s" % lmws) # Now interactively identify lines if requested - if self.config.instrument.plot_level >= 1: + if self.config.instrument.plot_level >= 2: yran = [np.min(obsspec[wl_good]), np.max(obsspec[wl_good])] # source = ColumnDataSource(data=dict(x=w, y=obsspec)) print("MASKING SHARP FEATURES: ABSORPTION LINES/COSMIC RAYS") @@ -580,7 +581,7 @@ def _perform(self): line_color='black', line_dash='dashdot') set_plot_lims(peff, xlim=[wall0, wall1], ylim=yran) bokeh_plot(peff, self.context.bokeh_session) - if self.config.instrument.plot_level >= 1: + if self.config.instrument.plot_level >= 2: input("Next? : ") else: time.sleep(2. * self.config.instrument.plot_pause) @@ -601,7 +602,7 @@ def _perform(self): pivs.line([wlm1, wlm1], yran, line_color='green') set_plot_lims(pivs, xlim=[wall0, wall1], ylim=yran) bokeh_plot(pivs, self.context.bokeh_session) - if self.config.instrument.plot_level >= 1: + if self.config.instrument.plot_level >= 2: input("Next? : ") else: time.sleep(2. * self.config.instrument.plot_pause) @@ -620,7 +621,7 @@ def _perform(self): pcal.line([wlm1, wlm1], yran, line_color='green') set_plot_lims(pcal, xlim=[wall0, wall1], ylim=yran) bokeh_plot(pcal, self.context.bokeh_session) - if self.config.instrument.plot_level >= 1: + if self.config.instrument.plot_level >= 2: input("Next? : ") else: time.sleep(2. * self.config.instrument.plot_pause) @@ -650,7 +651,7 @@ def _perform(self): line_color='black', line_dash='dashed') set_plot_lims(prsd, xlim=[wall0, wall1], ylim=yran) bokeh_plot(prsd, self.context.bokeh_session) - if self.config.instrument.plot_level >= 1: + if self.config.instrument.plot_level >= 2: qstr = input("Current fit order = %d, " "New fit order? , - done: " % ford) if len(qstr) <= 0: diff --git a/kcwidrp/primitives/MakeMasterArc.py b/kcwidrp/primitives/MakeMasterArc.py index 9acacfe..ea2d9ed 100644 --- a/kcwidrp/primitives/MakeMasterArc.py +++ b/kcwidrp/primitives/MakeMasterArc.py @@ -1,6 +1,6 @@ from keckdrpframework.primitives.base_img import BaseImg from kcwidrp.primitives.kcwi_file_primitives import kcwi_fits_reader, \ - kcwi_fits_writer, strip_fname # , get_master_name + kcwi_fits_writer, strip_fname , get_unique_STATEID_master_name import os import ccdproc @@ -58,14 +58,15 @@ def _perform(self): combine_list = list(self.combine_list['filename']) # get master arc output name - maname = strip_fname(combine_list[0]) + '_' + suffix + '.fits' + # maname = strip_fname(combine_list[0]) + '_' + suffix + '.fits' if self.action.args.min_files > 1: stack = [] stackf = [] for arc in combine_list: # get arc intensity (int) image file name in redux directory - stackf.append(arc.split('.fits')[0] + '_int.fits') - arcfn = os.path.join(args.in_directory, stackf[-1]) + stackf.append(strip_fname(arc) + '_int.fits') + arcfn = os.path.join(self.config.instrument.cwd, + self.config.instrument.output_directory, stackf[-1]) # using [0] gets just the image data stack.append(kcwi_fits_reader(arcfn)[0]) @@ -86,15 +87,18 @@ def _perform(self): "stack input file") stacked.header['HISTORY'] = log_string self.action.args.ccddata = stacked + # maname = get_unique_STATEID_master_name(stacked, suffix) + maname = strip_fname(combine_list[0]) + '_' + suffix + '.fits' kcwi_fits_writer(stacked, output_file=maname, output_dir=self.config.instrument.output_directory) self.context.proctab.update_proctab(frame=stacked, suffix=suffix, newtype=args.new_type, - filename=stacked.header[ - 'OFNAME']) - self.action.args.name = stacked.header['OFNAME'] + filename=self.action.args.name) ### HERE + # self.action.args.name = maname + # self.action.args.name = stacked.header['OFNAME'] else: + maname = strip_fname(combine_list[0]) + '_' + suffix + '.fits' self.action.args.ccddata.header['IMTYPE'] = args.new_type self.action.args.ccddata.header['HISTORY'] = log_string kcwi_fits_writer(self.action.args.ccddata, output_file=maname, diff --git a/kcwidrp/primitives/MakeMasterBias.py b/kcwidrp/primitives/MakeMasterBias.py index 801839e..d26498e 100644 --- a/kcwidrp/primitives/MakeMasterBias.py +++ b/kcwidrp/primitives/MakeMasterBias.py @@ -1,7 +1,7 @@ from keckdrpframework.models.arguments import Arguments from keckdrpframework.primitives.base_img import BaseImg from kcwidrp.primitives.kcwi_file_primitives import kcwi_fits_reader, \ - kcwi_fits_writer, parse_imsec, strip_fname + kcwi_fits_writer, parse_imsec, strip_fname, get_unique_CCD_master_name from kcwidrp.core.bokeh_plotting import bokeh_plot from kcwidrp.core.kcwi_plotting import save_plot @@ -64,7 +64,8 @@ def _perform(self): combine_list = list(self.combine_list['filename']) # get master bias output name # mbname = combine_list[-1].split('.fits')[0] + '_' + suffix + '.fits' - mbname = strip_fname(combine_list[0]) + '_' + suffix + '.fits' + # mbname = strip_fname(combine_list[0]) + '_' + suffix + '.fits' + # mbname = master_bias_name(self.action.args.ccddata) bsec, dsec, tsec, direc, amps, aoff = self.action.args.map_ccd @@ -72,7 +73,8 @@ def _perform(self): stack = [] stackf = [] for bias in combine_list: - inbias = bias.split('.fits')[0] + '_intb.fits' + # inbias = bias.split('.fits')[0] + '_intb.fits' + inbias = strip_fname(bias) + '_intb.fits' stackf.append(inbias) # using [0] drops the table and leaves just the image stack.append(kcwi_fits_reader( @@ -158,11 +160,14 @@ def _perform(self): stacked.header['HISTORY'] = log_string self.logger.info(log_string) + mbname = get_unique_CCD_master_name(stacked) + + kcwi_fits_writer(stacked, output_file=mbname, output_dir=self.config.instrument.output_directory) self.context.proctab.update_proctab(frame=stacked, suffix=suffix, newtype=self.action.args.new_type, - filename=stacked.header['OFNAME']) + filename=self.action.args.name) self.context.proctab.write_proctab(tfil=self.config.instrument.procfile) return Arguments(name=mbname) # END: class ProcessBias() diff --git a/kcwidrp/primitives/MakeMasterContbars.py b/kcwidrp/primitives/MakeMasterContbars.py index afc5480..3067d94 100644 --- a/kcwidrp/primitives/MakeMasterContbars.py +++ b/kcwidrp/primitives/MakeMasterContbars.py @@ -1,6 +1,6 @@ from keckdrpframework.primitives.base_img import BaseImg from kcwidrp.primitives.kcwi_file_primitives import kcwi_fits_reader, \ - kcwi_fits_writer, strip_fname # , get_master_name + kcwi_fits_writer, strip_fname , get_unique_STATEID_master_name import os import ccdproc @@ -57,15 +57,14 @@ def _perform(self): log_string = MakeMasterContbars.__module__ combine_list = list(self.combine_list['filename']) - # get master arc output name - maname = strip_fname(combine_list[0]) + '_' + suffix + '.fits' + if self.action.args.min_files > 1: stack = [] stackf = [] for cbs in combine_list: # get arc intensity (int) image file name in redux directory - stackf.append(cbs.split('.fits')[0] + '_int.fits') - cbsfn = os.path.join(args.in_directory, stackf[-1]) + stackf.append(strip_fname(cbs) + '_int.fits') + cbsfn = os.path.join(self.config.instrument.cwd, self.config.instrument.output_directory, stackf[-1]) # using [0] gets just the image data stack.append(kcwi_fits_reader(cbsfn)[0]) @@ -87,17 +86,20 @@ def _perform(self): stacked.header['HISTORY'] = log_string self.action.args.ccddata = stacked - kcwi_fits_writer(stacked, output_file=maname, + # get master arc output name + mcbars_name = strip_fname(combine_list[0]) + '_' + suffix + '.fits' + + kcwi_fits_writer(stacked, output_file=mcbars_name, output_dir=self.config.instrument.output_directory) self.context.proctab.update_proctab(frame=stacked, suffix=suffix, newtype=args.new_type, - filename=stacked.header[ - 'OFNAME']) - self.action.args.name = stacked.header['OFNAME'] + filename=self.action.args.name) ### HERE + # self.action.args.name = stacked.header['OFNAME'] else: + mcbars_name = strip_fname(combine_list[0]) + '_' + suffix + '.fits' self.action.args.ccddata.header['IMTYPE'] = args.new_type self.action.args.ccddata.header['HISTORY'] = log_string - kcwi_fits_writer(self.action.args.ccddata, output_file=maname, + kcwi_fits_writer(self.action.args.ccddata, output_file=mcbars_name, output_dir=self.config.instrument.output_directory) self.context.proctab.update_proctab(frame=self.action.args.ccddata, suffix=suffix, diff --git a/kcwidrp/primitives/MakeMasterDark.py b/kcwidrp/primitives/MakeMasterDark.py index fcacb64..a7be57a 100644 --- a/kcwidrp/primitives/MakeMasterDark.py +++ b/kcwidrp/primitives/MakeMasterDark.py @@ -1,6 +1,6 @@ from keckdrpframework.primitives.base_img import BaseImg from kcwidrp.primitives.kcwi_file_primitives import kcwi_fits_reader, \ - kcwi_fits_writer, strip_fname # , get_master_name + kcwi_fits_writer, strip_fname, get_unique_STATEID_master_name # , get_master_name import os import ccdproc @@ -54,13 +54,15 @@ def _perform(self): combine_list = list(self.combine_list['filename']) # get master dark output name - mdname = strip_fname(combine_list[0]) + '_' + suffix + '.fits' + # mdname = strip_fname(combine_list[0]) + '_' + suffix + '.fits' + mdname = get_unique_STATEID_master_name(self.action.args.ccddata, suffix="mdark") stack = [] stackf = [] for dark in combine_list: # get dark intensity (int) image file name in redux directory - stackf.append(dark.split('.fits')[0] + '_int.fits') - darkfn = os.path.join(args.in_directory, stackf[-1]) + stackf.append(strip_fname(dark) + '_int.fits') + darkfn = os.path.join(self.config.instrument.cwd, + self.config.instrument.output_directory, stackf[-1]) # using [0] gets just the image data stack.append(kcwi_fits_reader(darkfn)[0]) @@ -87,7 +89,7 @@ def _perform(self): output_dir=self.config.instrument.output_directory) self.context.proctab.update_proctab(frame=stacked, suffix=suffix, newtype=args.new_type, - filename=stacked.header['OFNAME']) + filename=self.action.args.name) ### HERE self.context.proctab.write_proctab(tfil=self.config.instrument.procfile) return self.action.args # END: class MakeMasterDark() diff --git a/kcwidrp/primitives/MakeMasterFlat.py b/kcwidrp/primitives/MakeMasterFlat.py index f2bda41..60464a7 100644 --- a/kcwidrp/primitives/MakeMasterFlat.py +++ b/kcwidrp/primitives/MakeMasterFlat.py @@ -99,7 +99,7 @@ def _perform(self): self.logger.error("Geometry not solved!") return self.action.args - mroot = strip_fname(tab['filename'][-1]) + mroot = strip_fname(tab['filename'][-1]) # ?????????????????? # Wavelength map image wmf = mroot + '_wavemap.fits' @@ -435,7 +435,7 @@ def _perform(self): trm = int(len(deriv)/5) deriv = deriv[trm:-trm] xvals = fpoints[trm:-trm] - peaks, _ = find_peaks(deriv, height=20) + peaks, properties = find_peaks(deriv, height=20) self.logger.info("%d Peak(s) found" % len(peaks)) p = figure(title=plab + @@ -466,11 +466,16 @@ def _perform(self): bokeh_plot(p, self.context.bokeh_session) if len(peaks) != 1: self.logger.warning("Single peak not found!") - print("Please indicate the integer pixel value of the peak") - spk = input("Peak? : ") - while not spk.isnumeric(): + if self.config.instrument.plot_level >= 2: + print("Please indicate the integer pixel value of the peak") spk = input("Peak? : ") - ipk = int(spk) + while not spk.isnumeric(): + spk = input("Peak? : ") + ipk = int(spk) + else: + # Pick the highest peak, using the peaks properties + self.logger.warning("Picking the highest peak. This may not be the correct peak!") + ipk = properties['peak_heights'].argmax() else: ipk = peaks[0] apk = xvals[ipk] @@ -915,7 +920,7 @@ def _perform(self): ratio.flat[qq] = 1.0 # get master flat output name - mfname = stack_list[0].split('.fits')[0] + '_' + suffix + '.fits' + mfname = strip_fname(stack_list[0]) + '_' + suffix + '.fits' log_string = MakeMasterFlat.__module__ stacked.header['IMTYPE'] = self.action.args.new_type @@ -933,9 +938,10 @@ def _perform(self): output_dir=self.config.instrument.output_directory) self.context.proctab.update_proctab(frame=stacked, suffix=suffix, newtype=self.action.args.new_type, - filename=stacked.header['OFNAME']) + filename=stack_list[0]) ### HERE self.context.proctab.write_proctab(tfil=self.config.instrument.procfile) - self.action.args.name = stacked.header['OFNAME'] + # self.action.args.name = stacked.header['OFNAME'] + # self.action.args.name = mfname self.logger.info(log_string) return self.action.args diff --git a/kcwidrp/primitives/MakeMasterObject.py b/kcwidrp/primitives/MakeMasterObject.py index 9bbfd2c..55a3197 100644 --- a/kcwidrp/primitives/MakeMasterObject.py +++ b/kcwidrp/primitives/MakeMasterObject.py @@ -1,6 +1,6 @@ from keckdrpframework.primitives.base_img import BaseImg from kcwidrp.primitives.kcwi_file_primitives import kcwi_fits_reader, \ - kcwi_fits_writer, strip_fname # , get_master_name + kcwi_fits_writer, strip_fname, get_master_name import os import ccdproc @@ -72,14 +72,15 @@ def _perform(self): self.logger.info("Combining Master Object with method %s" % method) # get master arc output name - maname = strip_fname(combine_list[0]) + '_' + suffix + '.fits' + # maname = strip_fname(combine_list[0]) + '_' + suffix + '.fits' if self.action.args.min_files > 1: stack = [] stacko = [] for obj in combine_list: # get object intensity (int) image file name in redux directory - stackf = obj.split('.fits')[0] + '_intf.fits' - objfn = os.path.join(args.in_directory, stackf) + stackf = strip_fname(obj) + '_intf.fits' + objfn = os.path.join(self.config.instrument.cwd, + self.config.instrument.output_directory, stackf) stacko.append(stackf) # using [0] gets just the image data stack.append(kcwi_fits_reader(objfn)[0]) @@ -100,18 +101,19 @@ def _perform(self): "stack input file") stacked.header['HISTORY'] = log_string self.action.args.ccddata = stacked - - kcwi_fits_writer(stacked, output_file=maname, + mobj_name = get_master_name(combine_list, "mobj") + kcwi_fits_writer(stacked, output_file=mobj_name, output_dir=self.config.instrument.output_directory) self.context.proctab.update_proctab(frame=stacked, suffix=suffix, newtype=args.new_type, - filename=stacked.header[ - 'OFNAME']) - self.action.args.name = stacked.header['OFNAME'] + filename=self.action.args.name) ### HERE + # self.action.args.name = mobj_name + # self.action.args.name = stacked.header['OFNAME'] else: + mobj_name = get_master_name(combine_list, "mobj") self.action.args.ccddata.header['IMTYPE'] = args.new_type self.action.args.ccddata.header['HISTORY'] = log_string - kcwi_fits_writer(self.action.args.ccddata, output_file=maname, + kcwi_fits_writer(self.action.args.ccddata, output_file=mobj_name, output_dir=self.config.instrument.output_directory) self.context.proctab.update_proctab(frame=self.action.args.ccddata, suffix=suffix, diff --git a/kcwidrp/primitives/MakeMasterSky.py b/kcwidrp/primitives/MakeMasterSky.py index 0112da1..97366ae 100644 --- a/kcwidrp/primitives/MakeMasterSky.py +++ b/kcwidrp/primitives/MakeMasterSky.py @@ -176,7 +176,7 @@ def _pre_condition(self): # Do we have a sky alternate? if skyfile: # Generate sky file name - msname = skyfile.split('.fits')[0] + '_' + suffix + '.fits' + msname = strip_fname(skyfile) + '_' + suffix + '.fits' mskyf = os.path.join(rdir, msname) # Does it exist? if os.path.exists(mskyf): diff --git a/kcwidrp/primitives/SendHTTP.py b/kcwidrp/primitives/SendHTTP.py index 28dc857..05b00f9 100644 --- a/kcwidrp/primitives/SendHTTP.py +++ b/kcwidrp/primitives/SendHTTP.py @@ -28,8 +28,8 @@ def _perform(self): self.logger.error(f"Encountered a file with no KOA ID: {self.action.args.name}") return self.action.args - data_directory = os.path.join(self.config.rti.cwd, - self.config.rti.output_directory) + data_directory = os.path.join(self.config.instrument.cwd, + self.config.instrument.output_directory) self.logger.info(f"Alerting RTI that {strip_fname(self.action.args.name)} is ready for ingestion") diff --git a/kcwidrp/primitives/SolveGeom.py b/kcwidrp/primitives/SolveGeom.py index b7b5fa3..f6167aa 100644 --- a/kcwidrp/primitives/SolveGeom.py +++ b/kcwidrp/primitives/SolveGeom.py @@ -204,10 +204,11 @@ def _perform(self): dichroic_fraction = 1. # Package geometry data - ofname = self.action.args.ccddata.header['OFNAME'] + # name = self.action.args.ccddata.header['OFNAME'] + name = self.action.args.name self.action.args.geometry_file = os.path.join( self.config.instrument.output_directory, - strip_fname(ofname) + '_geom.pkl') + strip_fname(name) + '_geom.pkl') if os.path.exists(self.action.args.geometry_file): self.logger.error("Geometry file already exists: %s" % self.action.args.geometry_file) diff --git a/kcwidrp/primitives/StackFlats.py b/kcwidrp/primitives/StackFlats.py index cedfe0a..51b737e 100644 --- a/kcwidrp/primitives/StackFlats.py +++ b/kcwidrp/primitives/StackFlats.py @@ -1,6 +1,6 @@ from keckdrpframework.primitives.base_img import BaseImg from kcwidrp.primitives.kcwi_file_primitives import kcwi_fits_reader, \ - kcwi_fits_writer, strip_fname + kcwi_fits_writer, strip_fname, get_unique_STATEID_master_name import os import ccdproc @@ -81,8 +81,9 @@ def _perform(self): self.logger.info("Stacking flats using method %s" % method) combine_list = list(self.combine_list['filename']) - # get flat stack output name + # get flat stack output name stname = strip_fname(combine_list[0]) + '_' + suffix + '.fits' + # stname = get_unique_STATEID_master_name(self.action.args.ccddata, suffix) stack = [] stackf = [] fmeds = [] @@ -91,7 +92,8 @@ def _perform(self): stackf.append(strip_fname(flat) + '_intd.fits') flatfn = os.path.join(self.config.instrument.cwd, self.config.instrument.output_directory, - stackf[-1]) + stackf[0]) + # stackf[-1]) # using [0] gets just the image data f = kcwi_fits_reader(flatfn)[0] # Set mask to None to prevent ccdproc.combine from masking @@ -139,7 +141,7 @@ def _perform(self): self.context.proctab.update_proctab(frame=stacked, suffix=suffix, newtype=self.action.args.stack_type, - filename=stacked.header['OFNAME']) + filename=combine_list[0]) ### HERE self.context.proctab.write_proctab(tfil=self.config.instrument.procfile) self.logger.info(log_string) diff --git a/kcwidrp/primitives/SubtractBias.py b/kcwidrp/primitives/SubtractBias.py index dd474c9..d117126 100644 --- a/kcwidrp/primitives/SubtractBias.py +++ b/kcwidrp/primitives/SubtractBias.py @@ -1,6 +1,6 @@ from keckdrpframework.primitives.base_primitive import BasePrimitive from kcwidrp.primitives.kcwi_file_primitives import kcwi_fits_reader, \ - get_master_name + get_master_name, get_unique_CCD_master_name import os @@ -33,7 +33,9 @@ def _perform(self): self.logger.info("%d master bias frames found" % len(tab)) if len(tab) > 0: - mbname = get_master_name(tab, target_type) + # mbname = get_master_name(tab, target_type) + mbname = get_unique_CCD_master_name(self.action.args.ccddata) + # mbname = master_bias_name(self.action.args.ccddata) self.logger.info("Reading image: %s" % mbname) mbias = kcwi_fits_reader( diff --git a/kcwidrp/primitives/SubtractDark.py b/kcwidrp/primitives/SubtractDark.py index 7115e36..26dbe7b 100644 --- a/kcwidrp/primitives/SubtractDark.py +++ b/kcwidrp/primitives/SubtractDark.py @@ -32,7 +32,7 @@ def _perform(self): if len(tab) > 0: mdname = get_master_name(tab, target_type) - print("*************** READING IMAGE: %s" % mdname) + self.logger.info("*************** READING IMAGE: %s" % mdname) mdark = kcwi_fits_reader( os.path.join(self.config.instrument.cwd, 'redux', mdname))[0] diff --git a/kcwidrp/primitives/TraceBars.py b/kcwidrp/primitives/TraceBars.py index a23ebb1..36d4910 100644 --- a/kcwidrp/primitives/TraceBars.py +++ b/kcwidrp/primitives/TraceBars.py @@ -159,14 +159,14 @@ def _perform(self): # in this line we pass the trace information to an argument # instead of writing it to a table self.context.trace = trace - ofname = strip_fname(self.action.args.contbar_image) + \ + name = strip_fname(self.action.args.contbar_image) + \ "_trace.fits" write_table(table=[src, dst, barid, slid], names=('src', 'dst', 'barid', 'slid'), output_dir=os.path.join( self.config.instrument.cwd, self.config.instrument.output_directory), - output_name=ofname, + output_name=name, clobber=self.config.instrument.clobber, comment=['Source and destination fiducial points', 'Derived from KCWI continuum bars images', diff --git a/kcwidrp/primitives/TrimOverscan.py b/kcwidrp/primitives/TrimOverscan.py index d361bf9..9e3fa78 100644 --- a/kcwidrp/primitives/TrimOverscan.py +++ b/kcwidrp/primitives/TrimOverscan.py @@ -108,9 +108,12 @@ def _perform(self): output_file=self.action.args.name, output_dir=self.config.instrument.output_directory, suffix="intb") + # self.context.proctab.update_proctab( + # frame=self.action.args.ccddata, suffix="intb", newtype='BIAS', + # filename=self.action.args.ccddata.header['OFNAME']) self.context.proctab.update_proctab( frame=self.action.args.ccddata, suffix="intb", newtype='BIAS', - filename=self.action.args.ccddata.header['OFNAME']) + filename=self.action.args.name) self.context.proctab.write_proctab( tfil=self.config.instrument.procfile) return self.action.args diff --git a/kcwidrp/primitives/WavelengthCorrections.py b/kcwidrp/primitives/WavelengthCorrections.py index aea221c..eecde60 100644 --- a/kcwidrp/primitives/WavelengthCorrections.py +++ b/kcwidrp/primitives/WavelengthCorrections.py @@ -2,6 +2,7 @@ import numpy as np from scipy.interpolate import interp1d +from scipy.interpolate import PchipInterpolator from astropy import units as u from astropy.coordinates import SkyCoord, EarthLocation @@ -10,7 +11,6 @@ kcwi_fits_reader, \ strip_fname - class WavelengthCorrections(BasePrimitive): """ Perform wavelength corrections @@ -59,17 +59,17 @@ def _perform(self): obj = self.locate_object_file(suffix) if "none" in correction_mode: + perform_helio = False self.logger.info("Skipping radial velocity correction") obj.header['VCORR'] = (0.0, 'km/s') obj.header['VCORRTYP'] = (correction_mode, 'Vcorr type') - else: - self.logger.info(f"Performing {correction_mode} correction") - obj = self.heliocentric(obj, correction_mode) + perform_helio = True - if self.config.instrument.air_to_vacuum: - self.logger.info("Performing Air to Vacuum Conversion") - obj = self.air2vac(obj) + perform_a2v = self.config.instrument.air_to_vacuum + + # does helio corr, then a2v, then resamples + obj = self.helio_a2v_conversion(obj, perform_helio, perform_a2v, correction_mode) log_string = WavelengthCorrections.__module__ obj.header['HISTORY'] = log_string @@ -89,158 +89,349 @@ def _perform(self): return self.action.args - def air2vac(self, obj, mask=False): + def helio_a2v_conversion(self, obj, perform_helio, perform_a2v, correction_mode, mask=False, vcorr=None, resample=True): """ - Convert wavelengths in a cube from standard air to vacuum. + Apply heliocentric and/or air to vacuum conversion to wavelengths in a cube. + + Adapted from https://github.com/dbosul/cwitools.git Args: obj (astropy HDU / HDUList): Input HDU/HDUList with 3D data. + perform_helio (bool): Set to apply heliocentric conversion. + perform_a2v (bool): Set to apply air to vac conversion. + correction_mode (str): "none", "barycentric", or "heliocentric" + vcorr (float): Use a different correction velocity. mask (bool): Set if the cube is a mask cube. - :returns: + Returns: HDU / HDUList: Trimmed FITS object with updated header. - Return type matches type of obj argument. - + float: (if vcorr is True) Correction velocity in km/s. + Return type matches type of fits_in argument. """ cube = np.nan_to_num(obj.data, nan=0, posinf=0, neginf=0) - if obj.header['CTYPE3'] == 'WAVE': - self.logger.warn("FITS already in vacuum wavelength.") - return + wav_old = self.get_wav_axis(obj.header) + + wav_air = wav_old * u.AA + + if perform_helio: + + self.logger.info(f"Performing {correction_mode} correction") + + v_old = 0. + if 'VCORR' in obj.header: + v_old = obj.header['VCORR'] + self.logger.info("Rolling back the existing correction with:") + self.logger.info("Vcorr = %.2f km/s." % v_old) + + # get correction velocity + vcorr = self.heliocentric(obj, correction_mode, vcorr) + v_tot = vcorr - v_old + if not resample: + obj.header['CRVAL3'] = str(obj.header['CRVAL3'] * (1 + v_tot / 2.99792458e5)) + obj.header['CD3_3'] = str(obj.header['CD3_3'] * (1 + v_tot / 2.99792458e5)) + + # update headers + obj.header['VCORR'] = (vcorr, 'km/s') + obj.header['VCORRTYP'] = (correction_mode, 'Vcorr type') + + # calculate new wavelengths with helio corrections + wav_new = wav_old * (1 + v_tot / 2.99792458e5) + cwave_hel = wav_old[int(wav_old.shape[0] / 2)] * (1 + v_tot / 2.99792458e5) + self.logger.info("Vcorr for CWAVE (%.3f) gives %.3f" % + (wav_old[int(wav_old.shape[0] / 2)], cwave_hel)) + + # if also doing air to vac, propogate the helio correction and convert units + if perform_a2v: + wav_air = wav_new * u.AA + + if perform_a2v: - wave_air = self.get_wav_axis(obj.header) * u.AA - wave_vac = self.a2v_conversion(wave_air) - cwave_air = wave_air[int(wave_air.shape[0] / 2)] - cwave_vac = wave_vac[int(wave_vac.shape[0] / 2)] - self.logger.info("Air to Vacuum for (%.3f) gives %.3f" % - (cwave_air.value, cwave_vac.value)) + self.logger.info("Performing Air to Vacuum Conversion") + + # get vacuum wavelengths + wav_new = self.air_to_vac(wav_air) + cwave_air = wav_air[int(wav_air.shape[0] / 2)] + cwave_vac = wav_new[int(wav_new.shape[0] / 2)] + + # update headers + self.logger.info("New Air to Vacuum for (%.3f) gives %.3f" % (cwave_air.value, cwave_vac.value)) + obj.header['CTYPE3'] = ('WAVE', 'Vacuum Wavelengths') # resample to uniform grid - cube_new = np.zeros_like(cube) - for i in range(cube.shape[2]): - for j in range(cube.shape[1]): - - spec0 = cube[:, j, i] - if not mask: - f_cubic = interp1d( - wave_vac, - spec0, - kind='cubic', - fill_value='extrapolate' - ) - spec_new = f_cubic(wave_air) - else: - f_pre = interp1d( - wave_vac, - spec0, - kind='previous', - bounds_error=False, - fill_value=128 - ) - spec_pre = f_pre(wave_air) - - f_nex = interp1d( - wave_vac, - spec0, - kind='next', - bounds_error=False, - fill_value=128 - ) - spec_nex = f_nex(wave_air) - - spec_new = np.zeros_like(spec0) - for k in range(spec0.shape[0]): - spec_new[k] = max(spec_pre[k], spec_nex[k]) - - cube_new[:, j, i] = spec_new - - obj.header['CTYPE3'] = ('WAVE', 'Vacuum Wavelengths') - obj.data = cube_new + if resample: + self.logger.info("Resampling to uniform grid") + cube_new = np.zeros_like(cube) + for i in range(cube.shape[2]): + for j in range(cube.shape[1]): + + spec0 = cube[:, j, i] + if not mask: + + f_cubic = interp1d(wav_new, spec0, kind='cubic', + fill_value='extrapolate') + + spec_new = f_cubic(wav_old) + + f_cubic = PchipInterpolator(wav_new, spec0) + spec_new = f_cubic(wav_old) + else: + f_pre = interp1d(wav_new, spec0, kind='previous', + bounds_error=False, fill_value=128) + spec_pre = f_pre(wav_old) + + f_nex = interp1d(wav_new, spec0, kind='next', + bounds_error=False, fill_value=128) + spec_nex = f_nex(wav_old) + + spec_new = np.zeros_like(spec0) + for k in range(spec0.shape[0]): + spec_new[k] = max(spec_pre[k], spec_nex[k]) + cube_new[:, j, i] = spec_new + + obj.data = cube_new return obj - def a2v_conversion(self, wave): + def air_to_vac(self, wave): """ - Convert air-based wavelengths to vacuum + Convert air-based wavelengths to vacuum wavelengths - Adapted from wave.py in: https://github.com/pypeit/PypeIt/ - Formula from https://ui.adsabs.harvard.edu/abs/1996ApOpt..35.1566C/ + Adapted from wave.py in: https://github.com/pypeit/PypeIt/. + Formulas from https://ui.adsabs.harvard.edu/abs/1996ApOpt..35.1566C/. Args: - wave (ndarray): Wavelengths + wave (ndarray): Wavelengths. Returns: - Wavelength array corrected to vacuum wavelengths + ndarray: Wavelength array corrected to vacuum wavelengths. """ # Convert to AA wave = wave.to(u.AA) wavelength = wave.value - # Standard conversion format - sigma_sq = (1.e4/wavelength)**2. # wavenumber squared - factor = 1 + (5.792105e-2/(238.0185-sigma_sq)) + \ - (1.67918e-3/(57.362-sigma_sq)) - rind = factor[int(factor.shape[0] / 2)] + # wave number (inverse wavelength in micrometers) + sigma = (1.e4/wavelength) + sigma_sq = sigma**2 + + # humidity: convert from % to decimal + h = self.action.args.dome_hum / 100 + # temperature: convert from [C] to [K] + T = self.action.args.dome_temp + 273.15 + # pressure: convert from mbar to Pa + p = self.action.args.pres * 100 + # ppm of CO2 + x_c = 450 + + # Constants for equation (1) from Appendix A + k0 = 238.0185 + k1 = 5792105 + k2 = 57.362 + k3 = 167917 + + # Index of refraction for standard air. From equation (1) + n_as = ((k1 / (k0 - sigma_sq) + k3 / (k2 - sigma_sq)) / 10**8) + 1 + + # Index of refraction for standard air with x_c ppm CO2. From equation (2) + n_axs = ((n_as - 1) * (1 + 0.534 * 10**-6 * (x_c - 450))) + 1 + + # Constants for equation (3) from Appendix A + cf = 1.022 # correction factor + w0 = 295.235 + w1 = 2.6422 + w2 = -0.032380 + w3 = 0.004028 + + # Index of refraction for standard water vapor. From equation (3) + n_ws = ((cf * (w0 + (w1 * sigma**2) + (w2 * sigma**4) + (w3 * sigma**6))) / 10e8) + 1 + + def get_molar_frac_water(T, p, h): + """ + Gets the molar fraction of water vapor in moist air [kg/m^3]. + Part of equation (4) in reference paper. + + Args: + T (float): Temperature in [K] Kelvin. + p (float): Pressure in [Pa] Pascal. + h (float): Percent relative humidity. + + Returns: + float: Molar fraction of water vapor in moist air. + """ + + # temperature [C] + t = T - 273.15 + + # constants for f from Appendix A + alpha = 1.00062 + beta = 3.14 * 10**-8 + gamma = 5.6 * 10**-7 + + # enchancement factor. part of Equation (4) + f = alpha + beta * p + gamma * t**2 + + # constants for svp from Appendix A + A = 1.2378847 * 10**-5 + B = -1.9121316 * 10**-2 + C = 33.93711047 + D = -6.3431645 * 10**3 + + # saturation vapor pressure (svp) of water vapor in air at temperature, T [K]. Part of Equation (4) + svp = np.exp((A * T**2 + B * T + C + D/T)) + + # molar fraction of water vapor in moist air. Part of Equation (4) + x_w = f * h * (svp/p) + + return x_w + + def get_compress(T, p, x_w): + """ + Gets the compressability of moist air. + From Equation (12) in Appendix A. + + Args: + T (float): Temperature in [K] Kelvin. + p (float): Pressure in [Pa] Pascals. + x_w (float): Molar fraction of water vapor. + + Returns: + float: Compressability of moist air. + """ + + # t = temperature [C] + t = T - 273.15 + + # constants for Z from Appendix A + a0 = 1.58123 * 10**-6 + a1 = -2.9331 * 10**-8 + a2 = 1.1043 * 10**-10 + b0 = 5.707 * 10**-6 + b1 = -2.051 * 10**-8 + c0 = 1.9898 * 10**-4 + c1 = -2.376 * 10**-6 + d = 1.83 * 10**-11 + e = -0.765 * 10**-8 + + # Compresibility of moist air. From equation (12) in Appendix A + Z = 1 - (p/T) * (a0 + a1*t + a2 * t**2 + (b0 + b1*t) * x_w + (c0 + c1*t) * x_w**2) + (p/T)**2 * (d + e * x_w**2) + return Z + + def get_density(T, p, h, x_c, x_w=None): + """ + Gets the density of moist air in [kg/m^3]. + From equation (4) in reference paper. + + Args: + T (float): Temperature in [K] Kelvin. + p (float): Pressure in [Pa] Pascal. + h (float): Percent relative humidity. + x_c (int): Ppm of CO2. + x_w (float, optional): Set a default water vaport molar fraction. Defaults to None. + + Returns: + float: Density of moist air. + """ + + # molar mass of water vapor [kg/mol] + M_w = 0.018015 + # molar mass of dry air with x_c ppm of CO2 [kg/mol] + M_a = 10**-3 * (28.9635 + 12.011 * 10**-6 * (x_c - 400)) + # gas constant [J/(mol*K)] + R = 8.314510 + + # molar fraction of water vapor in moist air + if x_w is None: + x_w = get_molar_frac_water(T, p, h) + + # compresibility of moist air + Z = get_compress(T, p, x_w) + + # density of moist air. From equation (4) + rho = (p * M_a / Z * R * T) * (1 - x_w * (1 - M_w / M_a)) + + return rho + + def get_rind(T, p, h, x_c): + """ + Gets the refractive index of moist air. + From Equation (5) in the reference paper. + Uses equations for dry air (found in step 8 of Appendix B) + and moist air (found in step 9 of Appendix B) components. + + Args: + T (float): Temperature in [K] Kelvin. + p (float): Pressure in [Pa] Pascal. + h (float): Percent relative humidity. + x_c (int): Ppm of CO2. + + Returns: + float: Refractive index of moist air. + """ + + # molar mass of water vapor [kg/mol] + M_w = 0.018015 + # molar mass of dry air with x_c ppm of CO2 [kg/mol] + M_a = 10**-3 * (28.9635 + 12.011 * 10**-6 * (x_c - 400)) + # gas constant [J/(mol*K)] + R = 8.314510 + + # density of standard dry air (15 deg C, 101325 Pa, x_w = 0, 450 ppm CO2) + rho_axs = get_density(288.15, 101315, 0, 450, 0) + # density of standard water vapor (20 deg C, 1333 Pa, x_w = 1) + rho_ws = get_density(293.15, 1333, 1, 0, 1) + # molar fraction of water vapor in moist air + x_w = get_molar_frac_water(T, p, h) + # compressability of moist air + Z = get_compress(T, p, x_w) + + # density of dry air component of moist air with actual conditions. From step 8 in Appendix B + rho_a = p * M_a * (1 - x_w) / Z * R * T + # density of water vapor component of moist air with actual conditons. From step 9. in Appendix B + rho_w = p * M_w * x_w / Z * R * T + + # refractive index of moist air. From Equation (5) + n_prop = 1 + ((rho_a/rho_axs) * (n_axs - 1) + (rho_w/rho_ws) * (n_ws - 1)) + + return n_prop + + # refractive index + n_prop = get_rind(T, p, h, x_c) + + new_rind = n_prop[int(n_prop.shape[0] / 2)] rwav = wavelength[int(wavelength.shape[0] / 2)] - self.logger.info("Refractive index = %.10f at %.3f Ang" % (rind, rwav)) + self.logger.info("Refractive index = %.10f at %.3f Ang" % (new_rind, rwav)) + # only modify above 2000A - factor = factor*(wavelength >= 2000.) + 1.*(wavelength < 2000.) + n_prop = n_prop*(wavelength >= 2000.) + 1.*(wavelength < 2000.) # Convert - wavelength = wavelength*factor + new_wavelength = wavelength*n_prop + # Units - new_wave = wavelength*u.AA + new_wave = new_wavelength*u.AA new_wave.to(wave.unit) return new_wave - def heliocentric(self, obj, correction_mode, mask=False, resample=True, - vcorr=None): + def heliocentric(self, obj, correction_mode, vcorr=None): """ - Apply heliocentric correction to the cubes. - - *Note that this only works for KCWI data because the location of - Keck Observatory is hard-coded in the function.* + Apply heliocentric correction to old wavelengths. Adapted from https://github.com/dbosul/cwitools.git Args: obj (astropy HDU / HDUList): Input HDU/HDUList with 3D data. - correction_mode (str): "none", "barycentric", or "heliocentric" - mask (bool): Set if the cube is a mask cube. This only works for - resampled cubes. - resample (bool): Resample the cube to the original wavelength grid? + correction_mode (str): "none", "barycentric", or "heliocentric". vcorr (float): Use a different correction velocity. Returns: - HDU / HDUList: Trimmed FITS object with updated header. - vcorr (float): (if vcorr is True) Correction velocity in km/s. - Return type matches type of fits_in argument. - - Examples: - - To apply heliocentric correction, - - >>> hdu_new = heliocentric(hdu_old) - - However, this resamples the wavelengths back to the original grid. - To use the new grid without resampling the data, - - >>> hdu_new = heliocentric(hdu_old, resample=False) + float: Correction velocity in km/s. """ + # check if barycentric correction barycentric = ("barycentric" in correction_mode) - cube = np.nan_to_num(obj.data, - nan=0, posinf=0, neginf=0) - - v_old = 0. - if 'VCORR' in obj.header: - v_old = obj.header['VCORR'] - self.logger.info("Rolling back the existing correction with:") - self.logger.info("Vcorr = %.2f km/s." % v_old) - if vcorr is None: targ = SkyCoord( obj.header['TARGRA'], @@ -265,49 +456,7 @@ def heliocentric(self, obj, correction_mode, mask=False, resample=True, self.logger.info("Helio/Barycentric correction:") self.logger.info("Vcorr = %.2f km/s." % vcorr) - v_tot = vcorr-v_old - - if not resample: - obj.header['CRVAL3'] *= (1 + v_tot / 2.99792458e5) - obj.header['CD3_3'] *= (1 + v_tot / 2.99792458e5) - obj.header['VCORR'] = (vcorr, 'km/s') - obj.header['VCORRTYP'] = (correction_mode, 'Vcorr type') - return obj - - wav_old = self.get_wav_axis(obj.header) - wav_hel = wav_old * (1 + v_tot / 2.99792458e5) - cwave_hel = self.action.args.cwave * (1 + v_tot / 2.99792458e5) - self.logger.info("Vcorr for CWAVE (%.3f) gives %.3f" % - (self.action.args.cwave, cwave_hel)) - - # resample to uniform grid - self.logger.info("Resampling to uniform grid") - cube_new = np.zeros_like(cube) - for i in range(cube.shape[2]): - for j in range(cube.shape[1]): - - spc0 = cube[:, j, i] - if not mask: - f_cubic = interp1d(wav_hel, spc0, kind='cubic', - fill_value='extrapolate') - spec_new = f_cubic(wav_old) - - else: - f_pre = interp1d(wav_hel, spc0, kind='previous', - bounds_error=False, fill_value=128) - spec_pre = f_pre(wav_old) - f_nex = interp1d(wav_hel, spc0, kind='next', - bounds_error=False, fill_value=128) - spec_nex = f_nex(wav_old) - spec_new = np.zeros_like(spc0) - for k in range(spc0.shape[0]): - spec_new[k] = max(spec_pre[k], spec_nex[k]) - cube_new[:, j, i] = spec_new - - obj.header['VCORR'] = (vcorr, 'km/s') - obj.header['VCORRTYP'] = (correction_mode, 'Vcorr type') - obj.data = cube_new - return obj + return vcorr def get_wav_axis(self, header): """Returns a NumPy array representing the wavelength axis of a cube. diff --git a/kcwidrp/primitives/kcwi_file_primitives.py b/kcwidrp/primitives/kcwi_file_primitives.py index b2ab771..dcd630f 100644 --- a/kcwidrp/primitives/kcwi_file_primitives.py +++ b/kcwidrp/primitives/kcwi_file_primitives.py @@ -819,6 +819,7 @@ def _perform(self): ccddata, table = kcwi_fits_reader(self.name) + # Are we already in proctab? out_args.in_proctab = self.context.proctab.in_proctab(frame=ccddata) if out_args.in_proctab: @@ -929,6 +930,12 @@ def _perform(self): out_args.calibration_lamp = self.calibration_lamp() # TTIME out_args.ttime = self.get_keyword('TTIME') + # WXOUTHUM + out_args.dome_hum = self.get_keyword('WXDOMHUM') + # WXOUTTMP + out_args.dome_temp = self.get_keyword('WXDOMTMP') + # WXPRESS + out_args.pres = self.get_keyword('WXPRESS') return out_args @@ -1290,7 +1297,7 @@ def get_master_name(tab, target_type, loc=0): return res -def master_bias_name(ccddata, target_type='MBIAS'): +def get_unique_CCD_master_name(ccddata, target_type='MBIAS'): # Currently NOT USED (DN, 7-Sep-2023) # Delivers a mbias filename that is unique for each CCD configuration # Any KCWI frame with a shared CCD configuration can use the same bias @@ -1298,7 +1305,7 @@ def master_bias_name(ccddata, target_type='MBIAS'): return name -def master_flat_name(ccddata, target_type): +def get_unique_STATEID_master_name(ccddata, target_type): # Currently NOT USED (DN, 7-Sep-2023) # Delivers a name that is unique across an observing block name = target_type.lower() + '_' + ccddata.header['STATEID'] + '.fits' diff --git a/kcwidrp/scripts/check_cals.py b/kcwidrp/scripts/check_cals.py index 034c7ca..d3246ce 100644 --- a/kcwidrp/scripts/check_cals.py +++ b/kcwidrp/scripts/check_cals.py @@ -27,6 +27,7 @@ from kcwidrp.core.kcwi_proctab import Proctab from keckdrpframework.config.framework_config import ConfigClass from kcwidrp.core.kcwi_get_std import kcwi_get_std +from kcwidrp.primitives.kcwi_file_primitives import fix_header warnings.simplefilter('ignore', category=AstropyWarning) @@ -50,6 +51,8 @@ def parse_args(): parser.add_argument('filepaths', help="Files to inspect", nargs="+") parser.add_argument('-v', '--verbose', dest="verbose", action="store_true", help="Print exhaustive information") + parser.add_argument('-a', '--auto', dest="auto", action="store_true", + help='For autonomy mode, only returns PASSED or FAILED.') parser.add_argument('-c', '--config', dest="config", type=str, help="KCWI configuration file", default=None) @@ -99,10 +102,13 @@ def main(): logger = logging.getLogger("Logger") logger.addHandler(logging.StreamHandler()) - if args.verbose: - logger.setLevel("DEBUG") + if args.auto: + logger.setLevel("ERROR") else: - logger.setLevel("INFO") + if args.verbose: + logger.setLevel("DEBUG") + else: + logger.setLevel("INFO") # Load config @@ -135,6 +141,13 @@ def main(): for file in files: try: frame = CCDData.read(file, unit='adu') + fix_header(frame) + if 'CCDCFG' not in frame.header: + ccdcfg = frame.header['CCDSUM'].replace(" ", "") + ccdcfg += "%1d" % frame.header['CCDMODE'] + ccdcfg += "%02d" % frame.header['GAINMUL'] + ccdcfg += "%02d" % frame.header['AMPMNUM'] + frame.header['CCDCFG'] = ccdcfg except FileNotFoundError as e: logger.error(f"Failed to open {file}") proctab.update_proctab(frame, filename=file.name) @@ -264,6 +277,12 @@ def main(): logger.info('\t' + str(objects[objects["CID"] == fail]['filename', 'TARGNAME']).replace('\n', '\n\t')) else: logger.info("\033[32mNo failures to report.\033[0m") + + if args.auto: + if len(passes) > 0 and len(fails) == 0: + print("PASSED") + else: + print("FAILED") if __name__ == "__main__": main() \ No newline at end of file diff --git a/kcwidrp/scripts/kcwi_rti.py b/kcwidrp/scripts/kcwi_rti.py index 988ed6e..6a8473a 100644 --- a/kcwidrp/scripts/kcwi_rti.py +++ b/kcwidrp/scripts/kcwi_rti.py @@ -13,11 +13,14 @@ import subprocess import time +import datetime import argparse import sys import traceback import os import pkg_resources +import psutil +import shutil from kcwidrp.pipelines.keck_rti_pipeline import Keck_RTI_Pipeline from kcwidrp.core.kcwi_proctab import Proctab @@ -32,6 +35,14 @@ def _parse_arguments(in_args: list) -> argparse.Namespace: description=description) parser.add_argument('-c', '--config', dest="kcwi_config_file", type=str, help="KCWI configuration file", default=None) + + parser.add_argument('--rti-cfg', dest="rti_config_file", type=str, + help="RTI configuration file", default=None) + parser.add_argument('--rti-ingesttype', choices=['lev1', 'lev2'], dest="rti_ingesttype", type=str, + help="RTI ingest type", default=None, required=True) + parser.add_argument('--write_config', dest="write_config", + help="Write out an editable config file in current dir" + " (kcwi.cfg)", action="store_true", default=False) parser.add_argument('-f', '--frames', nargs='*', type=str, help='input image files (full path, list ok)', default=None) @@ -47,6 +58,21 @@ def _parse_arguments(in_args: list) -> argparse.Namespace: default=None) parser.add_argument('-a', '--atlas_line_list', dest='atlas_line_list', type=str, help="Atlas line list file", default=None) + parser.add_argument('-M', '--middle_fraction', dest='middle_fraction', + type=float, help="Fraction of middle to use", + default=None) + parser.add_argument('-o', '--atlas_offset', dest='atlas_offset', + type=int, help="Atlas offset (px)", default=None) + parser.add_argument('-e', '--line_thresh', dest='line_thresh', + type=float, help="Line Cleaning Threshold (e-)", + default=None) + parser.add_argument('-u', '--tukey_alpha', dest='tukey_alpha', + type=float, help="Tukey Window Alpha (0.0 - 1.0)", + default=None) + parser.add_argument('-F', '--max_frac', dest='max_frac', + type=float, default=None, + help="Fraction of line max for fitting window " + "(default: 0.5)") # in this case, we are loading an entire directory, # and ingesting all the files in that directory @@ -74,9 +100,15 @@ def _parse_arguments(in_args: list) -> argparse.Namespace: dest="queue_manager_only", action="store_true", help="Starts queue manager only, no processing",) - # kcwi specific parameter + # kcwi specific parameters parser.add_argument("-p", "--proctab", dest='proctab', help='Proctab file', - default='kcwi.proc') + default=None) + parser.add_argument("-b", "--blue", dest='blue', action="store_true", + default=False, help="KCWI Blue processing") + parser.add_argument("-r", "--red", dest='red', action="store_true", + default=False, help="KCWI Red processing") + parser.add_argument("-k", "--skipsky", dest='skipsky', action="store_true", + default=False, help="Skip sky subtraction") out_args = parser.parse_args(in_args[1:]) return out_args @@ -90,6 +122,24 @@ def check_directory(directory): def main(): + # Package + pkg = 'kcwidrp' + + # get arguments + args = _parse_arguments(sys.argv) + + if args.write_config: + dest = os.path.join(os.getcwd(), 'kcwi.cfg') + if os.path.exists(dest): + print("Config file kcwi.cfg already exists in current dir") + else: + kcwi_config_file = 'configs/kcwi.cfg' + kcwi_config_fullpath = pkg_resources.resource_filename( + pkg, kcwi_config_file) + shutil.copy(kcwi_config_fullpath, os.getcwd()) + print("Copied kcwi.cfg into current dir. Edit and use with -c") + sys.exit(0) + def process_subset(in_subset): for in_frame in in_subset.index: arguments = Arguments(name=in_frame) @@ -100,10 +150,21 @@ def process_list(in_list): arguments = Arguments(name=in_frame) framework.append_event('next_file', arguments, recurrent=True) - args = _parse_arguments(sys.argv) + # make sure user has selected a channel + if not args.blue and not args.red: + print("\nERROR - DRP can process only one channel at a time\n\n" + "Please indicate a channel to process:\n" + "Either BLUE with -b or --blue or\n" + " RED with -r or --red\n") + sys.exit(0) + + if args.file_list: + if '.fits' in args.file_list: + print("\nERROR - trying to read in fits file as file list\n\n" + "Please use -f or --frames for direct input of fits files\n") + sys.exit(0) # START HANDLING OF CONFIGURATION FILES ########## - pkg = 'kcwidrp' # check for the logs diretory check_directory("logs") @@ -129,8 +190,11 @@ def process_list(in_list): # kcwi_config_fullpath = os.path.abspath(args.kcwi_config_file) kcwi_config = ConfigClass(args.kcwi_config_file, default_section='KCWI') - rti_config_file = "configs/rti.cfg" - rti_config_fullpath = pkg_resources.resource_filename(pkg, rti_config_file) + if args.rti_config_file is None: + rti_config_file = "configs/rti.cfg" + rti_config_fullpath = pkg_resources.resource_filename(pkg, rti_config_file) + else: + rti_config_fullpath = args.rti_config_file rti_config = ConfigClass(rti_config_fullpath, default_section='RTI') # END HANDLING OF CONFIGURATION FILES ########## @@ -146,6 +210,7 @@ def process_list(in_list): logging.config.fileConfig(framework_logcfg_fullpath) framework.config.instrument = kcwi_config framework.config.rti = rti_config + framework.config.rti.rti_ingesttype = args.rti_ingesttype except Exception as e: print("Failed to initialize framework, exiting ...", e) traceback.print_exc() @@ -158,6 +223,13 @@ def process_list(in_list): if args.infiles is not None: framework.config.file_type = args.infiles + # check for skipsky argument + if args.skipsky: + def_sk = getattr(framework.config.instrument, 'skipsky', None) + if def_sk is not None: + framework.context.pipeline_logger.info("Skipping sky subtraction") + framework.config.instrument.skipsky = args.skipsky + # check for taperfrac argument if args.taperfrac: def_tf = getattr(framework.config.instrument, 'TAPERFRAC', None) @@ -166,6 +238,68 @@ def process_list(in_list): "Setting new taperfrac = %.3f" % args.taperfrac) framework.config.instrument.TAPERFRAC = args.taperfrac + # check for middle_fraction argument + if args.middle_fraction: + def_mf = getattr(framework.config.instrument, 'MIDFRAC', None) + if def_mf is not None: + framework.context.pipeline_logger.info( + "Setting new middle_fraction = %.2f" % args.middle_fraction) + framework.config.instrument.MIDFRAC = args.middle_fraction + + # check for atlas_offset argument + if args.atlas_offset: + def_ao = getattr(framework.config.instrument, 'ATOFF', None) + if def_ao is not None: + framework.context.pipeline_logger.info( + "Setting new atlas offset = %.2f" % args.atlas_offset) + framework.config.instrument.ATOFF = args.atlas_offset + + # check for line_thresh argument + if args.line_thresh: + def_lt = getattr(framework.config.instrument, 'LINETHRESH', None) + if def_lt is not None: + framework.context.pipeline_logger.info( + "Setting new line thresh = %.2f" % args.line_thresh) + framework.config.instrument.LINETHRESH = args.line_thresh + else: + if args.blue: + framework.config.instrument.LINETHRESH = float( + kcwi_config.BLUE['linethresh']) + elif args.red: + framework.config.instrument.LINETHRESH = float( + kcwi_config.RED['linethresh']) + + # check for tukey_alpha argument + if args.tukey_alpha: + def_ta = getattr(framework.config.instrument, 'TUKEYALPHA', None) + if def_ta is not None: + framework.context.pipeline_logger.info( + "Setting new tukey alpha = %.2f" % args.tukey_alpha) + framework.config.instrument.TUKEYALPHA = args.tukey_alpha + else: + if args.blue: + framework.config.instrument.TUKEYALPHA = float( + kcwi_config.BLUE['tukeyalpha']) + elif args.red: + framework.config.instrument.TUKEYALPHA = float( + kcwi_config.RED['tukeyalpha']) + + # check for max_frac argument + if args.max_frac: + def_fm = getattr(framework.config.instrument, 'FRACMAX', None) + if def_fm is not None: + framework.context.pipeline_logger.info( + "Setting new line windowing max fraction = %.2f" % + args.max_frac) + framework.config.instrument.FRACMAX = args.max_frac + else: + if args.blue: + framework.config.instrument.FRACMAX = float( + kcwi_config.BLUE['fracmax']) + elif args.red: + framework.config.instrument.FRACMAX = float( + kcwi_config.RED['fracmax']) + # check for atlas line list argument if args.atlas_line_list: def_ll = getattr(framework.config.instrument, 'LINELIST', None) @@ -175,10 +309,62 @@ def process_list(in_list): args.atlas_line_list) framework.config.instrument.LINELIST = args.atlas_line_list + # update proc table argument + if args.proctab: + framework.context.pipeline_logger.info( + "Using proc table file %s" % args.proctab + ) + framework.config.instrument.procfile = args.proctab + else: + if args.blue: + proctab = kcwi_config.BLUE['procfile'] + elif args.red: + proctab = kcwi_config.RED['procfile'] + else: + proctab = kcwi_config.procfile + framework.context.pipeline_logger.info( + "Using proc table file %s" % proctab) + framework.config.instrument.procfile = proctab + + # set up channel specific parameters + if args.blue: + framework.config.instrument.arc_min_nframes = int( + kcwi_config.BLUE['arc_min_nframes']) + framework.config.instrument.contbars_min_nframes = int( + kcwi_config.BLUE['contbars_min_nframes']) + framework.config.instrument.object_min_nframes = int( + kcwi_config.BLUE['object_min_nframes']) + framework.config.instrument.minoscanpix = int( + kcwi_config.BLUE['minoscanpix']) + framework.config.instrument.oscanbuf = int( + kcwi_config.BLUE['oscanbuf']) + elif args.red: + framework.config.instrument.arc_min_nframes = int( + kcwi_config.RED['arc_min_nframes']) + framework.config.instrument.contbars_min_nframes = int( + kcwi_config.RED['contbars_min_nframes']) + framework.config.instrument.object_min_nframes = int( + kcwi_config.RED['object_min_nframes']) + framework.config.instrument.minoscanpix = int( + kcwi_config.RED['minoscanpix']) + framework.config.instrument.oscanbuf = int( + kcwi_config.RED['oscanbuf']) + else: + framework.config.instrument.arc_min_nframes = \ + kcwi_config.arc_min_nframes + framework.config.instrument.contbars_min_nframes = \ + kcwi_config.contbars_min_nframes + framework.config.instrument.object_min_nframes = \ + kcwi_config.object_min_nframes + framework.config.instrument.minoscanpix = kcwi_config.minoscanpix + framework.config.instrument.oscanbuf = kcwi_config.oscanbuf + # start the bokeh server is requested by the configuration parameters if framework.config.instrument.enable_bokeh is True: if check_running_process(process='bokeh') is False: - subprocess.Popen('bokeh serve', shell=True) + with open("bokeh_output.txt", "wb") as out: + subprocess.Popen('bokeh serve', shell=True, stderr=out, + stdout=out) # --session-ids=unsigned --session-token-expiration=86400', # shell=True) time.sleep(5) @@ -187,7 +373,7 @@ def process_list(in_list): # initialize the proctab and read it framework.context.proctab = Proctab() - framework.context.proctab.read_proctab(tfil=args.proctab) + framework.context.proctab.read_proctab(framework.config.instrument.procfile) framework.logger.info("Framework initialized") framework.logger.info(f"RTI url is {framework.config.rti.rti_url}") @@ -209,7 +395,7 @@ def process_list(in_list): if args.queue_manager_only: # The queue manager runs for ever. framework.logger.info("Starting queue manager only, no processing") - framework.start_queue_manager() + framework.start(args.queue_manager_only) # in the next two ingest_data command, if we are using standard mode, # the first event generated is next_file. @@ -219,6 +405,22 @@ def process_list(in_list): # single frame processing elif args.frames: + frames = [] + for frame in args.frames: + # Verify we have the correct channel selected + if args.blue and ('kr' in frame or 'KR' in frame): + print('Blue channel requested, but red files in list') + qstr = input('Proceed? =yes or Q=quit: ') + if 'Q' in qstr.upper(): + frames = [] + break + if args.red and ('kb' in frame or 'KB' in frame): + print('Red channel requested, but blue files in list') + qstr = input('Proceed? =yes or Q=quit: ') + if 'Q' in qstr.upper(): + frames = [] + break + frames.append(frame) framework.ingest_data(None, args.frames, False) # processing of a list of files contained in a file @@ -227,9 +429,26 @@ def process_list(in_list): with open(args.file_list) as file_list: for frame in file_list: if "#" not in frame: + # Verify we have the correct channel selected + if args.blue and ('kr' in frame or 'KR' in frame): + print('Blue channel requested, but red files in list') + qstr = input('Proceed? =yes or Q=quit: ') + if 'Q' in qstr.upper(): + frames = [] + break + if args.red and ('kb' in frame or 'KB' in frame): + print('Red channel requested, but blue files in list') + qstr = input('Proceed? =yes or Q=quit: ') + if 'Q' in qstr.upper(): + frames = [] + break frames.append(frame.strip('\n')) framework.ingest_data(None, frames, False) + with open(args.file_list + '_ingest', 'w') as ingest_f: + ingest_f.write('Files ingested at: ' + + datetime.datetime.now().isoformat()) + # ingest an entire directory, trigger "next_file" (which is an option # specified in the config file) on each file, # optionally continue to monitor if -m is specified