From 8c2081fab9f4de3d752f8246a89d01c262711f86 Mon Sep 17 00:00:00 2001 From: Tim Plummer Date: Fri, 27 Jun 2025 15:16:15 -0600 Subject: [PATCH] Add new xtce file for Hi packet definitions that includes new MEMDMP packets for HI90 and HI45 Update L1A DE field name to match change in xtce packet definition Rename Hi L1C calibration file to match expected --- .../cdf/config/imap_hi_variable_attrs.yaml | 16 +- imap_processing/hi/hi_l1a.py | 8 +- imap_processing/hi/hi_l1b.py | 4 +- .../TLM_HI_COMBINED_SCI.xml | 256 +++++++++++++++--- ...ap_hi_90sensor-cal-prod_20240101_v001.csv} | 0 imap_processing/tests/hi/test_hi_l1c.py | 4 +- imap_processing/tests/hi/test_l1a.py | 4 +- 7 files changed, 235 insertions(+), 57 deletions(-) rename imap_processing/tests/hi/data/l1/{imap_his_pset-calibration-prod-config_20240101_v001.csv => imap_hi_90sensor-cal-prod_20240101_v001.csv} (100%) diff --git a/imap_processing/cdf/config/imap_hi_variable_attrs.yaml b/imap_processing/cdf/config/imap_hi_variable_attrs.yaml index b72b77adba..cba537d1b0 100644 --- a/imap_processing/cdf/config/imap_hi_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_hi_variable_attrs.yaml @@ -161,21 +161,21 @@ hi_de_spin_invalids: hi_de_esa_step: <<: *hi_esa_step -hi_de_meta_seconds: +hi_de_esa_step_seconds: <<: *default_uint32 - CATDESC: Coarse mission elapsed time (MET) of last meta-event + CATDESC: Coarse mission elapsed time (MET) of when the ESA was stepped DISPLAY_TYPE: time_series - FIELDNAM: Coarse Meta-event MET - LABLAXIS: Coarse Meta MET + FIELDNAM: Coarse ESA step MET + LABLAXIS: Coarse ESA step MET UNITS: seconds VAR_TYPE: support_data -hi_de_meta_subseconds: +hi_de_esa_step_milliseconds: <<: *default_uint16 - CATDESC: Fine mission elapsed time (MET) of last meta-event + CATDESC: Fine mission elapsed time (MET) of when the ESA was stepped DISPLAY_TYPE: time_series - FIELDNAM: Fine Meta-event MET - LABLAXIS: Fine Meta MET + FIELDNAM: Fine ESA step MET + LABLAXIS: Fine ESA step MET UNITS: ms VAR_TYPE: support_data diff --git a/imap_processing/hi/hi_l1a.py b/imap_processing/hi/hi_l1a.py index 459713050d..9659ab7b40 100644 --- a/imap_processing/hi/hi_l1a.py +++ b/imap_processing/hi/hi_l1a.py @@ -170,8 +170,8 @@ def finish_de_dataset(packets_data: xr.Dataset) -> xr.Dataset: "last_spin_num": "last_spin_num", "spin_invalids": "spin_invalids", "esa_step_num": "esa_step", - "meta_seconds": "meta_seconds", - "meta_subseconds": "meta_subseconds", + "esa_step_seconds": "esa_step_seconds", + "esa_step_milliseconds": "esa_step_milliseconds", }.items(): de_data_dict[to_key] = packets_data[from_key].data @@ -228,8 +228,8 @@ def create_de_dataset(de_data_dict: dict[str, npt.ArrayLike]) -> xr.Dataset: # Compute the meta-event MET in seconds meta_event_met = ( - np.array(de_data_dict["meta_seconds"]).astype(np.float64) - + np.array(de_data_dict["meta_subseconds"]) * MILLISECOND_TO_S + np.array(de_data_dict["esa_step_seconds"]).astype(np.float64) + + np.array(de_data_dict["esa_step_milliseconds"]) * MILLISECOND_TO_S ) # Compute the MET of each event in seconds # event MET = meta_event_met + de_clock diff --git a/imap_processing/hi/hi_l1b.py b/imap_processing/hi/hi_l1b.py index 905163aeb5..a435105d97 100644 --- a/imap_processing/hi/hi_l1b.py +++ b/imap_processing/hi/hi_l1b.py @@ -169,8 +169,8 @@ def annotate_direct_events(l1a_dataset: xr.Dataset) -> xr.Dataset: "pkt_len", "last_spin_num", "spin_invalids", - "meta_seconds", - "meta_subseconds", + "esa_step_seconds", + "esa_step_milliseconds", "tof_1", "tof_2", "tof_3", diff --git a/imap_processing/hi/packet_definitions/TLM_HI_COMBINED_SCI.xml b/imap_processing/hi/packet_definitions/TLM_HI_COMBINED_SCI.xml index 591963826f..b1e648471f 100644 --- a/imap_processing/hi/packet_definitions/TLM_HI_COMBINED_SCI.xml +++ b/imap_processing/hi/packet_definitions/TLM_HI_COMBINED_SCI.xml @@ -1,6 +1,6 @@ - + @@ -24,6 +24,56 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -970,16 +1020,16 @@ - + - + - + @@ -1271,6 +1321,56 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -1386,7 +1486,7 @@ - + @@ -2216,16 +2316,16 @@ - + - + - + @@ -2540,6 +2640,28 @@ CCSDS Packet Length (number of bytes after Packet length minus 1) + + MISSION ELAPSED TIME + + + PACKET VERSION - THIS WILL BE INCREMENTED EACH TIME THE FORMAT OF THE PACKET CHANGES. + + + SPARE FOR ALIGNMENT + + + MEMORY DEVICE/TYPE IDENTIFIER. INDICATES THE SOURCE FOR THE DATA INCLUDED IN THE DATA FIELD + + + START ADDRESS WITHIN THE DEVICE + + + NUMBER OF BYTES INCLUDED IN THE DATA FIELD (NOT INCLUDING PAD BYTES AND THE PACKET CHECKSUM) + + + DUMPED DATA. SINCE THIS IS A VARIABLE LENGTH FIELD IT WILL ALSO ENCOMPASS ANY PAD BYTES AND THE PACKET CHECKSUM. + + MISSION ELAPSED TIME @@ -2894,23 +3016,24 @@ MOS SIGNIFICANT BIT IS ALSO THE HIGHEST SPIN - - - - - + + + ESA STEP NUMBER WHEN THE DATA IS COLLECTED STARTING AT 1 + + + TIMESTAMP OF ESA STEP STARTING TIME, MILLISECONDS + + + TIMESTAMP OF ESA STEP STARTING TIME, SECONDS + + DIRECT EVENT TOF DATA. -LENGTH IS VARIABLE. MAX IS 665 X 48 BITS +LENGTH IS VARIABLE. MAX IS 664 X 48 BITS START_BITMASK_DATA - 2 BITS (TA=1, TB=2, TC1=3, META=0) -IF START_BITMASK != 0 DE_TAG - 16 BITS TOF_1 - 10 BITS TOF_2 - 10 BITS -TOF_3 - 10 BITS -IF START_BIT_MASK == 0: -ESA_STEP_NUM - 4 BITS -MET_SUBSECONDS - 10 BITS -MET_SECONDS - 32 BITS +TOF_3 - 10 BITS @@ -3005,6 +3128,28 @@ MET_SECONDS - 32 BITS + + MISSION ELAPSED TIME + + + PACKET VERSION - THIS WILL BE INCREMENTED EACH TIME THE FORMAT OF THE PACKET CHANGES. + + + SPARE FOR ALIGNMENT + + + MEMORY DEVICE/TYPE IDENTIFIER. INDICATES THE SOURCE FOR THE DATA INCLUDED IN THE DATA FIELD + + + START ADDRESS WITHIN THE DEVICE + + + NUMBER OF BYTES INCLUDED IN THE DATA FIELD (NOT INCLUDING PAD BYTES AND THE PACKET CHECKSUM) + + + DUMPED DATA. SINCE THIS IS A VARIABLE LENGTH FIELD IT WILL ALSO ENCOMPASS ANY PAD BYTES AND THE PACKET CHECKSUM. + + MISSION ELAPSED TIME @@ -3359,23 +3504,24 @@ MET_SECONDS - 32 BITS MOS SIGNIFICANT BIT IS ALSO THE HIGHEST SPIN - - - - - + + + ESA STEP NUMBER WHEN THE DATA IS COLLECTED STARTING AT 1 + + + TIMESTAMP OF ESA STEP STARTING TIME, MILLISECONDS + + + TIMESTAMP OF ESA STEP STARTING TIME, SECONDS + + DIRECT EVENT TOF DATA. -LENGTH IS VARIABLE. MAX IS 665 X 48 BITS +LENGTH IS VARIABLE. MAX IS 664 X 48 BITS START_BITMASK_DATA - 2 BITS (TA=1, TB=2, TC1=3, META=0) -IF START_BITMASK != 0 DE_TAG - 16 BITS TOF_1 - 10 BITS TOF_2 - 10 BITS -TOF_3 - 10 BITS -IF START_BIT_MASK == 0: -ESA_STEP_NUM - 4 BITS -MET_SUBSECONDS - 10 BITS -MET_SECONDS - 32 BITS +TOF_3 - 10 BITS @@ -3483,6 +3629,23 @@ MET_SECONDS - 32 BITS + + + + + + + + + + + + + + + + + @@ -3627,10 +3790,10 @@ MET_SECONDS - 32 BITS - + - - + + @@ -3736,6 +3899,23 @@ MET_SECONDS - 32 BITS + + + + + + + + + + + + + + + + + @@ -3880,10 +4060,10 @@ MET_SECONDS - 32 BITS - + - - + + diff --git a/imap_processing/tests/hi/data/l1/imap_his_pset-calibration-prod-config_20240101_v001.csv b/imap_processing/tests/hi/data/l1/imap_hi_90sensor-cal-prod_20240101_v001.csv similarity index 100% rename from imap_processing/tests/hi/data/l1/imap_his_pset-calibration-prod-config_20240101_v001.csv rename to imap_processing/tests/hi/data/l1/imap_hi_90sensor-cal-prod_20240101_v001.csv diff --git a/imap_processing/tests/hi/test_hi_l1c.py b/imap_processing/tests/hi/test_hi_l1c.py index 383a38acf7..da9841d998 100644 --- a/imap_processing/tests/hi/test_hi_l1c.py +++ b/imap_processing/tests/hi/test_hi_l1c.py @@ -19,9 +19,7 @@ @pytest.fixture(scope="module") def hi_test_cal_prod_config_path(hi_l1_test_data_path): - return ( - hi_l1_test_data_path / "imap_his_pset-calibration-prod-config_20240101_v001.csv" - ) + return hi_l1_test_data_path / "imap_hi_90sensor-cal-prod_20240101_v001.csv" @mock.patch("imap_processing.hi.hi_l1c.generate_pset_dataset") diff --git a/imap_processing/tests/hi/test_l1a.py b/imap_processing/tests/hi/test_l1a.py index 9f2245af82..88235ece76 100644 --- a/imap_processing/tests/hi/test_l1a.py +++ b/imap_processing/tests/hi/test_l1a.py @@ -157,8 +157,8 @@ def test_create_de_dataset(): "last_spin_num": np.arange(n_packets) + 3, "spin_invalids": np.zeros(n_packets), "esa_step": np.full(n_packets, 4), - "meta_seconds": np.arange(n_packets) + 433522960, - "meta_subseconds": np.full(n_packets, 500), + "esa_step_seconds": np.arange(n_packets) + 433522960, + "esa_step_milliseconds": np.full(n_packets, 500), "trigger_id": [1 + i % 3 for i in range(n_events)], "tof_1": [450] * n_events, "tof_2": [450] * n_events,