added apid 897

Author: laspsandoval

imap_processing/cdf/config/imap_ultra_global_cdf_attrs.yaml

@@ -56,6 +56,18 @@ imap_ultra_l1a_90sensor-de:
     Logical_source: imap_ultra_l1a_90sensor-de
     Logical_source_description: IMAP-Ultra Instrument Level-1A Direct Event Data.
 
+imap_ultra_l1a_45sensor-energy-de:
+    <<: *instrument_base
+    Data_type: L1A_ENERGY_DE>Level-1A Energy Direct Event
+    Logical_source: imap_ultra_l1a_45sensor-energy-de
+    Logical_source_description: IMAP-Ultra Instrument Level-1A Energy Direct Event Data.
+
+imap_ultra_l1a_90sensor-energy-de:
+    <<: *instrument_base
+    Data_type: L1A_ENERGY_DE>Level-1A EnergyDirect Event
+    Logical_source: imap_ultra_l1a_90sensor-energy-de
+    Logical_source_description: IMAP-Ultra Instrument Level-1A Energy Direct Event Data.
+
 imap_ultra_l1b_45sensor-de:
     <<: *instrument_base
     Data_type: L1B_45Sensor-DE>Level-1B Direct Events for Ultra45

imap_processing/tests/external_test_data_config.py

@@ -63,6 +63,8 @@
     ("IMAP-Ultra45_r1_L1_V0_shortened.csv", "ultra/data/l1/"),
 
     # Ultra
+    ("ultra45_raw_sc_rawnrgevnt_FM45_UltraFM45_Functional_2024-01-22T0105_20240122T010548.csv", "ultra/data/l0/"),
+    ("ultra45_raw_sc_imgpriority1evnt_FM45_UltraFM45Extra_TV_Tests_2024-01-22T0930_20240122T093008.csv", "ultra/data/l0/"),
     ("imap_ultra_l0_raw_20260924_v001.pkts", "ultra/data/l0/"),
     ("imap_ultra_l1b_45sensor-de_20240207_v999.cdf", "ultra/data/l1/"),
     ("ultra-90_raw_event_data_shortened.csv", "ultra/data/l1/"),

imap_processing/tests/ultra/unit/conftest.py

@@ -12,6 +12,7 @@
 )
 from imap_processing.ultra.l0.ultra_utils import (
     ULTRA_AUX,
+    ULTRA_ENERGY_EVENTS,
     ULTRA_EVENTS,
     ULTRA_RATES,
     ULTRA_TOF,
@@ -86,6 +87,19 @@ def ccsds_path_tof():
     )
 
 
+@pytest.fixture
+def ccsds_path_functional():
+    """Returns the ccsds directory."""
+    return (
+        imap_module_directory
+        / "tests"
+        / "ultra"
+        / "data"
+        / "l0"
+        / "FM45_UltraFM45_Functional_2024-01-22T0105_20240122T010548.CCSDS"
+    )
+
+
 @pytest.fixture
 def xtce_path():
     """Returns the xtce image rates directory."""
@@ -147,15 +161,17 @@ def decom_test_data(request, xtce_path):
 
     strategy_dict = {
         ULTRA_TOF.apid[0]: process_ultra_tof,
+        ULTRA_ENERGY_EVENTS.apid[0]: process_ultra_events,
         ULTRA_EVENTS.apid[0]: process_ultra_events,
         ULTRA_RATES.apid[0]: process_ultra_rates,
         ULTRA_TOF.apid[1]: process_ultra_tof,
+        ULTRA_ENERGY_EVENTS.apid[1]: process_ultra_events,
         ULTRA_EVENTS.apid[1]: process_ultra_events,
         ULTRA_RATES.apid[1]: process_ultra_rates,
     }
 
     process_function = strategy_dict.get(apid, lambda *args: False)
-    data_packet_xarray = process_function(datasets_by_apid[apid])
+    data_packet_xarray = process_function(datasets_by_apid[apid], apid)
 
     return data_packet_xarray
 

imap_processing/tests/ultra/unit/test_ultra_l1a.py

@@ -8,6 +8,7 @@
 from imap_processing.ultra.l0.decom_ultra import get_event_id
 from imap_processing.ultra.l0.ultra_utils import (
     ULTRA_AUX,
+    ULTRA_ENERGY_EVENTS,
     ULTRA_EVENTS,
     ULTRA_RATES,
     ULTRA_TOF,
@@ -123,6 +124,20 @@ def test_cdf_events(ccsds_path_theta_0):
     )
 
 
+def test_cdf_energy_events(ccsds_path_functional):
+    """Tests that CDF file can be created."""
+    test_data = ultra_l1a(ccsds_path_functional, apid_input=ULTRA_ENERGY_EVENTS.apid[0])
+    test_data[0].attrs["Data_version"] = "999"
+    test_data[0].attrs["Repointing"] = "repoint99999"
+    test_data_path = write_cdf(test_data[0], istp=True)
+
+    assert test_data_path.exists()
+    assert (
+        test_data_path.name
+        == "imap_ultra_l1a_45sensor-energy-de_20240122-repoint99999_v999.cdf"
+    )
+
+
 def test_cdf_hk(ccsds_path_theta_0):
     """Tests that CDF file can be created."""
     test_data = ultra_l1a(ccsds_path_theta_0, apid_input=869)

imap_processing/ultra/l0/decom_tools.py

@@ -239,6 +239,7 @@ def decompress_image(
 def read_image_raw_events_binary(
     event_data: bytes,
     count: int,
+    field_ranges: dict,
 ) -> NDArray:
     """
     Convert contents of binary string 'EVENTDATA' into values.
@@ -249,22 +250,25 @@ def read_image_raw_events_binary(
         Event data.
     count : int
         Number of events.
+    field_ranges : dict
+        Field ranges for the event data.
 
     Returns
     -------
     event_data : NDArray
         Event data.
     """
     binary = convert_to_binary_string(event_data)
-    # 166 bits per event
-    event_length = 166 if count else 0
+    length = max(end for _, (_, end) in field_ranges.items())
+    # bits per event
+    event_length = length if count else 0
     event_data_list = []
 
     # For all packets with event data, parses the binary string
     for i in range(count):
         start_index = i * event_length
         event_binary = binary[start_index : start_index + event_length]
-        parsed_event = parse_event(event_binary)
+        parsed_event = parse_event(event_binary, field_ranges)
         event_data_list.append(parsed_event)
 
     return np.array(event_data_list)

imap_processing/ultra/l0/decom_ultra.py

@@ -18,6 +18,8 @@
     ENERGY_EVENT_FIELD_RANGES,
     EVENT_FIELD_RANGES,
     RATES_KEYS,
+    ULTRA_ENERGY_EVENTS,
+    ULTRA_EVENTS,
     ULTRA_RATES,
     ULTRA_TOF,
 )
@@ -137,93 +139,29 @@ def get_event_id(shcoarse: NDArray) -> NDArray:
     return np.array(event_ids, dtype=np.int64)
 
 
-def process_ultra_events(ds: xr.Dataset) -> xr.Dataset:
+def process_ultra_events(ds: xr.Dataset, apid: int) -> xr.Dataset:
     """
     Unpack and decode Ultra EVENTS packets.
 
     Parameters
     ----------
     ds : xarray.Dataset
         Events dataset.
+    apid : int
+        APID of the events dataset.
 
     Returns
     -------
     ds : xarray.Dataset
         Dataset containing the decoded and decompressed data.
     """
-    all_events = []
-    all_indices = []
-
-    attrs = ImapCdfAttributes()
-    attrs.add_instrument_variable_attrs("ultra", level="l1a")
-
-    empty_event = {
-        field: attrs.get_variable_attributes(field).get(
-            "FILLVAL", np.iinfo(np.int64).min
-        )
-        for field in EVENT_FIELD_RANGES
-    }
-
-    counts = ds["count"].values
-    eventdata_array = ds["eventdata"].values
-
-    for i, count in enumerate(counts):
-        if count == 0:
-            all_events.append(empty_event)
-            all_indices.append(i)
-        else:
-            # Here there are multiple images in a single packet,
-            # so we need to loop through each image and decompress it.
-            event_data_list = read_image_raw_events_binary(eventdata_array[i], count)
-            all_events.extend(event_data_list)
-            # Keep track of how many times does the event occurred at this epoch.
-            all_indices.extend([i] * count)
+    if apid in ULTRA_EVENTS.apid:
+        field_ranges = EVENT_FIELD_RANGES
+    elif apid in ULTRA_ENERGY_EVENTS.apid:
+        field_ranges = ENERGY_EVENT_FIELD_RANGES
+    else:
+        logger.error(f"APID {apid} not recognized for Ultra events processing.")
 
-    # Now we have the event data, we need to create the xarray dataset.
-    # We cannot append to the existing dataset (sorted_packets)
-    # because there are multiple events for each epoch.
-    idx = np.array(all_indices)
-
-    # Expand the existing dataset so that it is the same length as the event data.
-    expanded_data = {
-        var: ds[var].values[idx] for var in ds.data_vars if var != "eventdata"
-    }
-
-    # Add the event data to the expanded dataset.
-    for key in EVENT_FIELD_RANGES:
-        expanded_data[key] = np.array([event[key] for event in all_events])
-
-    event_ids = get_event_id(expanded_data["shcoarse"])
-
-    coords = {
-        "epoch": ds["epoch"].values[idx],
-        "event_id": ("epoch", event_ids),
-    }
-
-    dataset = xr.Dataset(coords=coords)
-    for key, data in expanded_data.items():
-        dataset[key] = xr.DataArray(
-            data,
-            dims=["epoch"],
-        )
-
-    return dataset
-
-
-def process_ultra_energy_events(ds: xr.Dataset) -> xr.Dataset:
-    """
-    Unpack and decode Ultra ENERGY EVENTS packets.
-
-    Parameters
-    ----------
-    ds : xarray.Dataset
-        Events dataset.
-
-    Returns
-    -------
-    ds : xarray.Dataset
-        Dataset containing the decoded and decompressed data.
-    """
     all_events = []
     all_indices = []
 
@@ -234,7 +172,7 @@ def process_ultra_energy_events(ds: xr.Dataset) -> xr.Dataset:
         field: attrs.get_variable_attributes(field).get(
             "FILLVAL", np.iinfo(np.int64).min
         )
-        for field in ENERGY_EVENT_FIELD_RANGES
+        for field in field_ranges
     }
 
     counts = ds["count"].values
@@ -247,7 +185,9 @@ def process_ultra_energy_events(ds: xr.Dataset) -> xr.Dataset:
         else:
             # Here there are multiple images in a single packet,
             # so we need to loop through each image and decompress it.
-            event_data_list = read_image_raw_events_binary(eventdata_array[i], count)
+            event_data_list = read_image_raw_events_binary(
+                eventdata_array[i], count, field_ranges
+            )
             all_events.extend(event_data_list)
             # Keep track of how many times does the event occurred at this epoch.
             all_indices.extend([i] * count)
@@ -263,7 +203,7 @@ def process_ultra_energy_events(ds: xr.Dataset) -> xr.Dataset:
     }
 
     # Add the event data to the expanded dataset.
-    for key in EVENT_FIELD_RANGES:
+    for key in field_ranges:
         expanded_data[key] = np.array([event[key] for event in all_events])
 
     event_ids = get_event_id(expanded_data["shcoarse"])

imap_processing/ultra/l0/ultra_utils.py

@@ -53,7 +53,7 @@ class PacketProperties(NamedTuple):
     mantissa_bit_length=4,
 )
 ULTRA_EVENTS = PacketProperties(
-    apid=[896, 960],
+    apid=[898, 960],
     logical_source=["imap_ultra_l1a_45sensor-de", "imap_ultra_l1a_90sensor-de"],
     addition_to_logical_desc="Single Events",
     width=None,
@@ -223,11 +223,11 @@ class PacketProperties(NamedTuple):
     # Stop Type
     "stop_type": (0, 4),
     # Energy/Pulse Height
-    "energy_ph": (5, 17),
+    "energy_ph": (4, 16),
     # Pulse Width
-    "pulse_width": (18, 29),
+    "pulse_width": (16, 27),
     # Bin
-    "bin": (30, 34),
+    "bin": (27, 33),
 }
 
 
@@ -348,22 +348,24 @@ class PacketProperties(NamedTuple):
 ]
 
 
-def parse_event(event_binary: str) -> dict:
+def parse_event(event_binary: str, field_ranges: dict) -> dict:
     """
     Parse a binary string representing a single event.
 
     Parameters
     ----------
     event_binary : str
         Event binary string.
+    field_ranges : dict
+        The field ranges for the event data.
 
     Returns
     -------
     fields_dict : dict
         Dict of the fields for a single event.
     """
     fields_dict = {}
-    for field, (start, end) in EVENT_FIELD_RANGES.items():
+    for field, (start, end) in field_ranges.items():
         field_value = int(event_binary[start:end], 2)
         fields_dict[field] = field_value
     return fields_dict

imap_processing/ultra/l1a/ultra_l1a.py

@@ -8,7 +8,6 @@
 from imap_processing import imap_module_directory
 from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
 from imap_processing.ultra.l0.decom_ultra import (
-    process_ultra_energy_events,
     process_ultra_events,
     process_ultra_rates,
     process_ultra_tof,
@@ -78,13 +77,13 @@ def ultra_l1a(packet_file: str, apid_input: Optional[int] = None) -> list[xr.Dat
             decom_ultra_dataset = decom_ultra_dataset.drop_vars("fastdata_00")
             gattr_key = ULTRA_RATES.logical_source[ULTRA_RATES.apid.index(apid)]
         elif apid in ULTRA_EVENTS.apid:
-            decom_ultra_dataset = process_ultra_events(datasets_by_apid[apid])
+            decom_ultra_dataset = process_ultra_events(datasets_by_apid[apid], apid)
             gattr_key = ULTRA_EVENTS.logical_source[ULTRA_EVENTS.apid.index(apid)]
             # Add coordinate attributes
             attrs = attr_mgr.get_variable_attributes("event_id")
             decom_ultra_dataset.coords["event_id"].attrs.update(attrs)
         elif apid in ULTRA_ENERGY_EVENTS.apid:
-            decom_ultra_dataset = process_ultra_energy_events(datasets_by_apid[apid])
+            decom_ultra_dataset = process_ultra_events(datasets_by_apid[apid], apid)
             gattr_key = ULTRA_ENERGY_EVENTS.logical_source[
                 ULTRA_ENERGY_EVENTS.apid.index(apid)
             ]