From 843e98b175016c746eafe4ab060760af41a1c127 Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Wed, 9 Jul 2025 10:08:35 -0600 Subject: [PATCH 1/5] squash --- imap_processing/codice/codice_l1a.py | 49 ++++++++++++---------------- imap_processing/codice/constants.py | 22 ++++++------- 2 files changed, 30 insertions(+), 41 deletions(-) diff --git a/imap_processing/codice/codice_l1a.py b/imap_processing/codice/codice_l1a.py index 7e19b31a8d..6cbae1f88e 100644 --- a/imap_processing/codice/codice_l1a.py +++ b/imap_processing/codice/codice_l1a.py @@ -143,8 +143,8 @@ def define_coordinates(self) -> None: self.coords = {} coord_names = [ - *self.config["output_dims"].keys(), - *[key + "_label" for key in self.config["output_dims"].keys()], + *self.config["input_dims"].keys(), + *[key + "_label" for key in self.config["input_dims"].keys()], ] # Define epoch coordinates @@ -174,7 +174,7 @@ def define_coordinates(self) -> None: "spin_sector_index", "ssd_index", ]: - values = np.arange(self.config["output_dims"][name]) + values = np.arange(self.config["input_dims"][name]) dims = [name] elif name == "spin_sector_pairs_label": values = np.array( @@ -196,7 +196,7 @@ def define_coordinates(self) -> None: "ssd_index_label", ]: key = name.removesuffix("_label") - values = np.arange(self.config["output_dims"][key]).astype(str) + values = np.arange(self.config["input_dims"][key]).astype(str) dims = [key] coord = xr.DataArray( @@ -230,8 +230,8 @@ def define_data_variables(self) -> xr.Dataset: # Stack the data so that it is easier to reshape and iterate over all_data = np.stack(self.data) - # The dimension of all_data is something like (epoch, num_counters, - # num_energy_steps, num_positions, num_spin_sectors) (or may be slightly + # The dimension of all_data is something like (epoch, num_energy_steps, + # num_positions, num_spin_sectors, num_counters) (or may be slightly # different depending on the data product). In any case, iterate over # the num_counters dimension to isolate the data for each counter so # each counter's data can be placed in a separate CDF data variable. @@ -239,7 +239,7 @@ def define_data_variables(self) -> xr.Dataset: range(all_data.shape[1]), self.config["variable_names"] ): # Extract the counter data - counter_data = all_data[:, counter, ...] + counter_data = all_data[..., counter] # Get the CDF attributes descriptor = self.config["dataset_name"].split("imap_codice_l1a_")[-1] @@ -249,7 +249,9 @@ def define_data_variables(self) -> xr.Dataset: # For most products, the final CDF dimensions always has "epoch" as # the first dimension followed by the dimensions for the specific # data product - dims = ["epoch", *list(self.config["output_dims"].keys())] + dims = ["epoch", *list(self.config["input_dims"].keys())] + print(descriptor) + print(dims) # However, CoDICE-Hi products use specific energy bins for the # energy dimension @@ -589,8 +591,8 @@ def reshape_data(self) -> None: These data need to be divided up by species or priorities (or what I am calling "counters" as a general term), and re-arranged into - 4D arrays representing dimensions such as time, spin sectors, positions, - and energies (depending on the data product). + multidimensional arrays representing dimensions such as time, + spin sectors, positions, and energies (depending on the data product). However, the existence and order of these dimensions can vary depending on the specific data product, so we define this in the "input_dims" @@ -603,30 +605,19 @@ def reshape_data(self) -> None: self.data = [] # First reshape the data based on how it is written to the data array of - # the packet data. The number of counters is the first dimension / axis, - # with the exception of lo-counters-aggregated which is treated slightly - # differently - if self.config["dataset_name"] != "imap_codice_l1a_lo-counters-aggregated": - reshape_dims = ( - self.config["num_counters"], - *self.config["input_dims"].values(), - ) - else: - reshape_dims = ( - *self.config["input_dims"].values(), - self.config["num_counters"], - ) + # the packet data. The number of counters is the last dimension / axis. + reshape_dims = ( + *self.config["input_dims"].values(), + self.config["num_counters"], + ) # Then, transpose the data based on how the dimensions should be written # to the CDF file. Since this is specific to each data product, we need # to determine this dynamically based on the "output_dims" config. # Again, lo-counters-aggregated is treated slightly differently input_keys = ["num_counters", *self.config["input_dims"].keys()] - output_keys = ["num_counters", *self.config["output_dims"].keys()] - if self.config["dataset_name"] != "imap_codice_l1a_lo-counters-aggregated": - transpose_axes = [input_keys.index(dim) for dim in output_keys] - else: - transpose_axes = [1, 2, 0] # [esa_step, spin_sector_pairs, num_counters] + output_keys = ["num_counters", *self.config["input_dims"].keys()] + transpose_axes = [input_keys.index(dim) for dim in output_keys] for packet_data in self.raw_data: reshaped_packet_data = np.array(packet_data, dtype=np.uint32).reshape( @@ -1540,7 +1531,7 @@ def process_codice_l1a(file_path: Path) -> list[xr.Dataset]: elif apid in [CODICEAPID.COD_LO_IAL, CODICEAPID.COD_HI_IAL]: processed_dataset = create_ialirt_dataset(apid, dataset) logger.info(f"\nFinal data product:\n{processed_dataset}\n") - + # # hi-omni data elif apid == CODICEAPID.COD_HI_OMNI_SPECIES_COUNTS: science_values = [packet.data for packet in dataset.data] diff --git a/imap_processing/codice/constants.py b/imap_processing/codice/constants.py index 6577b1d81d..856535010a 100644 --- a/imap_processing/codice/constants.py +++ b/imap_processing/codice/constants.py @@ -405,8 +405,6 @@ "fe": [0.0125, 0.025, 0.05, 0.1, 0.2, 0.4, 0.8, 1.6, 3.2], } -# TODO: Possibly move to consistent order of dimensions with other instruments -# TBD after discussion with Joey and at the Science Team Meeting in Feb # Various configurations to support processing of individual data products # Much of these are described in the algorithm document in chapter 10 ("Data # Level 1A") @@ -510,10 +508,10 @@ }, CODICEAPID.COD_LO_IAL: { "dataset_name": "imap_codice_l1a_lo-ialirt", - "input_dims": {"spin_sector": 1, "esa_step": 128}, + "input_dims": {"esa_step": 128, "spin_sector": 1}, "instrument": "lo", "num_counters": 9, - "output_dims": {"spin_sector": 1, "esa_step": 128}, + "output_dims": {"esa_step": 128, "spin_sector": 1}, "support_variables": [ "energy_table", "acquisition_time_per_step", @@ -533,7 +531,7 @@ "num_counters": len( LO_COUNTERS_AGGREGATED_VARIABLE_NAMES ), # The number of counters depends on the number of active counters - "output_dims": {"spin_sector_pairs": 6, "esa_step": 128}, + "output_dims": {"esa_step": 128, "spin_sector_pairs": 6}, "support_variables": [ "energy_table", "acquisition_time_per_step", @@ -551,7 +549,7 @@ "input_dims": {"esa_step": 128, "inst_az": 24, "spin_sector_pairs": 6}, "instrument": "lo", "num_counters": 1, - "output_dims": {"inst_az": 24, "spin_sector_pairs": 6, "esa_step": 128}, + "output_dims": {"esa_step": 128, "inst_az": 24, "spin_sector_pairs": 6}, "support_variables": [ "energy_table", "acquisition_time_per_step", @@ -569,7 +567,7 @@ "input_dims": {"esa_step": 128, "inst_az": 5, "spin_sector": 12}, "instrument": "lo", "num_counters": 4, - "output_dims": {"inst_az": 5, "spin_sector": 12, "esa_step": 128}, + "output_dims": {"esa_step": 128, "inst_az": 5, "spin_sector": 12}, "support_variables": [ "energy_table", "acquisition_time_per_step", @@ -587,7 +585,7 @@ "input_dims": {"esa_step": 128, "inst_az": 19, "spin_sector": 12}, "instrument": "lo", "num_counters": 1, - "output_dims": {"inst_az": 19, "spin_sector": 12, "esa_step": 128}, + "output_dims": {"esa_step": 128, "inst_az": 19, "spin_sector": 12}, "support_variables": [ "energy_table", "acquisition_time_per_step", @@ -605,7 +603,7 @@ "input_dims": {"esa_step": 128, "spin_sector": 12}, "instrument": "lo", "num_counters": 5, - "output_dims": {"spin_sector": 12, "esa_step": 128}, + "output_dims": {"esa_step": 128, "spin_sector": 12}, "support_variables": [ "energy_table", "acquisition_time_per_step", @@ -623,7 +621,7 @@ "input_dims": {"esa_step": 128, "spin_sector": 12}, "instrument": "lo", "num_counters": 2, - "output_dims": {"spin_sector": 12, "esa_step": 128}, + "output_dims": {"esa_step": 128, "spin_sector": 12}, "support_variables": [ "energy_table", "acquisition_time_per_step", @@ -641,7 +639,7 @@ "input_dims": {"esa_step": 128, "spin_sector": 1}, "instrument": "lo", "num_counters": 16, - "output_dims": {"spin_sector": 1, "esa_step": 128}, + "output_dims": {"esa_step": 128, "spin_sector": 1}, "support_variables": [ "energy_table", "acquisition_time_per_step", @@ -659,7 +657,7 @@ "input_dims": {"esa_step": 128, "spin_sector": 1}, "instrument": "lo", "num_counters": 8, - "output_dims": {"spin_sector": 1, "esa_step": 128}, + "output_dims": {"esa_step": 128, "spin_sector": 1}, "support_variables": [ "energy_table", "acquisition_time_per_step", From d9285c05edd23afaa5bedeaa5df18dea6e03715c Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Wed, 9 Jul 2025 10:40:20 -0600 Subject: [PATCH 2/5] squash --- imap_processing/codice/codice_l1a.py | 18 +++---- imap_processing/codice/constants.py | 51 ++++++------------- .../tests/codice/test_codice_l1a.py | 18 +++---- 3 files changed, 32 insertions(+), 55 deletions(-) diff --git a/imap_processing/codice/codice_l1a.py b/imap_processing/codice/codice_l1a.py index 6cbae1f88e..228c061781 100644 --- a/imap_processing/codice/codice_l1a.py +++ b/imap_processing/codice/codice_l1a.py @@ -143,8 +143,8 @@ def define_coordinates(self) -> None: self.coords = {} coord_names = [ - *self.config["input_dims"].keys(), - *[key + "_label" for key in self.config["input_dims"].keys()], + *self.config["dims"].keys(), + *[key + "_label" for key in self.config["dims"].keys()], ] # Define epoch coordinates @@ -174,7 +174,7 @@ def define_coordinates(self) -> None: "spin_sector_index", "ssd_index", ]: - values = np.arange(self.config["input_dims"][name]) + values = np.arange(self.config["dims"][name]) dims = [name] elif name == "spin_sector_pairs_label": values = np.array( @@ -196,7 +196,7 @@ def define_coordinates(self) -> None: "ssd_index_label", ]: key = name.removesuffix("_label") - values = np.arange(self.config["input_dims"][key]).astype(str) + values = np.arange(self.config["dims"][key]).astype(str) dims = [key] coord = xr.DataArray( @@ -249,9 +249,7 @@ def define_data_variables(self) -> xr.Dataset: # For most products, the final CDF dimensions always has "epoch" as # the first dimension followed by the dimensions for the specific # data product - dims = ["epoch", *list(self.config["input_dims"].keys())] - print(descriptor) - print(dims) + dims = ["epoch", *list(self.config["dims"].keys())] # However, CoDICE-Hi products use specific energy bins for the # energy dimension @@ -607,7 +605,7 @@ def reshape_data(self) -> None: # First reshape the data based on how it is written to the data array of # the packet data. The number of counters is the last dimension / axis. reshape_dims = ( - *self.config["input_dims"].values(), + *self.config["dims"].values(), self.config["num_counters"], ) @@ -615,8 +613,8 @@ def reshape_data(self) -> None: # to the CDF file. Since this is specific to each data product, we need # to determine this dynamically based on the "output_dims" config. # Again, lo-counters-aggregated is treated slightly differently - input_keys = ["num_counters", *self.config["input_dims"].keys()] - output_keys = ["num_counters", *self.config["input_dims"].keys()] + input_keys = ["num_counters", *self.config["dims"].keys()] + output_keys = ["num_counters", *self.config["dims"].keys()] transpose_axes = [input_keys.index(dim) for dim in output_keys] for packet_data in self.raw_data: diff --git a/imap_processing/codice/constants.py b/imap_processing/codice/constants.py index 856535010a..66e41b24ba 100644 --- a/imap_processing/codice/constants.py +++ b/imap_processing/codice/constants.py @@ -412,11 +412,10 @@ CODICEAPID.COD_HI_IAL: { "dataset_name": "imap_codice_l1a_hi-ialirt", "energy_table": IALIRT_ENERGY_TABLE, - "input_dims": {"esa_step": 15, "inst_az": 4}, + "dims": {"esa_step": 15, "inst_az": 4}, "instrument": "hi", "num_counters": 1, "num_spins": 4, - "output_dims": {"esa_step": 15, "inst_az": 4}, "support_variables": [ "data_quality", "spin_period", @@ -426,45 +425,39 @@ }, CODICEAPID.COD_HI_INST_COUNTS_AGGREGATED: { "dataset_name": "imap_codice_l1a_hi-counters-aggregated", - "input_dims": {}, + "dims": {}, "instrument": "hi", "num_counters": len( HI_COUNTERS_AGGREGATED_VARIABLE_NAMES ), # The number of counters depends on the number of active counters - "output_dims": {}, "support_variables": ["data_quality", "spin_period"], "variable_names": HI_COUNTERS_AGGREGATED_VARIABLE_NAMES, }, CODICEAPID.COD_HI_INST_COUNTS_SINGLES: { "dataset_name": "imap_codice_l1a_hi-counters-singles", - "input_dims": { + "dims": { "ssd_index": 12, }, "instrument": "hi", "num_counters": 3, - "output_dims": { - "ssd_index": 12, - }, "support_variables": ["data_quality", "spin_period"], "variable_names": HI_COUNTERS_SINGLES_VARIABLE_NAMES, }, CODICEAPID.COD_HI_INST_COUNTS_PRIORITIES: { "dataset_name": "imap_codice_l1a_hi-priority", - "input_dims": {}, + "dims": {}, "instrument": "hi", "num_counters": 6, - "output_dims": {}, "support_variables": ["data_quality", "spin_period"], "variable_names": HI_PRIORITY_VARIABLE_NAMES, }, CODICEAPID.COD_HI_OMNI_SPECIES_COUNTS: { "dataset_name": "imap_codice_l1a_hi-omni", "energy_table": OMNI_ENERGY_TABLE, - "input_dims": {"esa_step": 15, "inst_az": 4}, + "dims": {"esa_step": 15, "inst_az": 4}, "instrument": "hi", "num_counters": 8, "num_spins": 4, - "output_dims": {"esa_step": 15, "inst_az": 4}, "support_variables": [ "data_quality", "spin_period", @@ -483,7 +476,7 @@ CODICEAPID.COD_HI_SECT_SPECIES_COUNTS: { "dataset_name": "imap_codice_l1a_hi-sectored", "energy_table": SECTORED_ENERGY_TABLE, - "input_dims": { + "dims": { "esa_step": 8, "ssd_index": 12, "spin_sector_index": 12, @@ -491,11 +484,6 @@ "instrument": "hi", "num_counters": 4, "num_spins": 16, - "output_dims": { - "esa_step": 8, - "ssd_index": 12, - "spin_sector_index": 12, - }, "support_variables": [ "data_quality", "spin_period", @@ -508,10 +496,9 @@ }, CODICEAPID.COD_LO_IAL: { "dataset_name": "imap_codice_l1a_lo-ialirt", - "input_dims": {"esa_step": 128, "spin_sector": 1}, + "dims": {"esa_step": 128, "spin_sector": 1}, "instrument": "lo", "num_counters": 9, - "output_dims": {"esa_step": 128, "spin_sector": 1}, "support_variables": [ "energy_table", "acquisition_time_per_step", @@ -526,12 +513,11 @@ }, CODICEAPID.COD_LO_INST_COUNTS_AGGREGATED: { "dataset_name": "imap_codice_l1a_lo-counters-aggregated", - "input_dims": {"esa_step": 128, "spin_sector_pairs": 6}, + "dims": {"esa_step": 128, "spin_sector_pairs": 6}, "instrument": "lo", "num_counters": len( LO_COUNTERS_AGGREGATED_VARIABLE_NAMES ), # The number of counters depends on the number of active counters - "output_dims": {"esa_step": 128, "spin_sector_pairs": 6}, "support_variables": [ "energy_table", "acquisition_time_per_step", @@ -546,10 +532,9 @@ }, CODICEAPID.COD_LO_INST_COUNTS_SINGLES: { "dataset_name": "imap_codice_l1a_lo-counters-singles", - "input_dims": {"esa_step": 128, "inst_az": 24, "spin_sector_pairs": 6}, + "dims": {"esa_step": 128, "inst_az": 24, "spin_sector_pairs": 6}, "instrument": "lo", "num_counters": 1, - "output_dims": {"esa_step": 128, "inst_az": 24, "spin_sector_pairs": 6}, "support_variables": [ "energy_table", "acquisition_time_per_step", @@ -564,10 +549,9 @@ }, CODICEAPID.COD_LO_SW_ANGULAR_COUNTS: { "dataset_name": "imap_codice_l1a_lo-sw-angular", - "input_dims": {"esa_step": 128, "inst_az": 5, "spin_sector": 12}, + "dims": {"esa_step": 128, "inst_az": 5, "spin_sector": 12}, "instrument": "lo", "num_counters": 4, - "output_dims": {"esa_step": 128, "inst_az": 5, "spin_sector": 12}, "support_variables": [ "energy_table", "acquisition_time_per_step", @@ -582,10 +566,9 @@ }, CODICEAPID.COD_LO_NSW_ANGULAR_COUNTS: { "dataset_name": "imap_codice_l1a_lo-nsw-angular", - "input_dims": {"esa_step": 128, "inst_az": 19, "spin_sector": 12}, + "dims": {"esa_step": 128, "inst_az": 19, "spin_sector": 12}, "instrument": "lo", "num_counters": 1, - "output_dims": {"esa_step": 128, "inst_az": 19, "spin_sector": 12}, "support_variables": [ "energy_table", "acquisition_time_per_step", @@ -600,10 +583,9 @@ }, CODICEAPID.COD_LO_SW_PRIORITY_COUNTS: { "dataset_name": "imap_codice_l1a_lo-sw-priority", - "input_dims": {"esa_step": 128, "spin_sector": 12}, + "dims": {"esa_step": 128, "spin_sector": 12}, "instrument": "lo", "num_counters": 5, - "output_dims": {"esa_step": 128, "spin_sector": 12}, "support_variables": [ "energy_table", "acquisition_time_per_step", @@ -618,10 +600,9 @@ }, CODICEAPID.COD_LO_NSW_PRIORITY_COUNTS: { "dataset_name": "imap_codice_l1a_lo-nsw-priority", - "input_dims": {"esa_step": 128, "spin_sector": 12}, + "dims": {"esa_step": 128, "spin_sector": 12}, "instrument": "lo", "num_counters": 2, - "output_dims": {"esa_step": 128, "spin_sector": 12}, "support_variables": [ "energy_table", "acquisition_time_per_step", @@ -636,10 +617,9 @@ }, CODICEAPID.COD_LO_SW_SPECIES_COUNTS: { "dataset_name": "imap_codice_l1a_lo-sw-species", - "input_dims": {"esa_step": 128, "spin_sector": 1}, + "dims": {"esa_step": 128, "spin_sector": 1}, "instrument": "lo", "num_counters": 16, - "output_dims": {"esa_step": 128, "spin_sector": 1}, "support_variables": [ "energy_table", "acquisition_time_per_step", @@ -654,10 +634,9 @@ }, CODICEAPID.COD_LO_NSW_SPECIES_COUNTS: { "dataset_name": "imap_codice_l1a_lo-nsw-species", - "input_dims": {"esa_step": 128, "spin_sector": 1}, + "dims": {"esa_step": 128, "spin_sector": 1}, "instrument": "lo", "num_counters": 8, - "output_dims": {"esa_step": 128, "spin_sector": 1}, "support_variables": [ "energy_table", "acquisition_time_per_step", diff --git a/imap_processing/tests/codice/test_codice_l1a.py b/imap_processing/tests/codice/test_codice_l1a.py index 825fa0b8ed..e19c1b6c20 100644 --- a/imap_processing/tests/codice/test_codice_l1a.py +++ b/imap_processing/tests/codice/test_codice_l1a.py @@ -41,16 +41,16 @@ EXPECTED_ARRAY_SHAPES = [ (304, 15), # hi-ialirt - (76, 1, 128), # lo-ialirt + (76, 128, 1), # lo-ialirt (31778,), # hskp - (77, 6, 128), # lo-counters-aggregated - (77, 24, 6, 128), # lo-counters-singles - (77, 12, 128), # lo-sw-priority - (77, 12, 128), # lo-nsw-priority - (77, 1, 128), # lo-sw-species - (77, 1, 128), # lo-nsw-species - (77, 5, 12, 128), # lo-sw-angular - (77, 19, 12, 128), # lo-nsw-angular + (77, 128, 6), # lo-counters-aggregated + (77, 128, 24, 6), # lo-counters-singles + (77, 128, 12), # lo-sw-priority + (77, 128, 12), # lo-nsw-priority + (77, 128, 1), # lo-sw-species + (77, 128, 1), # lo-nsw-species + (77, 128, 5, 12), # lo-sw-angular + (77, 128, 19, 12), # lo-nsw-angular (77,), # hi-counters-aggregated (77, 12), # hi-counters-singles (), # hi-omni, shapes are specific to species From f87f92dd367b3c449584959c3e4f971517a1a9a7 Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Wed, 9 Jul 2025 11:18:07 -0600 Subject: [PATCH 3/5] simplified code for reshaping data --- imap_processing/codice/codice_l1a.py | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/imap_processing/codice/codice_l1a.py b/imap_processing/codice/codice_l1a.py index f4a652d107..2d50c20b14 100644 --- a/imap_processing/codice/codice_l1a.py +++ b/imap_processing/codice/codice_l1a.py @@ -593,37 +593,23 @@ def reshape_data(self) -> None: spin sectors, positions, and energies (depending on the data product). However, the existence and order of these dimensions can vary depending - on the specific data product, so we define this in the "input_dims" - and "output_dims" values configuration dictionary; the "input_dims" - defines how the dimensions are written into the packet data, while - "output_dims" defines how the dimensions should be written to the final - CDF product. + on the specific data product, so we define this in the "dims" key of the + configuration dictionary. """ # This will contain the reshaped data for all counters self.data = [] - # First reshape the data based on how it is written to the data array of + # Reshape the data based on how it is written to the data array of # the packet data. The number of counters is the last dimension / axis. reshape_dims = ( *self.config["dims"].values(), self.config["num_counters"], ) - - # Then, transpose the data based on how the dimensions should be written - # to the CDF file. Since this is specific to each data product, we need - # to determine this dynamically based on the "output_dims" config. - # Again, lo-counters-aggregated is treated slightly differently - input_keys = ["num_counters", *self.config["dims"].keys()] - output_keys = ["num_counters", *self.config["dims"].keys()] - transpose_axes = [input_keys.index(dim) for dim in output_keys] - for packet_data in self.raw_data: reshaped_packet_data = np.array(packet_data, dtype=np.uint32).reshape( reshape_dims ) - reshaped_cdf_data = np.transpose(reshaped_packet_data, axes=transpose_axes) - - self.data.append(reshaped_cdf_data) + self.data.append(reshaped_packet_data) # No longer need to keep the raw data around del self.raw_data From 137507612d5169cfa9edf279531ce3511a52e130 Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Wed, 9 Jul 2025 12:53:27 -0600 Subject: [PATCH 4/5] squash --- imap_processing/codice/codice_l1a.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imap_processing/codice/codice_l1a.py b/imap_processing/codice/codice_l1a.py index 2d50c20b14..9e3f331c62 100644 --- a/imap_processing/codice/codice_l1a.py +++ b/imap_processing/codice/codice_l1a.py @@ -1515,7 +1515,7 @@ def process_codice_l1a(file_path: Path) -> list[xr.Dataset]: elif apid in [CODICEAPID.COD_LO_IAL, CODICEAPID.COD_HI_IAL]: processed_dataset = create_ialirt_dataset(apid, dataset) logger.info(f"\nFinal data product:\n{processed_dataset}\n") - # + # hi-omni data elif apid == CODICEAPID.COD_HI_OMNI_SPECIES_COUNTS: science_values = [packet.data for packet in dataset.data] From 551a5cce02d18834b2589ae16a5207f3658c4cfe Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Wed, 9 Jul 2025 15:51:05 -0600 Subject: [PATCH 5/5] Added test for testing dimension order --- imap_processing/codice/codice_l1a.py | 2 +- .../tests/codice/test_codice_l1a.py | 50 +++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/imap_processing/codice/codice_l1a.py b/imap_processing/codice/codice_l1a.py index 9e3f331c62..f2c48423bb 100644 --- a/imap_processing/codice/codice_l1a.py +++ b/imap_processing/codice/codice_l1a.py @@ -236,7 +236,7 @@ def define_data_variables(self) -> xr.Dataset: # the num_counters dimension to isolate the data for each counter so # each counter's data can be placed in a separate CDF data variable. for counter, variable_name in zip( - range(all_data.shape[1]), self.config["variable_names"] + range(all_data.shape[-1]), self.config["variable_names"] ): # Extract the counter data counter_data = all_data[..., counter] diff --git a/imap_processing/tests/codice/test_codice_l1a.py b/imap_processing/tests/codice/test_codice_l1a.py index c8f787154b..7bb40bbb7d 100644 --- a/imap_processing/tests/codice/test_codice_l1a.py +++ b/imap_processing/tests/codice/test_codice_l1a.py @@ -259,6 +259,56 @@ def test_l1a_validate_data_arrays(test_l1a_data: xr.Dataset, index): ) +@pytest.mark.parametrize("index", range(len(DESCRIPTORS))) +def test_l1a_validate_dimensions(test_l1a_data, index): + """Tests that the dimensions of the data are in the expected order. + + Parameters + ---------- + test_l1a_data : list[xarray.Dataset] + A list of ``xarray`` datasets containing the test data + index : int + The index of the list to test + """ + + descriptor = DESCRIPTORS[index] + dataset = test_l1a_data[index] + + # This is the expected order of dimensions. Not all of these appear in every + # data product, but for those that do appear, they should be in this order. + expected_dims_order = [ + "epoch", + "esa_step", + "inst_az", + "spin_sector", + "spin_sector_pairs", + "ssd_index", + ] + + # We don't need to check hskp, direct events, or binned datasets since they + # are not multidimensional + if descriptor not in [ + "hskp", + "lo-direct-events", + "hi-direct-events", + "hi-omni", + "hi-ialirt", + "hi-sectored", + ]: + # Get the variables that have dimensions that need to be checked + counters = getattr( + constants, f"{descriptor.upper().replace('-', '_')}_VARIABLE_NAMES" + ) + + # Ensure that, of the dimensions in the particular variable, they occur + # in the expected order. + for counter in counters: + positions = [ + expected_dims_order.index(dim) for dim in dataset[counter].dims + ] + assert positions == sorted(positions) + + @pytest.mark.parametrize("index", range(len(DESCRIPTORS))) @pytest.mark.xfail(reason="Validation test turned off; awaiting fixes") def test_l1a_validate_epoch_values(test_l1a_data, index):