diff --git a/imap_processing/codice/codice_l1a.py b/imap_processing/codice/codice_l1a.py index 23e719480a..f2c48423bb 100644 --- a/imap_processing/codice/codice_l1a.py +++ b/imap_processing/codice/codice_l1a.py @@ -143,8 +143,8 @@ def define_coordinates(self) -> None: self.coords = {} coord_names = [ - *self.config["output_dims"].keys(), - *[key + "_label" for key in self.config["output_dims"].keys()], + *self.config["dims"].keys(), + *[key + "_label" for key in self.config["dims"].keys()], ] # Define epoch coordinates @@ -174,7 +174,7 @@ def define_coordinates(self) -> None: "spin_sector_index", "ssd_index", ]: - values = np.arange(self.config["output_dims"][name]) + values = np.arange(self.config["dims"][name]) dims = [name] elif name == "spin_sector_pairs_label": values = np.array( @@ -196,7 +196,7 @@ def define_coordinates(self) -> None: "ssd_index_label", ]: key = name.removesuffix("_label") - values = np.arange(self.config["output_dims"][key]).astype(str) + values = np.arange(self.config["dims"][key]).astype(str) dims = [key] coord = xr.DataArray( @@ -230,16 +230,16 @@ def define_data_variables(self) -> xr.Dataset: # Stack the data so that it is easier to reshape and iterate over all_data = np.stack(self.data) - # The dimension of all_data is something like (epoch, num_counters, - # num_energy_steps, num_positions, num_spin_sectors) (or may be slightly + # The dimension of all_data is something like (epoch, num_energy_steps, + # num_positions, num_spin_sectors, num_counters) (or may be slightly # different depending on the data product). In any case, iterate over # the num_counters dimension to isolate the data for each counter so # each counter's data can be placed in a separate CDF data variable. for counter, variable_name in zip( - range(all_data.shape[1]), self.config["variable_names"] + range(all_data.shape[-1]), self.config["variable_names"] ): # Extract the counter data - counter_data = all_data[:, counter, ...] + counter_data = all_data[..., counter] # Get the CDF attributes descriptor = self.config["dataset_name"].split("imap_codice_l1a_")[-1] @@ -249,7 +249,7 @@ def define_data_variables(self) -> xr.Dataset: # For most products, the final CDF dimensions always has "epoch" as # the first dimension followed by the dimensions for the specific # data product - dims = ["epoch", *list(self.config["output_dims"].keys())] + dims = ["epoch", *list(self.config["dims"].keys())] # However, CoDICE-Hi products use specific energy bins for the # energy dimension @@ -589,52 +589,27 @@ def reshape_data(self) -> None: These data need to be divided up by species or priorities (or what I am calling "counters" as a general term), and re-arranged into - 4D arrays representing dimensions such as time, spin sectors, positions, - and energies (depending on the data product). + multidimensional arrays representing dimensions such as time, + spin sectors, positions, and energies (depending on the data product). However, the existence and order of these dimensions can vary depending - on the specific data product, so we define this in the "input_dims" - and "output_dims" values configuration dictionary; the "input_dims" - defines how the dimensions are written into the packet data, while - "output_dims" defines how the dimensions should be written to the final - CDF product. + on the specific data product, so we define this in the "dims" key of the + configuration dictionary. """ # This will contain the reshaped data for all counters self.data = [] - # First reshape the data based on how it is written to the data array of - # the packet data. The number of counters is the first dimension / axis, - # with the exception of lo-counters-aggregated which is treated slightly - # differently - if self.config["dataset_name"] != "imap_codice_l1a_lo-counters-aggregated": - reshape_dims = ( - self.config["num_counters"], - *self.config["input_dims"].values(), - ) - else: - reshape_dims = ( - *self.config["input_dims"].values(), - self.config["num_counters"], - ) - - # Then, transpose the data based on how the dimensions should be written - # to the CDF file. Since this is specific to each data product, we need - # to determine this dynamically based on the "output_dims" config. - # Again, lo-counters-aggregated is treated slightly differently - input_keys = ["num_counters", *self.config["input_dims"].keys()] - output_keys = ["num_counters", *self.config["output_dims"].keys()] - if self.config["dataset_name"] != "imap_codice_l1a_lo-counters-aggregated": - transpose_axes = [input_keys.index(dim) for dim in output_keys] - else: - transpose_axes = [1, 2, 0] # [esa_step, spin_sector_pairs, num_counters] - + # Reshape the data based on how it is written to the data array of + # the packet data. The number of counters is the last dimension / axis. + reshape_dims = ( + *self.config["dims"].values(), + self.config["num_counters"], + ) for packet_data in self.raw_data: reshaped_packet_data = np.array(packet_data, dtype=np.uint32).reshape( reshape_dims ) - reshaped_cdf_data = np.transpose(reshaped_packet_data, axes=transpose_axes) - - self.data.append(reshaped_cdf_data) + self.data.append(reshaped_packet_data) # No longer need to keep the raw data around del self.raw_data diff --git a/imap_processing/codice/constants.py b/imap_processing/codice/constants.py index 8752456045..9e629e007c 100644 --- a/imap_processing/codice/constants.py +++ b/imap_processing/codice/constants.py @@ -401,8 +401,6 @@ "fe": [0.0125, 0.025, 0.05, 0.1, 0.2, 0.4, 0.8, 1.6, 3.2], } -# TODO: Possibly move to consistent order of dimensions with other instruments -# TBD after discussion with Joey and at the Science Team Meeting in Feb # Various configurations to support processing of individual data products # Much of these are described in the algorithm document in chapter 10 ("Data # Level 1A") @@ -410,11 +408,10 @@ CODICEAPID.COD_HI_IAL: { "dataset_name": "imap_codice_l1a_hi-ialirt", "energy_table": IALIRT_ENERGY_TABLE, - "input_dims": {"esa_step": 15, "inst_az": 4}, + "dims": {"esa_step": 15, "inst_az": 4}, "instrument": "hi", "num_counters": 1, "num_spins": 4, - "output_dims": {"esa_step": 15, "inst_az": 4}, "support_variables": [ "data_quality", "spin_period", @@ -424,45 +421,39 @@ }, CODICEAPID.COD_HI_INST_COUNTS_AGGREGATED: { "dataset_name": "imap_codice_l1a_hi-counters-aggregated", - "input_dims": {}, + "dims": {}, "instrument": "hi", "num_counters": len( HI_COUNTERS_AGGREGATED_VARIABLE_NAMES ), # The number of counters depends on the number of active counters - "output_dims": {}, "support_variables": ["data_quality", "spin_period"], "variable_names": HI_COUNTERS_AGGREGATED_VARIABLE_NAMES, }, CODICEAPID.COD_HI_INST_COUNTS_SINGLES: { "dataset_name": "imap_codice_l1a_hi-counters-singles", - "input_dims": { + "dims": { "ssd_index": 12, }, "instrument": "hi", "num_counters": 3, - "output_dims": { - "ssd_index": 12, - }, "support_variables": ["data_quality", "spin_period"], "variable_names": HI_COUNTERS_SINGLES_VARIABLE_NAMES, }, CODICEAPID.COD_HI_INST_COUNTS_PRIORITIES: { "dataset_name": "imap_codice_l1a_hi-priority", - "input_dims": {}, + "dims": {}, "instrument": "hi", "num_counters": 6, - "output_dims": {}, "support_variables": ["data_quality", "spin_period"], "variable_names": HI_PRIORITY_VARIABLE_NAMES, }, CODICEAPID.COD_HI_OMNI_SPECIES_COUNTS: { "dataset_name": "imap_codice_l1a_hi-omni", "energy_table": OMNI_ENERGY_TABLE, - "input_dims": {"esa_step": 15, "inst_az": 4}, + "dims": {"esa_step": 15, "inst_az": 4}, "instrument": "hi", "num_counters": 8, "num_spins": 4, - "output_dims": {"esa_step": 15, "inst_az": 4}, "support_variables": [ "data_quality", "spin_period", @@ -481,7 +472,7 @@ CODICEAPID.COD_HI_SECT_SPECIES_COUNTS: { "dataset_name": "imap_codice_l1a_hi-sectored", "energy_table": SECTORED_ENERGY_TABLE, - "input_dims": { + "dims": { "esa_step": 8, "ssd_index": 12, "spin_sector_index": 12, @@ -489,11 +480,6 @@ "instrument": "hi", "num_counters": 4, "num_spins": 16, - "output_dims": { - "esa_step": 8, - "ssd_index": 12, - "spin_sector_index": 12, - }, "support_variables": [ "data_quality", "spin_period", @@ -506,10 +492,9 @@ }, CODICEAPID.COD_LO_IAL: { "dataset_name": "imap_codice_l1a_lo-ialirt", - "input_dims": {"spin_sector": 1, "esa_step": 128}, + "dims": {"esa_step": 128, "spin_sector": 1}, "instrument": "lo", "num_counters": 9, - "output_dims": {"spin_sector": 1, "esa_step": 128}, "support_variables": [ "energy_table", "acquisition_time_per_step", @@ -524,12 +509,11 @@ }, CODICEAPID.COD_LO_INST_COUNTS_AGGREGATED: { "dataset_name": "imap_codice_l1a_lo-counters-aggregated", - "input_dims": {"esa_step": 128, "spin_sector_pairs": 6}, + "dims": {"esa_step": 128, "spin_sector_pairs": 6}, "instrument": "lo", "num_counters": len( LO_COUNTERS_AGGREGATED_VARIABLE_NAMES ), # The number of counters depends on the number of active counters - "output_dims": {"spin_sector_pairs": 6, "esa_step": 128}, "support_variables": [ "energy_table", "acquisition_time_per_step", @@ -544,10 +528,9 @@ }, CODICEAPID.COD_LO_INST_COUNTS_SINGLES: { "dataset_name": "imap_codice_l1a_lo-counters-singles", - "input_dims": {"esa_step": 128, "inst_az": 24, "spin_sector_pairs": 6}, + "dims": {"esa_step": 128, "inst_az": 24, "spin_sector_pairs": 6}, "instrument": "lo", "num_counters": 1, - "output_dims": {"inst_az": 24, "spin_sector_pairs": 6, "esa_step": 128}, "support_variables": [ "energy_table", "acquisition_time_per_step", @@ -562,10 +545,9 @@ }, CODICEAPID.COD_LO_SW_ANGULAR_COUNTS: { "dataset_name": "imap_codice_l1a_lo-sw-angular", - "input_dims": {"esa_step": 128, "inst_az": 5, "spin_sector": 12}, + "dims": {"esa_step": 128, "inst_az": 5, "spin_sector": 12}, "instrument": "lo", "num_counters": 4, - "output_dims": {"inst_az": 5, "spin_sector": 12, "esa_step": 128}, "support_variables": [ "energy_table", "acquisition_time_per_step", @@ -580,10 +562,9 @@ }, CODICEAPID.COD_LO_NSW_ANGULAR_COUNTS: { "dataset_name": "imap_codice_l1a_lo-nsw-angular", - "input_dims": {"esa_step": 128, "inst_az": 19, "spin_sector": 12}, + "dims": {"esa_step": 128, "inst_az": 19, "spin_sector": 12}, "instrument": "lo", "num_counters": 1, - "output_dims": {"inst_az": 19, "spin_sector": 12, "esa_step": 128}, "support_variables": [ "energy_table", "acquisition_time_per_step", @@ -598,10 +579,9 @@ }, CODICEAPID.COD_LO_SW_PRIORITY_COUNTS: { "dataset_name": "imap_codice_l1a_lo-sw-priority", - "input_dims": {"esa_step": 128, "spin_sector": 12}, + "dims": {"esa_step": 128, "spin_sector": 12}, "instrument": "lo", "num_counters": 5, - "output_dims": {"spin_sector": 12, "esa_step": 128}, "support_variables": [ "energy_table", "acquisition_time_per_step", @@ -616,10 +596,9 @@ }, CODICEAPID.COD_LO_NSW_PRIORITY_COUNTS: { "dataset_name": "imap_codice_l1a_lo-nsw-priority", - "input_dims": {"esa_step": 128, "spin_sector": 12}, + "dims": {"esa_step": 128, "spin_sector": 12}, "instrument": "lo", "num_counters": 2, - "output_dims": {"spin_sector": 12, "esa_step": 128}, "support_variables": [ "energy_table", "acquisition_time_per_step", @@ -634,10 +613,9 @@ }, CODICEAPID.COD_LO_SW_SPECIES_COUNTS: { "dataset_name": "imap_codice_l1a_lo-sw-species", - "input_dims": {"esa_step": 128, "spin_sector": 1}, + "dims": {"esa_step": 128, "spin_sector": 1}, "instrument": "lo", "num_counters": 16, - "output_dims": {"spin_sector": 1, "esa_step": 128}, "support_variables": [ "energy_table", "acquisition_time_per_step", @@ -652,10 +630,9 @@ }, CODICEAPID.COD_LO_NSW_SPECIES_COUNTS: { "dataset_name": "imap_codice_l1a_lo-nsw-species", - "input_dims": {"esa_step": 128, "spin_sector": 1}, + "dims": {"esa_step": 128, "spin_sector": 1}, "instrument": "lo", "num_counters": 8, - "output_dims": {"spin_sector": 1, "esa_step": 128}, "support_variables": [ "energy_table", "acquisition_time_per_step", diff --git a/imap_processing/tests/codice/test_codice_l1a.py b/imap_processing/tests/codice/test_codice_l1a.py index 851bd6bfba..7bb40bbb7d 100644 --- a/imap_processing/tests/codice/test_codice_l1a.py +++ b/imap_processing/tests/codice/test_codice_l1a.py @@ -41,16 +41,16 @@ EXPECTED_ARRAY_SHAPES = [ (304, 15), # hi-ialirt - (76, 1, 128), # lo-ialirt + (76, 128, 1), # lo-ialirt (31778,), # hskp - (77, 6, 128), # lo-counters-aggregated - (77, 24, 6, 128), # lo-counters-singles - (77, 12, 128), # lo-sw-priority - (77, 12, 128), # lo-nsw-priority - (77, 1, 128), # lo-sw-species - (77, 1, 128), # lo-nsw-species - (77, 5, 12, 128), # lo-sw-angular - (77, 19, 12, 128), # lo-nsw-angular + (77, 128, 6), # lo-counters-aggregated + (77, 128, 24, 6), # lo-counters-singles + (77, 128, 12), # lo-sw-priority + (77, 128, 12), # lo-nsw-priority + (77, 128, 1), # lo-sw-species + (77, 128, 1), # lo-nsw-species + (77, 128, 5, 12), # lo-sw-angular + (77, 128, 19, 12), # lo-nsw-angular (77,), # hi-counters-aggregated (77, 12), # hi-counters-singles (), # hi-omni, shapes are specific to species @@ -259,6 +259,56 @@ def test_l1a_validate_data_arrays(test_l1a_data: xr.Dataset, index): ) +@pytest.mark.parametrize("index", range(len(DESCRIPTORS))) +def test_l1a_validate_dimensions(test_l1a_data, index): + """Tests that the dimensions of the data are in the expected order. + + Parameters + ---------- + test_l1a_data : list[xarray.Dataset] + A list of ``xarray`` datasets containing the test data + index : int + The index of the list to test + """ + + descriptor = DESCRIPTORS[index] + dataset = test_l1a_data[index] + + # This is the expected order of dimensions. Not all of these appear in every + # data product, but for those that do appear, they should be in this order. + expected_dims_order = [ + "epoch", + "esa_step", + "inst_az", + "spin_sector", + "spin_sector_pairs", + "ssd_index", + ] + + # We don't need to check hskp, direct events, or binned datasets since they + # are not multidimensional + if descriptor not in [ + "hskp", + "lo-direct-events", + "hi-direct-events", + "hi-omni", + "hi-ialirt", + "hi-sectored", + ]: + # Get the variables that have dimensions that need to be checked + counters = getattr( + constants, f"{descriptor.upper().replace('-', '_')}_VARIABLE_NAMES" + ) + + # Ensure that, of the dimensions in the particular variable, they occur + # in the expected order. + for counter in counters: + positions = [ + expected_dims_order.index(dim) for dim in dataset[counter].dims + ] + assert positions == sorted(positions) + + @pytest.mark.parametrize("index", range(len(DESCRIPTORS))) @pytest.mark.xfail(reason="Validation test turned off; awaiting fixes") def test_l1a_validate_epoch_values(test_l1a_data, index):