Skip to content

CoDICE: consistent dimension order #1920

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
bourque merged 6 commits into from
Jul 11, 2025
Merged

CoDICE: consistent dimension order #1920

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
843e98b
squash
bourque Jul 9, 2025
d9285c0
squash
bourque Jul 9, 2025
79e448c
Merge branch 'dev' of github.com:IMAP-Science-Operations-Center/imap_…
bourque Jul 9, 2025
f87f92d
simplified code for reshaping data
bourque Jul 9, 2025
1375076
squash
bourque Jul 9, 2025
551a5cc
Added test for testing dimension order
bourque Jul 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 20 additions & 45 deletions imap_processing/codice/codice_l1a.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,8 +143,8 @@ def define_coordinates(self) -> None:
self.coords = {}

coord_names = [
*self.config["output_dims"].keys(),
*[key + "_label" for key in self.config["output_dims"].keys()],
*self.config["dims"].keys(),
*[key + "_label" for key in self.config["dims"].keys()],
]

# Define epoch coordinates
Expand Down Expand Up @@ -174,7 +174,7 @@ def define_coordinates(self) -> None:
"spin_sector_index",
"ssd_index",
]:
values = np.arange(self.config["output_dims"][name])
values = np.arange(self.config["dims"][name])
dims = [name]
elif name == "spin_sector_pairs_label":
values = np.array(
Expand All @@ -196,7 +196,7 @@ def define_coordinates(self) -> None:
"ssd_index_label",
]:
key = name.removesuffix("_label")
values = np.arange(self.config["output_dims"][key]).astype(str)
values = np.arange(self.config["dims"][key]).astype(str)
dims = [key]

coord = xr.DataArray(
Expand Down Expand Up @@ -230,16 +230,16 @@ def define_data_variables(self) -> xr.Dataset:
# Stack the data so that it is easier to reshape and iterate over
all_data = np.stack(self.data)

# The dimension of all_data is something like (epoch, num_counters,
# num_energy_steps, num_positions, num_spin_sectors) (or may be slightly
# The dimension of all_data is something like (epoch, num_energy_steps,
# num_positions, num_spin_sectors, num_counters) (or may be slightly
# different depending on the data product). In any case, iterate over
# the num_counters dimension to isolate the data for each counter so
# each counter's data can be placed in a separate CDF data variable.
for counter, variable_name in zip(
range(all_data.shape[1]), self.config["variable_names"]
range(all_data.shape[-1]), self.config["variable_names"]
):
# Extract the counter data
counter_data = all_data[:, counter, ...]
counter_data = all_data[..., counter]

# Get the CDF attributes
descriptor = self.config["dataset_name"].split("imap_codice_l1a_")[-1]
Expand All @@ -249,7 +249,7 @@ def define_data_variables(self) -> xr.Dataset:
# For most products, the final CDF dimensions always has "epoch" as
# the first dimension followed by the dimensions for the specific
# data product
dims = ["epoch", *list(self.config["output_dims"].keys())]
dims = ["epoch", *list(self.config["dims"].keys())]

# However, CoDICE-Hi products use specific energy bins for the
# energy dimension
Expand Down Expand Up @@ -589,52 +589,27 @@ def reshape_data(self) -> None:

These data need to be divided up by species or priorities (or
what I am calling "counters" as a general term), and re-arranged into
4D arrays representing dimensions such as time, spin sectors, positions,
and energies (depending on the data product).
multidimensional arrays representing dimensions such as time,
spin sectors, positions, and energies (depending on the data product).

However, the existence and order of these dimensions can vary depending
on the specific data product, so we define this in the "input_dims"
and "output_dims" values configuration dictionary; the "input_dims"
defines how the dimensions are written into the packet data, while
"output_dims" defines how the dimensions should be written to the final
CDF product.
on the specific data product, so we define this in the "dims" key of the
configuration dictionary.
"""
# This will contain the reshaped data for all counters
self.data = []

# First reshape the data based on how it is written to the data array of
# the packet data. The number of counters is the first dimension / axis,
# with the exception of lo-counters-aggregated which is treated slightly
# differently
if self.config["dataset_name"] != "imap_codice_l1a_lo-counters-aggregated":
reshape_dims = (
self.config["num_counters"],
*self.config["input_dims"].values(),
)
else:
reshape_dims = (
*self.config["input_dims"].values(),
self.config["num_counters"],
)

# Then, transpose the data based on how the dimensions should be written
# to the CDF file. Since this is specific to each data product, we need
# to determine this dynamically based on the "output_dims" config.
# Again, lo-counters-aggregated is treated slightly differently
input_keys = ["num_counters", *self.config["input_dims"].keys()]
output_keys = ["num_counters", *self.config["output_dims"].keys()]
if self.config["dataset_name"] != "imap_codice_l1a_lo-counters-aggregated":
transpose_axes = [input_keys.index(dim) for dim in output_keys]
else:
transpose_axes = [1, 2, 0] # [esa_step, spin_sector_pairs, num_counters]

# Reshape the data based on how it is written to the data array of
# the packet data. The number of counters is the last dimension / axis.
reshape_dims = (
*self.config["dims"].values(),
self.config["num_counters"],
)
for packet_data in self.raw_data:
reshaped_packet_data = np.array(packet_data, dtype=np.uint32).reshape(
reshape_dims
)
reshaped_cdf_data = np.transpose(reshaped_packet_data, axes=transpose_axes)

self.data.append(reshaped_cdf_data)
self.data.append(reshaped_packet_data)

# No longer need to keep the raw data around
del self.raw_data
Expand Down
53 changes: 15 additions & 38 deletions imap_processing/codice/constants.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -401,20 +401,17 @@
"fe": [0.0125, 0.025, 0.05, 0.1, 0.2, 0.4, 0.8, 1.6, 3.2],
}

# TODO: Possibly move to consistent order of dimensions with other instruments
# TBD after discussion with Joey and at the Science Team Meeting in Feb
# Various configurations to support processing of individual data products
# Much of these are described in the algorithm document in chapter 10 ("Data
# Level 1A")
DATA_PRODUCT_CONFIGURATIONS: dict[CODICEAPID | int, dict] = {
CODICEAPID.COD_HI_IAL: {
"dataset_name": "imap_codice_l1a_hi-ialirt",
"energy_table": IALIRT_ENERGY_TABLE,
"input_dims": {"esa_step": 15, "inst_az": 4},
"dims": {"esa_step": 15, "inst_az": 4},
"instrument": "hi",
"num_counters": 1,
"num_spins": 4,
"output_dims": {"esa_step": 15, "inst_az": 4},
"support_variables": [
"data_quality",
"spin_period",
Expand All @@ -424,45 +421,39 @@
},
CODICEAPID.COD_HI_INST_COUNTS_AGGREGATED: {
"dataset_name": "imap_codice_l1a_hi-counters-aggregated",
"input_dims": {},
"dims": {},
"instrument": "hi",
"num_counters": len(
HI_COUNTERS_AGGREGATED_VARIABLE_NAMES
), # The number of counters depends on the number of active counters
"output_dims": {},
"support_variables": ["data_quality", "spin_period"],
"variable_names": HI_COUNTERS_AGGREGATED_VARIABLE_NAMES,
},
CODICEAPID.COD_HI_INST_COUNTS_SINGLES: {
"dataset_name": "imap_codice_l1a_hi-counters-singles",
"input_dims": {
"dims": {
"ssd_index": 12,
},
"instrument": "hi",
"num_counters": 3,
"output_dims": {
"ssd_index": 12,
},
"support_variables": ["data_quality", "spin_period"],
"variable_names": HI_COUNTERS_SINGLES_VARIABLE_NAMES,
},
CODICEAPID.COD_HI_INST_COUNTS_PRIORITIES: {
"dataset_name": "imap_codice_l1a_hi-priority",
"input_dims": {},
"dims": {},
"instrument": "hi",
"num_counters": 6,
"output_dims": {},
"support_variables": ["data_quality", "spin_period"],
"variable_names": HI_PRIORITY_VARIABLE_NAMES,
},
CODICEAPID.COD_HI_OMNI_SPECIES_COUNTS: {
"dataset_name": "imap_codice_l1a_hi-omni",
"energy_table": OMNI_ENERGY_TABLE,
"input_dims": {"esa_step": 15, "inst_az": 4},
"dims": {"esa_step": 15, "inst_az": 4},
"instrument": "hi",
"num_counters": 8,
"num_spins": 4,
"output_dims": {"esa_step": 15, "inst_az": 4},
"support_variables": [
"data_quality",
"spin_period",
Expand All @@ -481,19 +472,14 @@
CODICEAPID.COD_HI_SECT_SPECIES_COUNTS: {
"dataset_name": "imap_codice_l1a_hi-sectored",
"energy_table": SECTORED_ENERGY_TABLE,
"input_dims": {
"dims": {
"esa_step": 8,
"ssd_index": 12,
"spin_sector_index": 12,
},
"instrument": "hi",
"num_counters": 4,
"num_spins": 16,
"output_dims": {
"esa_step": 8,
"ssd_index": 12,
"spin_sector_index": 12,
},
"support_variables": [
"data_quality",
"spin_period",
Expand All @@ -506,10 +492,9 @@
},
CODICEAPID.COD_LO_IAL: {
"dataset_name": "imap_codice_l1a_lo-ialirt",
"input_dims": {"spin_sector": 1, "esa_step": 128},
"dims": {"esa_step": 128, "spin_sector": 1},
"instrument": "lo",
"num_counters": 9,
"output_dims": {"spin_sector": 1, "esa_step": 128},
"support_variables": [
"energy_table",
"acquisition_time_per_step",
Expand All @@ -524,12 +509,11 @@
},
CODICEAPID.COD_LO_INST_COUNTS_AGGREGATED: {
"dataset_name": "imap_codice_l1a_lo-counters-aggregated",
"input_dims": {"esa_step": 128, "spin_sector_pairs": 6},
"dims": {"esa_step": 128, "spin_sector_pairs": 6},
"instrument": "lo",
"num_counters": len(
LO_COUNTERS_AGGREGATED_VARIABLE_NAMES
), # The number of counters depends on the number of active counters
"output_dims": {"spin_sector_pairs": 6, "esa_step": 128},
"support_variables": [
"energy_table",
"acquisition_time_per_step",
Expand All @@ -544,10 +528,9 @@
},
CODICEAPID.COD_LO_INST_COUNTS_SINGLES: {
"dataset_name": "imap_codice_l1a_lo-counters-singles",
"input_dims": {"esa_step": 128, "inst_az": 24, "spin_sector_pairs": 6},
"dims": {"esa_step": 128, "inst_az": 24, "spin_sector_pairs": 6},
"instrument": "lo",
"num_counters": 1,
"output_dims": {"inst_az": 24, "spin_sector_pairs": 6, "esa_step": 128},
"support_variables": [
"energy_table",
"acquisition_time_per_step",
Expand All @@ -562,10 +545,9 @@
},
CODICEAPID.COD_LO_SW_ANGULAR_COUNTS: {
"dataset_name": "imap_codice_l1a_lo-sw-angular",
"input_dims": {"esa_step": 128, "inst_az": 5, "spin_sector": 12},
"dims": {"esa_step": 128, "inst_az": 5, "spin_sector": 12},
"instrument": "lo",
"num_counters": 4,
"output_dims": {"inst_az": 5, "spin_sector": 12, "esa_step": 128},
"support_variables": [
"energy_table",
"acquisition_time_per_step",
Expand All @@ -580,10 +562,9 @@
},
CODICEAPID.COD_LO_NSW_ANGULAR_COUNTS: {
"dataset_name": "imap_codice_l1a_lo-nsw-angular",
"input_dims": {"esa_step": 128, "inst_az": 19, "spin_sector": 12},
"dims": {"esa_step": 128, "inst_az": 19, "spin_sector": 12},
"instrument": "lo",
"num_counters": 1,
"output_dims": {"inst_az": 19, "spin_sector": 12, "esa_step": 128},
"support_variables": [
"energy_table",
"acquisition_time_per_step",
Expand All @@ -598,10 +579,9 @@
},
CODICEAPID.COD_LO_SW_PRIORITY_COUNTS: {
"dataset_name": "imap_codice_l1a_lo-sw-priority",
"input_dims": {"esa_step": 128, "spin_sector": 12},
"dims": {"esa_step": 128, "spin_sector": 12},
"instrument": "lo",
"num_counters": 5,
"output_dims": {"spin_sector": 12, "esa_step": 128},
"support_variables": [
"energy_table",
"acquisition_time_per_step",
Expand All @@ -616,10 +596,9 @@
},
CODICEAPID.COD_LO_NSW_PRIORITY_COUNTS: {
"dataset_name": "imap_codice_l1a_lo-nsw-priority",
"input_dims": {"esa_step": 128, "spin_sector": 12},
"dims": {"esa_step": 128, "spin_sector": 12},
"instrument": "lo",
"num_counters": 2,
"output_dims": {"spin_sector": 12, "esa_step": 128},
"support_variables": [
"energy_table",
"acquisition_time_per_step",
Expand All @@ -634,10 +613,9 @@
},
CODICEAPID.COD_LO_SW_SPECIES_COUNTS: {
"dataset_name": "imap_codice_l1a_lo-sw-species",
"input_dims": {"esa_step": 128, "spin_sector": 1},
"dims": {"esa_step": 128, "spin_sector": 1},
"instrument": "lo",
"num_counters": 16,
"output_dims": {"spin_sector": 1, "esa_step": 128},
"support_variables": [
"energy_table",
"acquisition_time_per_step",
Expand All @@ -652,10 +630,9 @@
},
CODICEAPID.COD_LO_NSW_SPECIES_COUNTS: {
"dataset_name": "imap_codice_l1a_lo-nsw-species",
"input_dims": {"esa_step": 128, "spin_sector": 1},
"dims": {"esa_step": 128, "spin_sector": 1},
"instrument": "lo",
"num_counters": 8,
"output_dims": {"spin_sector": 1, "esa_step": 128},
"support_variables": [
"energy_table",
"acquisition_time_per_step",
Expand Down
Loading