fix: Handle none values and column grouping in results/steps dataframe utility

nisystemlink/clients/testmonitor/utilities/_dataframe_utilities.py

@@ -110,10 +110,11 @@ def __normalize_status(
 
     """
     status = data.get("status", {})
-    if status.get("status_type") == "CUSTOM":
-        data["status"] = status.get("status_name", None)
-    else:
-        data["status"] = getattr(status.get("status_type", None), "value", None)
+    if status:
+        if status.get("status_type") == "CUSTOM":
+            data["status"] = status.get("status_name", None)
+        else:
+            data["status"] = getattr(status.get("status_type", None), "value", None)
 
 
 def __format_results_columns(results_dataframe: pd.DataFrame) -> pd.DataFrame:
@@ -229,11 +230,16 @@ def __filter_invalid_measurements(
     if step.data and step.data.parameters and is_valid_measurement is not None:
         valid_measurement_parameters = []
         for measurement in step.data.parameters:
-            if is_valid_measurement and is_valid_measurement(measurement):
+            if (
+                measurement
+                and is_valid_measurement
+                and is_valid_measurement(measurement)
+            ):
                 valid_measurement_parameters.append(measurement)
 
         step_dict["data"]["parameters"] = [
-            measurement.dict() for measurement in valid_measurement_parameters
+            measurement.dict(exclude_none=True)
+            for measurement in valid_measurement_parameters
         ]
 
 

nisystemlink/clients/testmonitor/utilities/constants.py

@@ -5,6 +5,7 @@ class DataFrameHeaders:
 
     CATEGORY_COLUMN_HEADERS = [
         "general",
+        "inputs",
         "outputs",
         "data",
         "properties",

tests/testmonitor/test_testmonitor_dataframe_utilities.py

@@ -455,7 +455,6 @@ def test__convert_steps_to_dataframe_with_invalid_data_parameters__returns_only_
             "name",
             "step_id",
             "result_id",
-            "status",
             "data.text",
         ]
         expected_steps_dataframe = self.__get_expected_steps_dataframe(
@@ -473,6 +472,64 @@ def test__convert_steps_to_dataframe_with_invalid_data_parameters__returns_only_
             steps_dataframe, expected_steps_dataframe, check_dtype=True
         )
 
+    def test__convert_steps_to_dataframe_with_many_unset_measurement_fields__not_returns_unset_measurements_fields(
+        self,
+    ):
+        step_data = Step(
+            name="step_name",
+            step_id="5ffb2bf6771fa11e877838dd6",
+            result_id="5ffb2bf6771fa11e877838dd8",
+            data=StepData(
+                text="data1",
+                parameters=[
+                    Measurement(
+                        name="parameter_11",
+                        measurement="11.0",
+                    ),
+                    Measurement(
+                        name="parameter_21",
+                        measurement="11.3",
+                    ),
+                ],
+            ),
+        )
+        expected_column_order = [
+            "name",
+            "step_id",
+            "result_id",
+            "data.text",
+            "data.measurement.name",
+            "data.measurement.measurement",
+        ]
+        expected_data_parameters = [
+            [
+                {
+                    "data.measurement.name": "parameter_11",
+                    "data.measurement.measurement": "11.0",
+                },
+                {
+                    "data.measurement.name": "parameter_21",
+                    "data.measurement.measurement": "11.3",
+                },
+            ]
+        ]
+        expected_steps_dataframe = self.__get_expected_steps_dataframe(
+            [step_data],
+            expected_column_order=expected_column_order,
+            expected_data_parameters=expected_data_parameters,
+        )
+
+        steps_dataframe = convert_steps_to_dataframe([step_data])
+
+        assert not steps_dataframe.empty
+        assert (
+            steps_dataframe.columns.to_list()
+            == expected_steps_dataframe.columns.to_list()
+        )
+        pd.testing.assert_frame_equal(
+            steps_dataframe, expected_steps_dataframe, check_dtype=True
+        )
+
     def test__convert_steps_to_dataframe_with_none_callback__returns_step_with_all_data_parameters(
         self, mock_steps_data: List[Step]
     ):