Add diff (value + percentge) in cmp report table if exactly two results are compared

Author: amaslenn

src/cloudai/report_generator/comparison_report.py

@@ -118,6 +118,24 @@ def generate(self):
 
         logging.info(f"Comparison report created: {html_file}")
 
+    @staticmethod
+    def _extract_cmp_values(data: list) -> tuple[float | None, float | None]:
+        val1, val2 = None, None
+        try:
+            val1 = float(data[-2])
+            val2 = float(data[-1])
+        except Exception as e:
+            logging.debug(f"Could not extract comparison values from data {data}: {e}")
+        return val1, val2
+
+    @staticmethod
+    def _format_diff_cell(val1: float | None, val2: float | None) -> str:
+        if val1 is None or val2 is None or val2 == 0:
+            return "n/a"
+        diff = val1 - val2
+        diff_percent = (diff / val2) * 100
+        return f"{diff:+.2f} ({diff_percent:+.2f}%)"
+
     def create_table(
         self,
         group: GroupedTestRuns,
@@ -131,9 +149,12 @@ def create_table(
         table = Table(title=f"{title}: {group.name}", title_justify="left")
         for col in info_columns:
             table.add_column(col)
-        for item in group.items:
-            style = next(style_cycle)
-            for col in data_columns:
+
+        enable_diff_column = len(group.items) == 2
+
+        for col in data_columns:
+            for item in group.items:
+                style = next(style_cycle)
                 name_str = "\n".join(item.name.split())
                 table.add_column(
                     f"{name_str}\n[white on {style}]{col}",
@@ -143,11 +164,23 @@ def create_table(
                     no_wrap=False,
                 )
 
+            if enable_diff_column:
+                diff_style = next(style_cycle)
+                table.add_column(f"diff\n{col}", justify="right", style=diff_style, header_style=diff_style)
+
         df_with_max_rows = max(dfs, key=len)
         for row_idx in range(len(df_with_max_rows)):
             data = []
-            for df in dfs:
-                data.extend([str(df[col].get(row_idx, "n/a")) for col in data_columns])
+            for col in data_columns:
+                data_points = []
+                for df in dfs:
+                    data_points.append(str(df[col].get(row_idx, "n/a")))
+
+                if enable_diff_column:
+                    val1, val2 = self._extract_cmp_values(data_points)
+                    data_points.append(self._format_diff_cell(val1, val2))
+
+                data.extend(data_points)
 
             table.add_row(*[str(df_with_max_rows[col][row_idx]) for col in info_columns], *data)
 

tests/report_generation_strategy/test_comparison_report.py

@@ -90,11 +90,16 @@ def test_two_data_points(self, cmp_report: MyComparisonReport, nccl_tr: TestRun)
             ["value"],
         )
 
-        assert len(table.columns) == 3
+        assert len(table.columns) == 4
         assert len(table.rows) == 3
         assert list(table.columns[0].cells) == ["1", "2", "4"]
         assert list(table.columns[1].cells) == ["10", "20", "40"]
         assert list(table.columns[2].cells) == ["100", "200", "400"]
+        assert list(table.columns[3].cells) == [
+            ComparisonReport._format_diff_cell(10, 100),
+            ComparisonReport._format_diff_cell(20, 200),
+            ComparisonReport._format_diff_cell(40, 400),
+        ]
 
     def test_one_data_point_is_empty(self, cmp_report: MyComparisonReport, nccl_tr: TestRun) -> None:
         table = cmp_report.create_table(
@@ -114,11 +119,72 @@ def test_one_data_point_is_empty(self, cmp_report: MyComparisonReport, nccl_tr:
             ["value"],
         )
 
-        assert len(table.columns) == 3
+        assert len(table.columns) == 4
+        assert table.columns[-1].header == "diff\nvalue"
         assert len(table.rows) == 3
         assert list(table.columns[0].cells) == ["1", "2", "4"]
         assert list(table.columns[1].cells) == ["n/a", "n/a", "n/a"]
         assert list(table.columns[2].cells) == ["10", "20", "40"]
+        assert list(table.columns[3].cells) == ["n/a", "n/a", "n/a"]
+
+    def test_two_data_points_with_two_data_columns(self, cmp_report: MyComparisonReport, nccl_tr: TestRun) -> None:
+        table = cmp_report.create_table(
+            GroupedTestRuns(
+                name="grp_name",
+                items=[
+                    TRGroupItem(name="item_name", tr=nccl_tr),
+                    TRGroupItem(name="item_name2", tr=nccl_tr),
+                ],
+            ),
+            [
+                pd.DataFrame({"size": [1], "value1": [10], "value2": [5]}),
+                pd.DataFrame({"size": [1], "value1": [100], "value2": [50]}),
+            ],
+            "title",
+            ["size"],
+            ["value1", "value2"],
+        )
+
+        assert len(table.columns) == 7  # 1 info + 1 size + 2*2 data + 2 diff
+        assert len(table.rows) == 1
+        assert list(table.columns[0].cells) == ["1"]
+        assert list(table.columns[1].cells) == ["10"]
+        assert list(table.columns[2].cells) == ["100"]
+        assert list(table.columns[3].cells) == [
+            ComparisonReport._format_diff_cell(10, 100),
+        ]
+        assert list(table.columns[4].cells) == ["5"]
+        assert list(table.columns[5].cells) == ["50"]
+        assert list(table.columns[6].cells) == [
+            ComparisonReport._format_diff_cell(5, 50),
+        ]
+
+    def test_three_data_points(self, cmp_report: MyComparisonReport, nccl_tr: TestRun) -> None:
+        table = cmp_report.create_table(
+            GroupedTestRuns(
+                name="grp_name",
+                items=[
+                    TRGroupItem(name="item_name", tr=nccl_tr),
+                    TRGroupItem(name="item_name2", tr=nccl_tr),
+                    TRGroupItem(name="item_name3", tr=nccl_tr),
+                ],
+            ),
+            [
+                pd.DataFrame({"size": [1], "value": [10]}),
+                pd.DataFrame({"size": [1], "value": [100]}),
+                pd.DataFrame({"size": [1], "value": [1000]}),
+            ],
+            "title",
+            ["size"],
+            ["value"],
+        )
+
+        assert len(table.columns) == 4  # 1 info + 3*1 data (NO diff)
+        assert len(table.rows) == 1
+        assert list(table.columns[0].cells) == ["1"]
+        assert list(table.columns[1].cells) == ["10"]
+        assert list(table.columns[2].cells) == ["100"]
+        assert list(table.columns[3].cells) == ["1000"]
 
 
 def test_create_charts(cmp_report: MyComparisonReport, nccl_tr: TestRun) -> None: