updated pydarshan to support DFS/DAOS heatmaps

Shane Snyder · Shane Snyder · commit 47a1a9797b53 · 2024-11-11T17:30:43.000-06:00
diff --git a/darshan-util/pydarshan/darshan/examples/example_logs/snyder_ior-DFS_id1057716-201712_11-8-64400-1922568413188514066_1.darshan b/darshan-util/pydarshan/darshan/examples/example_logs/snyder_ior-DFS_id1057716-201712_11-8-64400-1922568413188514066_1.darshan
diff --git a/darshan-util/pydarshan/darshan/examples/example_logs/snyder_ior-DFS_id1069383-23981_11-11-71996-14940917661149127511_1.darshan b/darshan-util/pydarshan/darshan/examples/example_logs/snyder_ior-DFS_id1069383-23981_11-11-71996-14940917661149127511_1.darshan
diff --git a/darshan-util/pydarshan/darshan/report.py b/darshan-util/pydarshan/darshan/report.py
@@ -608,7 +608,9 @@ def read_all_heatmap_records(self):
         _nrecs_heatmap = {  
             16592106915301738621: "heatmap:POSIX",
             3989511027826779520: "heatmap:STDIO",
-            3668870418325792824: "heatmap:MPIIO"
+            3668870418325792824: "heatmap:MPIIO",
+            1597927878319380788: "heatmap:DFS",
+            4131494093108637317: "heatmap:DAOS"
         }
 
         def heatmap_rec_to_module_name(rec, nrecs=None):
diff --git a/darshan-util/pydarshan/darshan/tests/test_plot_exp_common.py b/darshan-util/pydarshan/darshan/tests/test_plot_exp_common.py
@@ -14,14 +14,14 @@
     "log_path, mod, func, expected_xticklabels",
     [
         (
-            "snyder_ior-DFS_id1057716-201712_11-8-64400-1922568413188514066_1.darshan",
+            "snyder_ior-DFS_id1069383-23981_11-11-71996-14940917661149127511_1.darshan",
             "DFS",
             plot_access_histogram,
             ["0-100", "101-1K", "1K-10K", "10K-100K", "100K-1M",
             "1M-4M", "4M-10M", "10M-100M", "100M-1G", "1G+"]
         ),
         (
-            "snyder_ior-DFS_id1057716-201712_11-8-64400-1922568413188514066_1.darshan",
+            "snyder_ior-DFS_id1069383-23981_11-11-71996-14940917661149127511_1.darshan",
             "DAOS",
             plot_access_histogram,
             ["0-100", "101-1K", "1K-10K", "10K-100K", "100K-1M",
@@ -79,13 +79,13 @@
             "1M-4M", "4M-10M", "10M-100M", "100M-1G", "1G+"],
         ),
         (
-            "snyder_ior-DFS_id1057716-201712_11-8-64400-1922568413188514066_1.darshan",
+            "snyder_ior-DFS_id1069383-23981_11-11-71996-14940917661149127511_1.darshan",
             "DFS",
             plot_opcounts,
             ['Read', 'Readx', 'Write', 'Writex', 'Open', 'GlobalOpen', 'Lookup', 'Get Size', 'Punch', 'Remove', 'Stat'],
         ),
         (
-            "snyder_ior-DFS_id1057716-201712_11-8-64400-1922568413188514066_1.darshan",
+            "snyder_ior-DFS_id1069383-23981_11-11-71996-14940917661149127511_1.darshan",
             "DAOS",
             plot_opcounts,
             ['ObjFetch', 'ObjUpdate', 'ObjOpen', 'ArrRead', 'ArrWrite', 'ArrOpen', 'KVGet', 'KVPut', 'KVOpen'],
diff --git a/darshan-util/pydarshan/darshan/tests/test_plot_io_cost.py b/darshan-util/pydarshan/darshan/tests/test_plot_io_cost.py
@@ -57,13 +57,13 @@
             ),
         ),
         (
-            "snyder_ior-DFS_id1057716-201712_11-8-64400-1922568413188514066_1.darshan",
+            "snyder_ior-DFS_id1069383-23981_11-11-71996-14940917661149127511_1.darshan",
             pd.DataFrame(
                 np.array([
-                    [0.0, 0.0, 0.0, 0.0],
-                    [0.0, 4.515051841e-06, 0.0, 0.0],
-                    [0.001456562, 0.002266062, 0.007923812, 0.0],
-                    [0.001492562, 0.002273217, 0.007910812, 0.0],
+                    [0.0, 0.0, 1.490116119e-08, 0.0],
+                    [0.0, 4.872679710e-06, 0.0, 0.0],
+                    [0.001452714, 0.001865715, 0.007635623, 0.0],
+                    [0.001490027, 0.001865178, 0.007622942, 0.0],
                 ]),
                 ["POSIX", "STDIO", "DFS", "DAOS"],
                 ["Read", "Write", "Meta", "Wait"],
@@ -101,8 +101,8 @@ def test_get_io_cost_df(logname, expected_df):
             [0.0, 1111.0],
         ),
         (
-            "snyder_ior-DFS_id1057716-201712_11-8-64400-1922568413188514066_1.darshan",
-            [0.0, 0.80276728],
+            "snyder_ior-DFS_id1069383-23981_11-11-71996-14940917661149127511_1.darshan",
+            [0.0, 0.75089884],
         ),
     ],
 )
@@ -134,9 +134,9 @@ def test_plot_io_cost_ylims(logname, expected_ylims):
             ['0', '156', '312', '468', '624', '780'],
         ),
         (
-            "snyder_ior-DFS_id1057716-201712_11-8-64400-1922568413188514066_1.darshan",
-            [0.0 , 0.16055346, 0.32110691, 0.48166037, 0.64221382, 0.80276728],
-            ['0.0000' , '0.1606', '0.3211', '0.4817', '0.6422', '0.8028'],
+            "snyder_ior-DFS_id1069383-23981_11-11-71996-14940917661149127511_1.darshan",
+            [0.0 , 0.15017977, 0.30035954, 0.4505393 , 0.60071907, 0.75089884],
+            ['0.0000', '0.1502', '0.3004', '0.4505', '0.6007', '0.7509'],
         ),
     ],
 )
diff --git a/darshan-util/pydarshan/darshan/tests/test_report.py b/darshan-util/pydarshan/darshan/tests/test_report.py
@@ -81,7 +81,7 @@ def test_dfs_daos_posix_match():
     # backend should produce matching counters where
     # comparable data fields exist
     posix_ior_report = darshan.DarshanReport(get_log_path("snyder_ior-POSIX_id1057716-202103_11-8-64415-6936117869459351096_1.darshan"))
-    dfs_ior_report = darshan.DarshanReport(get_log_path("snyder_ior-DFS_id1057716-201712_11-8-64400-1922568413188514066_1.darshan"))
+    dfs_ior_report = darshan.DarshanReport(get_log_path("snyder_ior-DFS_id1069383-23981_11-11-71996-14940917661149127511_1.darshan"))
     posix_ior_report.mod_read_all_records("POSIX")
     dfs_ior_report.mod_read_all_records("DFS")
     dfs_ior_report.mod_read_all_records("DAOS")
diff --git a/darshan-util/pydarshan/darshan/tests/test_summary.py b/darshan-util/pydarshan/darshan/tests/test_summary.py
@@ -494,7 +494,7 @@ def test_metadata_table(self, log_path, expected_df):
                 2,
             ),
             (
-                "snyder_ior-DFS_id1057716-201712_11-8-64400-1922568413188514066_1.darshan",
+                "snyder_ior-DFS_id1069383-23981_11-11-71996-14940917661149127511_1.darshan",
                 pd.DataFrame(
                     index=[
                         "Log Filename", "Runtime Library Version", "Log Format Version",
@@ -503,9 +503,9 @@ def test_metadata_table(self, log_path, expected_df):
                         "DFS (ver=1)", "DAOS (ver=1)"
                     ],
                     data=[
-                        ["snyder_ior-DFS_id1057716-201712_11-8-64400-1922568413188514066_1.darshan"], ["3.4.7"], ["3.41"],
-                        ["0.07 KiB"], ["0.07 KiB"], ["0.05 KiB"],
-                        ["0.17 KiB"], ["0.39 KiB"],
+                        ["snyder_ior-DFS_id1069383-23981_11-11-71996-14940917661149127511_1.darshan"], ["3.4.7"], ["3.41"],
+                        ["0.08 KiB"], ["0.07 KiB"], ["1.09 KiB"],
+                        ["0.17 KiB"], ["0.40 KiB"],
                     ],
                 ),
                 0,
