Skip to content

Commit a1af3c8

Browse files
gujguj
committed
Generating plots using the updated script entries,
and summarizing writer activies
1 parent a8fe8ef commit a1af3c8

11 files changed

Lines changed: 307 additions & 0 deletions

File tree

‎source/utils/profiler_simplified/data/f/1.json‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
[
2+
{ "rank":0, "start":"Wed_Dec_03_22:24:41_2025","ES_DSB_Broadcast_mus": 0, "ES_DSB_Broadcast":{"mus":0, "nCalls":1},"ES_DSB_mus": 6, "ES_DSB":{"mus":6, "nCalls":1},"ES_DSB_AllGather_mus": 2, "ES_DSB_AllGather":{"mus":2, "nCalls":1},"ES_MDAgg_GatherWriteMeta_mus": 254, "ES_MDAgg_GatherWriteMeta":{"mus":254, "nCalls":1},"ES_MDAgg_AggInfo_MetaInfoBcast_mus": 2, "ES_MDAgg_AggInfo_MetaInfoBcast":{"mus":2, "nCalls":1},"ES_MDAgg_AggInfo_FixedMetaInfoGather_mus": 15, "ES_MDAgg_AggInfo_FixedMetaInfoGather":{"mus":15, "nCalls":1},"InitAgg-dsb_GPI_partition_mus": 40, "InitAgg-dsb_GPI_partition":{"mus":40, "nCalls":1},"InitAgg-dsb_GPI_AllGather_mus": 5, "InitAgg-dsb_GPI_AllGather":{"mus":5, "nCalls":1},"InitAgg-dsb_mus": 4358, "InitAgg-dsb":{"mus":4358, "nCalls":1},"InitAgg-dsb_GPI_mus": 55, "InitAgg-dsb_GPI":{"mus":55, "nCalls":1},"ES_MDAgg_GatherWriteMeta_MDBlocks_mus": 9, "ES_MDAgg_GatherWriteMeta_MDBlocks":{"mus":9, "nCalls":1},"DC_mus": 1687, "DC":{"mus":1687, "nCalls":1},"WriteMD_Blocks_mus": 9, "WriteMD_Blocks":{"mus":9, "nCalls":1},"WriteMD_mus": 65, "WriteMD":{"mus":65, "nCalls":1},"WriteData_mus": 161095, "WriteData":{"mus":161095, "nCalls":2},"ES_WriteData_mus": 156350, "ES_WriteData":{"mus":156350, "nCalls":1},"BS_mus": 2, "BS":{"mus":2, "nCalls":1},"ES_MDAgg_AggInfo_mus": 313, "ES_MDAgg_AggInfo":{"mus":313, "nCalls":1},"ES_MDAgg_mus": 580, "ES_MDAgg":{"mus":580, "nCalls":1},"PDW_mus": 4753, "PDW":{"mus":4753, "nCalls":1},"ES_MDAgg_AggInfo_SelectMetaInfoGather_mus": 9, "ES_MDAgg_AggInfo_SelectMetaInfoGather":{"mus":9, "nCalls":1},"ES_CloseTS_mus": 122, "ES_CloseTS":{"mus":122, "nCalls":1},"ES_mus": 157065, "ES":{"mus":157065, "nCalls":1},"WriteMmD_mus": 69, "WriteMmD":{"mus":69, "nCalls":1}, "databytes":0, "metadatabytes":0, "metametadatabytes":0, "transport_0":{"type":"File_POSIX", "wbytes":335544320, "close":{"mus":1411, "nCalls":1}, "write":{"mus":156312, "nCalls":10}, "open":{"mus":87, "nCalls":1}}, "transport_1":{"type":"File_POSIX", "wbytes":10448, "close":{"mus":241, "nCalls":1}, "write":{"mus":51, "nCalls":5}, "open":{"mus":65, "nCalls":1}} },
3+
{ "rank":1, "start":"Wed_Dec_03_22:24:41_2025","ES_DSB_Broadcast_mus": 0, "ES_DSB_Broadcast":{"mus":0, "nCalls":1},"ES_DSB_mus": 149, "ES_DSB":{"mus":149, "nCalls":1},"ES_DSB_AllGather_mus": 146, "ES_DSB_AllGather":{"mus":146, "nCalls":1},"ES_MDAgg_GatherWriteMeta_mus": 331, "ES_MDAgg_GatherWriteMeta":{"mus":331, "nCalls":1},"ES_MDAgg_AggInfo_MetaInfoBcast_mus": 2009, "ES_MDAgg_AggInfo_MetaInfoBcast":{"mus":2009, "nCalls":1},"ES_MDAgg_AggInfo_FixedMetaInfoGather_mus": 5, "ES_MDAgg_AggInfo_FixedMetaInfoGather":{"mus":5, "nCalls":1},"InitAgg-dsb_GPI_partition_mus": 41, "InitAgg-dsb_GPI_partition":{"mus":41, "nCalls":1},"InitAgg-dsb_GPI_AllGather_mus": 72, "InitAgg-dsb_GPI_AllGather":{"mus":72, "nCalls":1},"InitAgg-dsb_mus": 4419, "InitAgg-dsb":{"mus":4419, "nCalls":1},"InitAgg-dsb_GPI_mus": 121, "InitAgg-dsb_GPI":{"mus":121, "nCalls":1},"DC_mus": 175, "DC":{"mus":175, "nCalls":1},"WriteData_mus": 159845, "WriteData":{"mus":159845, "nCalls":2},"ES_WriteData_mus": 155082, "ES_WriteData":{"mus":155082, "nCalls":1},"BS_mus": 24, "BS":{"mus":24, "nCalls":1},"ES_MDAgg_AggInfo_mus": 2075, "ES_MDAgg_AggInfo":{"mus":2075, "nCalls":1},"ES_MDAgg_mus": 2430, "ES_MDAgg":{"mus":2430, "nCalls":1},"PDW_mus": 4771, "PDW":{"mus":4771, "nCalls":1},"ES_MDAgg_AggInfo_SelectMetaInfoGather_mus": 1, "ES_MDAgg_AggInfo_SelectMetaInfoGather":{"mus":1, "nCalls":1},"ES_CloseTS_mus": 120, "ES_CloseTS":{"mus":120, "nCalls":1},"ES_mus": 157792, "ES":{"mus":157792, "nCalls":1}, "databytes":0, "metadatabytes":0, "metametadatabytes":0, "transport_0":{"type":"File_POSIX", "wbytes":335544320, "close":{"mus":172, "nCalls":1}, "write":{"mus":155023, "nCalls":10}, "open":{"mus":69, "nCalls":1}} },
4+
{ "rank":2, "start":"Wed_Dec_03_22:24:41_2025","ES_DSB_Broadcast_mus": 0, "ES_DSB_Broadcast":{"mus":0, "nCalls":1},"ES_DSB_mus": 150, "ES_DSB":{"mus":150, "nCalls":1},"ES_DSB_AllGather_mus": 147, "ES_DSB_AllGather":{"mus":147, "nCalls":1},"ES_MDAgg_GatherWriteMeta_mus": 331, "ES_MDAgg_GatherWriteMeta":{"mus":331, "nCalls":1},"ES_MDAgg_AggInfo_MetaInfoBcast_mus": 14, "ES_MDAgg_AggInfo_MetaInfoBcast":{"mus":14, "nCalls":1},"ES_MDAgg_AggInfo_FixedMetaInfoGather_mus": 1971, "ES_MDAgg_AggInfo_FixedMetaInfoGather":{"mus":1971, "nCalls":1},"InitAgg-dsb_GPI_partition_mus": 40, "InitAgg-dsb_GPI_partition":{"mus":40, "nCalls":1},"InitAgg-dsb_GPI_AllGather_mus": 68, "InitAgg-dsb_GPI_AllGather":{"mus":68, "nCalls":1},"InitAgg-dsb_mus": 4417, "InitAgg-dsb":{"mus":4417, "nCalls":1},"InitAgg-dsb_GPI_mus": 118, "InitAgg-dsb_GPI":{"mus":118, "nCalls":1},"DC_mus": 239, "DC":{"mus":239, "nCalls":1},"WriteData_mus": 159534, "WriteData":{"mus":159534, "nCalls":2},"ES_WriteData_mus": 154726, "ES_WriteData":{"mus":154726, "nCalls":1},"BS_mus": 24, "BS":{"mus":24, "nCalls":1},"ES_MDAgg_AggInfo_mus": 2041, "ES_MDAgg_AggInfo":{"mus":2041, "nCalls":1},"ES_MDAgg_mus": 2385, "ES_MDAgg":{"mus":2385, "nCalls":1},"PDW_mus": 4812, "PDW":{"mus":4812, "nCalls":1},"ES_MDAgg_AggInfo_SelectMetaInfoGather_mus": 1, "ES_MDAgg_AggInfo_SelectMetaInfoGather":{"mus":1, "nCalls":1},"ES_CloseTS_mus": 125, "ES_CloseTS":{"mus":125, "nCalls":1},"ES_mus": 157395, "ES":{"mus":157395, "nCalls":1}, "databytes":0, "metadatabytes":0, "metametadatabytes":0, "transport_0":{"type":"File_POSIX", "wbytes":335544320, "close":{"mus":236, "nCalls":1}, "write":{"mus":154689, "nCalls":10}, "open":{"mus":58, "nCalls":1}} },
5+
{ "rank":3, "start":"Wed_Dec_03_22:24:41_2025","ES_DSB_Broadcast_mus": 0, "ES_DSB_Broadcast":{"mus":0, "nCalls":1},"ES_DSB_mus": 147, "ES_DSB":{"mus":147, "nCalls":1},"ES_DSB_AllGather_mus": 143, "ES_DSB_AllGather":{"mus":143, "nCalls":1},"ES_MDAgg_GatherWriteMeta_mus": 332, "ES_MDAgg_GatherWriteMeta":{"mus":332, "nCalls":1},"ES_MDAgg_AggInfo_MetaInfoBcast_mus": 12294, "ES_MDAgg_AggInfo_MetaInfoBcast":{"mus":12294, "nCalls":1},"ES_MDAgg_AggInfo_FixedMetaInfoGather_mus": 6, "ES_MDAgg_AggInfo_FixedMetaInfoGather":{"mus":6, "nCalls":1},"InitAgg-dsb_GPI_partition_mus": 41, "InitAgg-dsb_GPI_partition":{"mus":41, "nCalls":1},"InitAgg-dsb_GPI_AllGather_mus": 72, "InitAgg-dsb_GPI_AllGather":{"mus":72, "nCalls":1},"InitAgg-dsb_mus": 4417, "InitAgg-dsb":{"mus":4417, "nCalls":1},"InitAgg-dsb_GPI_mus": 122, "InitAgg-dsb_GPI":{"mus":122, "nCalls":1},"DC_mus": 202, "DC":{"mus":202, "nCalls":1},"WriteData_mus": 150070, "WriteData":{"mus":150070, "nCalls":2},"ES_WriteData_mus": 145258, "ES_WriteData":{"mus":145258, "nCalls":1},"BS_mus": 23, "BS":{"mus":23, "nCalls":1},"ES_MDAgg_AggInfo_mus": 12368, "ES_MDAgg_AggInfo":{"mus":12368, "nCalls":1},"ES_MDAgg_mus": 12726, "ES_MDAgg":{"mus":12726, "nCalls":1},"PDW_mus": 4820, "PDW":{"mus":4820, "nCalls":1},"ES_MDAgg_AggInfo_SelectMetaInfoGather_mus": 1, "ES_MDAgg_AggInfo_SelectMetaInfoGather":{"mus":1, "nCalls":1},"ES_CloseTS_mus": 136, "ES_CloseTS":{"mus":136, "nCalls":1},"ES_mus": 158279, "ES":{"mus":158279, "nCalls":1}, "databytes":0, "metadatabytes":0, "metametadatabytes":0, "transport_0":{"type":"File_POSIX", "wbytes":335544320, "close":{"mus":199, "nCalls":1}, "write":{"mus":145188, "nCalls":10}, "open":{"mus":99, "nCalls":1}} }
6+
]

‎source/utils/profiler_simplified/data/f/10.json‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
[
2+
{ "rank":0, "start":"Wed_Dec_03_22:24:44_2025","ES_DSB_Broadcast_mus": 0, "ES_DSB_Broadcast":{"mus":0, "nCalls":1},"ES_DSB_mus": 5, "ES_DSB":{"mus":5, "nCalls":1},"ES_DSB_AllGather_mus": 2, "ES_DSB_AllGather":{"mus":2, "nCalls":1},"ES_MDAgg_GatherWriteMeta_mus": 255, "ES_MDAgg_GatherWriteMeta":{"mus":255, "nCalls":1},"ES_MDAgg_AggInfo_MetaInfoBcast_mus": 2, "ES_MDAgg_AggInfo_MetaInfoBcast":{"mus":2, "nCalls":1},"ES_MDAgg_AggInfo_FixedMetaInfoGather_mus": 13428, "ES_MDAgg_AggInfo_FixedMetaInfoGather":{"mus":13428, "nCalls":1},"InitAgg-dsb_GPI_partition_mus": 14, "InitAgg-dsb_GPI_partition":{"mus":14, "nCalls":1},"InitAgg-dsb_GPI_AllGather_mus": 2, "InitAgg-dsb_GPI_AllGather":{"mus":2, "nCalls":1},"InitAgg-dsb_mus": 46, "InitAgg-dsb":{"mus":46, "nCalls":1},"InitAgg-dsb_GPI_mus": 23, "InitAgg-dsb_GPI":{"mus":23, "nCalls":1},"ES_MDAgg_GatherWriteMeta_MDBlocks_mus": 4, "ES_MDAgg_GatherWriteMeta_MDBlocks":{"mus":4, "nCalls":1},"DC_mus": 493, "DC":{"mus":493, "nCalls":1},"WriteMD_Blocks_mus": 10, "WriteMD_Blocks":{"mus":10, "nCalls":1},"WriteMD_mus": 64, "WriteMD":{"mus":64, "nCalls":1},"WriteData_mus": 150467, "WriteData":{"mus":150467, "nCalls":2},"ES_WriteData_mus": 150208, "ES_WriteData":{"mus":150208, "nCalls":1},"BS_mus": 1, "BS":{"mus":1, "nCalls":1},"ES_MDAgg_AggInfo_mus": 13693, "ES_MDAgg_AggInfo":{"mus":13693, "nCalls":1},"ES_MDAgg_mus": 13967, "ES_MDAgg":{"mus":13967, "nCalls":1},"PDW_mus": 264, "PDW":{"mus":264, "nCalls":1},"ES_MDAgg_AggInfo_SelectMetaInfoGather_mus": 3, "ES_MDAgg_AggInfo_SelectMetaInfoGather":{"mus":3, "nCalls":1},"ES_CloseTS_mus": 122, "ES_CloseTS":{"mus":122, "nCalls":1},"ES_mus": 164310, "ES":{"mus":164310, "nCalls":1},"WriteMmD_mus": 88, "WriteMmD":{"mus":88, "nCalls":1}, "databytes":0, "metadatabytes":0, "metametadatabytes":0, "transport_0":{"type":"File_POSIX", "wbytes":335544320, "close":{"mus":186, "nCalls":1}, "write":{"mus":150145, "nCalls":10}, "open":{"mus":90, "nCalls":1}}, "transport_1":{"type":"File_POSIX", "wbytes":10480, "close":{"mus":276, "nCalls":1}, "write":{"mus":54, "nCalls":5}, "open":{"mus":50, "nCalls":1}} },
3+
{ "rank":1, "start":"Wed_Dec_03_22:24:44_2025","ES_DSB_Broadcast_mus": 0, "ES_DSB_Broadcast":{"mus":0, "nCalls":1},"ES_DSB_mus": 134, "ES_DSB":{"mus":134, "nCalls":1},"ES_DSB_AllGather_mus": 131, "ES_DSB_AllGather":{"mus":131, "nCalls":1},"ES_MDAgg_GatherWriteMeta_mus": 324, "ES_MDAgg_GatherWriteMeta":{"mus":324, "nCalls":1},"ES_MDAgg_AggInfo_MetaInfoBcast_mus": 17, "ES_MDAgg_AggInfo_MetaInfoBcast":{"mus":17, "nCalls":1},"ES_MDAgg_AggInfo_FixedMetaInfoGather_mus": 3, "ES_MDAgg_AggInfo_FixedMetaInfoGather":{"mus":3, "nCalls":1},"InitAgg-dsb_GPI_partition_mus": 14, "InitAgg-dsb_GPI_partition":{"mus":14, "nCalls":1},"InitAgg-dsb_GPI_AllGather_mus": 85, "InitAgg-dsb_GPI_AllGather":{"mus":85, "nCalls":1},"InitAgg-dsb_mus": 129, "InitAgg-dsb":{"mus":129, "nCalls":1},"InitAgg-dsb_GPI_mus": 106, "InitAgg-dsb_GPI":{"mus":106, "nCalls":1},"DC_mus": 2297, "DC":{"mus":2297, "nCalls":1},"WriteData_mus": 163332, "WriteData":{"mus":163332, "nCalls":2},"ES_WriteData_mus": 162999, "ES_WriteData":{"mus":162999, "nCalls":1},"BS_mus": 2, "BS":{"mus":2, "nCalls":1},"ES_MDAgg_AggInfo_mus": 74, "ES_MDAgg_AggInfo":{"mus":74, "nCalls":1},"ES_MDAgg_mus": 411, "ES_MDAgg":{"mus":411, "nCalls":1},"PDW_mus": 337, "PDW":{"mus":337, "nCalls":1},"ES_MDAgg_AggInfo_SelectMetaInfoGather_mus": 1, "ES_MDAgg_AggInfo_SelectMetaInfoGather":{"mus":1, "nCalls":1},"ES_CloseTS_mus": 105, "ES_CloseTS":{"mus":105, "nCalls":1},"ES_mus": 163667, "ES":{"mus":163667, "nCalls":1}, "databytes":0, "metadatabytes":0, "metametadatabytes":0, "transport_0":{"type":"File_POSIX", "wbytes":335544320, "close":{"mus":2295, "nCalls":1}, "write":{"mus":162966, "nCalls":10}, "open":{"mus":114, "nCalls":1}} },
4+
{ "rank":2, "start":"Wed_Dec_03_22:24:44_2025","ES_DSB_Broadcast_mus": 0, "ES_DSB_Broadcast":{"mus":0, "nCalls":1},"ES_DSB_mus": 134, "ES_DSB":{"mus":134, "nCalls":1},"ES_DSB_AllGather_mus": 131, "ES_DSB_AllGather":{"mus":131, "nCalls":1},"ES_MDAgg_GatherWriteMeta_mus": 325, "ES_MDAgg_GatherWriteMeta":{"mus":325, "nCalls":1},"ES_MDAgg_AggInfo_MetaInfoBcast_mus": 22, "ES_MDAgg_AggInfo_MetaInfoBcast":{"mus":22, "nCalls":1},"ES_MDAgg_AggInfo_FixedMetaInfoGather_mus": 11, "ES_MDAgg_AggInfo_FixedMetaInfoGather":{"mus":11, "nCalls":1},"InitAgg-dsb_GPI_partition_mus": 14, "InitAgg-dsb_GPI_partition":{"mus":14, "nCalls":1},"InitAgg-dsb_GPI_AllGather_mus": 85, "InitAgg-dsb_GPI_AllGather":{"mus":85, "nCalls":1},"InitAgg-dsb_mus": 129, "InitAgg-dsb":{"mus":129, "nCalls":1},"InitAgg-dsb_GPI_mus": 107, "InitAgg-dsb_GPI":{"mus":107, "nCalls":1},"DC_mus": 2161, "DC":{"mus":2161, "nCalls":1},"WriteData_mus": 163467, "WriteData":{"mus":163467, "nCalls":2},"ES_WriteData_mus": 163124, "ES_WriteData":{"mus":163124, "nCalls":1},"BS_mus": 2, "BS":{"mus":2, "nCalls":1},"ES_MDAgg_AggInfo_mus": 89, "ES_MDAgg_AggInfo":{"mus":89, "nCalls":1},"ES_MDAgg_mus": 426, "ES_MDAgg":{"mus":426, "nCalls":1},"PDW_mus": 347, "PDW":{"mus":347, "nCalls":1},"ES_MDAgg_AggInfo_SelectMetaInfoGather_mus": 1, "ES_MDAgg_AggInfo_SelectMetaInfoGather":{"mus":1, "nCalls":1},"ES_CloseTS_mus": 113, "ES_CloseTS":{"mus":113, "nCalls":1},"ES_mus": 163803, "ES":{"mus":163803, "nCalls":1}, "databytes":0, "metadatabytes":0, "metametadatabytes":0, "transport_0":{"type":"File_POSIX", "wbytes":335544320, "close":{"mus":2159, "nCalls":1}, "write":{"mus":163086, "nCalls":10}, "open":{"mus":78, "nCalls":1}} },
5+
{ "rank":3, "start":"Wed_Dec_03_22:24:44_2025","ES_DSB_Broadcast_mus": 0, "ES_DSB_Broadcast":{"mus":0, "nCalls":1},"ES_DSB_mus": 134, "ES_DSB":{"mus":134, "nCalls":1},"ES_DSB_AllGather_mus": 130, "ES_DSB_AllGather":{"mus":130, "nCalls":1},"ES_MDAgg_GatherWriteMeta_mus": 325, "ES_MDAgg_GatherWriteMeta":{"mus":325, "nCalls":1},"ES_MDAgg_AggInfo_MetaInfoBcast_mus": 418, "ES_MDAgg_AggInfo_MetaInfoBcast":{"mus":418, "nCalls":1},"ES_MDAgg_AggInfo_FixedMetaInfoGather_mus": 6, "ES_MDAgg_AggInfo_FixedMetaInfoGather":{"mus":6, "nCalls":1},"InitAgg-dsb_GPI_partition_mus": 15, "InitAgg-dsb_GPI_partition":{"mus":15, "nCalls":1},"InitAgg-dsb_GPI_AllGather_mus": 85, "InitAgg-dsb_GPI_AllGather":{"mus":85, "nCalls":1},"InitAgg-dsb_mus": 129, "InitAgg-dsb":{"mus":129, "nCalls":1},"InitAgg-dsb_GPI_mus": 107, "InitAgg-dsb_GPI":{"mus":107, "nCalls":1},"DC_mus": 1793, "DC":{"mus":1793, "nCalls":1},"WriteData_mus": 162327, "WriteData":{"mus":162327, "nCalls":2},"ES_WriteData_mus": 161986, "ES_WriteData":{"mus":161986, "nCalls":1},"BS_mus": 2, "BS":{"mus":2, "nCalls":1},"ES_MDAgg_AggInfo_mus": 482, "ES_MDAgg_AggInfo":{"mus":482, "nCalls":1},"ES_MDAgg_mus": 828, "ES_MDAgg":{"mus":828, "nCalls":1},"PDW_mus": 346, "PDW":{"mus":346, "nCalls":1},"ES_MDAgg_AggInfo_SelectMetaInfoGather_mus": 1, "ES_MDAgg_AggInfo_SelectMetaInfoGather":{"mus":1, "nCalls":1},"ES_CloseTS_mus": 112, "ES_CloseTS":{"mus":112, "nCalls":1},"ES_mus": 163069, "ES":{"mus":163069, "nCalls":1}, "databytes":0, "metadatabytes":0, "metametadatabytes":0, "transport_0":{"type":"File_POSIX", "wbytes":335544320, "close":{"mus":1791, "nCalls":1}, "write":{"mus":161932, "nCalls":10}, "open":{"mus":128, "nCalls":1}} }
6+
]

0 commit comments

Comments
 (0)