workload_analytical.txt

HYBRID_TRANSFORMER_FWD_IN_BCKWD model_parallel_NPU_group: 2 ep: 16 pp: 12 vpp: 8 ga: 24 all_gpus: 9216 checkpoints: 0 checkpoint_initiates: 0 pp_comm 50331648
1789
grad_gather	-1	1	NONE	0	1	NONE	0	1	ALLGATHER	2807758848	100
grad_param_comm	-1	1	NONE	0	1	NONE	0	1	REDUCESCATTER	5615517696	100
grad_param_compute	-1	1	NONE	0	29700224	NONE	0	1	NONE	0	100
embedding_grads	-1	1	NONE	0	1	ALLREDUCE	50331648	1	NONE	0	100
moe_grad_norm1	-1	1	NONE	0	1	NONE	0	1	ALLGATHER_DP_EP	19327352832	100
moe_grad_norm2	-1	1	NONE	0	1	NONE	0	1	REDUCESCATTER_DP_EP	38654705664	100
embedding_layer	-1	622731	ALLREDUCE	50331648	1	NONE	0	15091072	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
final_column	-1	875420	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
embedding_layer	-1	622731	ALLREDUCE	50331648	1	NONE	0	875420	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
final_column	-1	875420	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
embedding_layer	-1	622731	ALLREDUCE	50331648	1	NONE	0	875420	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
final_column	-1	875420	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
embedding_layer	-1	622731	ALLREDUCE	50331648	1	NONE	0	875420	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
final_column	-1	875420	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
embedding_layer	-1	622731	ALLREDUCE	50331648	1	NONE	0	875420	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
final_column	-1	875420	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
embedding_layer	-1	622731	ALLREDUCE	50331648	1	NONE	0	875420	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
final_column	-1	875420	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
embedding_layer	-1	622731	ALLREDUCE	50331648	1	NONE	0	875420	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
final_column	-1	875420	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
embedding_layer	-1	622731	ALLREDUCE	50331648	1	NONE	0	875420	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
final_column	-1	875420	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
embedding_layer	-1	622731	ALLREDUCE	50331648	1	NONE	0	875420	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
final_column	-1	875420	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
embedding_layer	-1	622731	ALLREDUCE	50331648	1	NONE	0	875420	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
final_column	-1	875420	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
embedding_layer	-1	622731	ALLREDUCE	50331648	1	NONE	0	875420	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
final_column	-1	875420	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
embedding_layer	-1	622731	ALLREDUCE	50331648	1	NONE	0	875420	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
final_column	-1	875420	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
embedding_layer	-1	622731	ALLREDUCE	50331648	1	NONE	0	875420	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
final_column	-1	875420	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
embedding_layer	-1	622731	ALLREDUCE	50331648	1	NONE	0	875420	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
final_column	-1	875420	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
embedding_layer	-1	622731	ALLREDUCE	50331648	1	NONE	0	875420	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
final_column	-1	875420	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
embedding_layer	-1	622731	ALLREDUCE	50331648	1	NONE	0	875420	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
final_column	-1	875420	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
embedding_layer	-1	622731	ALLREDUCE	50331648	1	NONE	0	875420	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
final_column	-1	875420	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
embedding_layer	-1	622731	ALLREDUCE	50331648	1	NONE	0	875420	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
final_column	-1	875420	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
embedding_layer	-1	622731	ALLREDUCE	50331648	1	NONE	0	875420	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
final_column	-1	875420	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
embedding_layer	-1	622731	ALLREDUCE	50331648	1	NONE	0	875420	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
final_column	-1	875420	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
embedding_layer	-1	622731	ALLREDUCE	50331648	1	NONE	0	875420	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
final_column	-1	875420	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
embedding_layer	-1	622731	ALLREDUCE	50331648	1	NONE	0	875420	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
final_column	-1	875420	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
embedding_layer	-1	622731	ALLREDUCE	50331648	1	NONE	0	875420	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
final_column	-1	875420	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
embedding_layer	-1	622731	ALLREDUCE	50331648	1	NONE	0	875420	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
attention_column	-1	1750840	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
attention_row	-1	1750840	REDUCESCATTER	50331648	875420	ALLGATHER	50331648	875420	NONE	0	100
mlp_moelayer	-1	7956367	ALLGATHER	1572864	7956367	ALLGATHER	1572864	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLGATHER	201326592	1	REDUCESCATTER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	REDUCESCATTER	201326592	1	ALLGATHER	201326592	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL_EP	100663296	1	ALLTOALL_EP	100663296	1	NONE	0	100
mlp_moelayer	-1	1	ALLTOALL	25165824	1	ALLTOALL	25165824	1	NONE	0	100
final_column	-1	875420	ALLGATHER	50331648	875420	REDUCESCATTER	0	875420	NONE	0	100
cross_entropy1	-1	0	ALLREDUCE	16384	0	NONE	0	0	NONE	0	100
cross_entropy2	-1	0	ALLREDUCE	16384	0	NONE	0	0	NONE	0	100
cross_entropy3	-1	0	ALLREDUCE	16384	0	NONE	0	0	NONE	0	100
optimizer1	-1	0	ALLREDUCE	4	0	NONE	0	0	NONE	0	100
optimizer2	-1	0	ALLREDUCE	4	0	NONE	0	0	NONE	0	100
optimizer3	-1	0	ALLREDUCE	4	0	NONE	0	0	NONE	0	100
optimizer4	-1	0	ALLREDUCE	4	0	NONE	0	0	NONE	0	100