enable async save for functional tests

tests/functional_tests/test_cases/gpt/gpt3_15b_8t_release/model_config.yaml

@@ -90,6 +90,7 @@ MODEL_ARGS:
   --bf16: true
   --exit-interval: 51000
   --wandb-save-dir: ${WANDB_SAVE_PATH}
+  --async-save: true
 METRICS:
   - "iteration-time"
   - "lm loss"

tests/functional_tests/test_cases/gpt/gpt3_15b_8t_release_gb200/model_config.yaml

@@ -93,6 +93,7 @@ MODEL_ARGS:
   --bf16: true
   --exit-interval: 51000
   --wandb-save-dir: ${WANDB_SAVE_PATH}
+  --async-save: true
 METRICS:
   - "iteration-time"
   - "lm loss"

tests/functional_tests/test_cases/gpt/gpt3_15b_8t_release_sm/model_config.yaml

@@ -90,6 +90,7 @@ MODEL_ARGS:
   --bf16: true
   --exit-interval: 13000
   --wandb-save-dir: ${WANDB_SAVE_PATH}
+  --async-save: true
 METRICS:
   - "iteration-time"
   - "lm loss"

tests/functional_tests/test_cases/gpt/gpt3_15b_8t_release_sm_gb200/model_config.yaml

@@ -93,6 +93,7 @@ MODEL_ARGS:
   --bf16: true
   --exit-interval: 13000
   --wandb-save-dir: ${WANDB_SAVE_PATH}
+  --async-save: true
 METRICS:
   - "iteration-time"
   - "lm loss"

tests/functional_tests/test_cases/gpt/gpt3_7b_tp1_pp4_memory_speed/model_config.yaml

@@ -58,6 +58,7 @@ MODEL_ARGS:
   --bf16: true
   --log-memory-to-tensorboard: true
   --exit-interval: 25
+  --async-save: true
 TEST_TYPE: regular
 METRICS:
   - "iteration-time"

tests/functional_tests/test_cases/gpt/gpt3_7b_tp4_pp1_memory_speed/model_config.yaml

@@ -56,6 +56,7 @@ MODEL_ARGS:
   --data-cache-path: ${DATA_CACHE_PATH}
   --bf16: true
   --log-memory-to-tensorboard: true
+  --async-save: true
 TEST_TYPE: regular
 METRICS:
   - "iteration-time"

tests/functional_tests/test_cases/gpt/gpt3_mcore_reruns_disable/model_config.yaml

@@ -77,6 +77,7 @@ MODEL_ARGS:
   --log-memory-to-tensorboard: true
   # rerun settings
   --rerun-mode: disabled
+  --async-save: true
 AFTER_SCRIPT: |
   check_log() { if [[ -z $(grep -r $1 "$2" $LOG_DIR) ]]; then exit 1; else echo OK; fi }
   check_log_not() { if [[ -z $(grep -r $1 "$2" $LOG_DIR) ]]; then echo OK; else exit 1; fi }

tests/functional_tests/test_cases/gpt/gpt3_mcore_reruns_enable/model_config.yaml

@@ -76,6 +76,7 @@ MODEL_ARGS:
   --log-memory-to-tensorboard: true
   # rerun settings
   --rerun-mode: validate_results
+  --async-save: true
 AFTER_SCRIPT: |
   check_log() { if [[ -z $(grep -r $1 "$2" $LOG_DIR) ]]; then exit 1; else echo OK; fi }
   check_log -F "WARNING:megatron.core.rerun_state_machine:Result validation enabled"

tests/functional_tests/test_cases/gpt/gpt3_mcore_reruns_persistent_1/model_config.yaml

@@ -78,6 +78,7 @@ MODEL_ARGS:
   --rerun-mode: validate_results
   --error-injection-rate: 100
   --error-injection-type: persistent_error
+  --async-save: true
 AFTER_SCRIPT: |
   check_log() { if [[ -z $(grep -r $1 "$2" $LOG_DIR) ]]; then exit 1; else echo OK; fi }
   check_log -F "Result validation enabled"

tests/functional_tests/test_cases/gpt/gpt3_mcore_reruns_persistent_2/model_config.yaml

@@ -76,6 +76,7 @@ MODEL_ARGS:
   --log-memory-to-tensorboard: true
   # rerun settings
   --rerun-mode: validate_results
+  --async-save: true
 AFTER_SCRIPT: |
   check_log() { if [[ -z $(grep -r $1 "$2" $LOG_DIR) ]]; then exit 1; else echo OK; fi }
   check_log -F "WARNING:megatron.core.rerun_state_machine:Result validation enabled"

tests/functional_tests/test_cases/gpt/gpt3_mcore_reruns_reshard/model_config.yaml

@@ -77,6 +77,7 @@ MODEL_ARGS:
   --log-memory-to-tensorboard: true
   # rerun settings
   --rerun-mode: validate_results
+  --async-save: true
 AFTER_SCRIPT: |
   check_log() { if [[ -z $(grep -r $1 "$2" $LOG_DIR) ]]; then exit 1; else echo OK; fi }
   check_log -F "WARNING:megatron.core.rerun_state_machine:Result validation enabled"

tests/functional_tests/test_cases/gpt/gpt3_mcore_reruns_resume/model_config.yaml

@@ -76,6 +76,7 @@ MODEL_ARGS:
   --log-memory-to-tensorboard: true
   # rerun settings
   --rerun-mode: validate_results
+  --async-save: true
 AFTER_SCRIPT: |
   check_log() { if [[ -z $(grep -r $1 "$2" $LOG_DIR) ]]; then exit 1; else echo OK; fi }
   check_log -F "successfully loaded checkpoint"

tests/functional_tests/test_cases/gpt/gpt3_mcore_reruns_resume_check_grads/model_config.yaml

@@ -88,6 +88,7 @@ BASE_MODEL_ARGS: &BASE_MODEL_ARGS
   --dist-ckpt-optim-fully-reshardable: true
   --save: ${CHECKPOINT_SAVE_PATH}
   --load: ${CHECKPOINT_LOAD_PATH}/model/mcore_gpt/gpt3_4b_pyt/25.03.05_bf16_rerun-enabled_v2
+  --async-save: true
   # data settings
   --data-cache-path: ${DATA_CACHE_PATH}
   --data-path: ${DATA_PATH}/text/the_pile/shard00/my-gpt3_00_text_document

tests/functional_tests/test_cases/gpt/gpt3_mcore_reruns_transient/model_config.yaml

@@ -79,6 +79,7 @@ MODEL_ARGS:
   --rerun-mode: validate_results
   --error-injection-rate: 100
   --error-injection-type: transient_error
+  --async-save: true
 AFTER_SCRIPT: |
   check_log() { if [[ -z $(grep -r $1 "$2" $LOG_DIR) ]]; then exit 1; else echo OK; fi }
   check_log -F "WARNING:megatron.core.rerun_state_machine:Result validation enabled"

tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_fim_dataset/model_config.yaml

@@ -53,4 +53,5 @@ MODEL_ARGS:
   --fim-data: true
   --fim-rate: 0.5
   --fim-spm-rate: 0.5
+  --async-save: true
 TEST_TYPE: regular

tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files/model_config.yaml

@@ -51,4 +51,5 @@ MODEL_ARGS:
   --bf16: true
   --attention-backend: unfused
   --log-memory-to-tensorboard: true
+  --async-save: true
 TEST_TYPE: regular

tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_mup/model_config.yaml

@@ -54,4 +54,5 @@ MODEL_ARGS:
   # MuP (Maximal Update Parameterization) args
   --use-mup: true
   --mup-base-hidden-size: 256
+  --async-save: true
 TEST_TYPE: regular

tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_dist_optimizer_no_mmap_bin_files/model_config.yaml

@@ -52,4 +52,5 @@ MODEL_ARGS:
   --data-cache-path: ${DATA_CACHE_PATH}
   --bf16: true
   --log-memory-to-tensorboard: true
+  --async-save: true
 TEST_TYPE: ckpt-resume

tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute/model_config.yaml

@@ -52,4 +52,5 @@ MODEL_ARGS:
   --data-cache-path: ${DATA_CACHE_PATH}
   --bf16: true
   --log-memory-to-tensorboard: true
+  --async-save: true
 TEST_TYPE: ckpt-resume

tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_uniform_full_recompute/model_config.yaml

@@ -53,4 +53,5 @@ MODEL_ARGS:
   --bf16: true
   --attention-backend: unfused
   --log-memory-to-tensorboard: true
+  --async-save: true
 TEST_TYPE: regular

tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_cp4_a2a_p2p_nondeterministic/model_config.yaml

@@ -52,4 +52,5 @@ MODEL_ARGS:
   --bf16: true
   --attention-backend: flash
   --log-memory-to-tensorboard: true
+  --async-save: true
 TEST_TYPE: regular

tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_cp4_a2a_p2p_nondeterministic/model_config.yaml

@@ -53,4 +53,5 @@ MODEL_ARGS:
   --bf16: true
   --attention-backend: flash
   --log-memory-to-tensorboard: true
+  --async-save: true
 TEST_TYPE: ckpt-resume

tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings/model_config.yaml

@@ -51,4 +51,5 @@ MODEL_ARGS:
   --data-cache-path: ${DATA_CACHE_PATH}
   --bf16: true
   --log-memory-to-tensorboard: true
+  --async-save: true
 TEST_TYPE: ckpt-resume

tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion/model_config.yaml

@@ -52,4 +52,5 @@ MODEL_ARGS:
   --data-cache-path: ${DATA_CACHE_PATH}
   --bf16: true
   --log-memory-to-tensorboard: true
+  --async-save: true
 TEST_TYPE: ckpt-resume

tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_rope_embeddings/model_config.yaml

@@ -51,4 +51,5 @@ MODEL_ARGS:
   --bf16: true
   --attention-backend: unfused
   --log-memory-to-tensorboard: true
+  --async-save: true
 TEST_TYPE: regular

tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_rope_embeddings_interleaved_no_fusion/model_config.yaml

@@ -53,4 +53,5 @@ MODEL_ARGS:
   --bf16: true
   --attention-backend: unfused
   --log-memory-to-tensorboard: true
+  --async-save: true
 TEST_TYPE: regular

tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_sequence_parallel/model_config.yaml

@@ -51,4 +51,5 @@ MODEL_ARGS:
   --data-cache-path: ${DATA_CACHE_PATH}
   --bf16: true
   --log-memory-to-tensorboard: true
+  --async-save: true
 TEST_TYPE: ckpt-resume

tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs/model_config.yaml

@@ -50,4 +50,5 @@ MODEL_ARGS:
   --data-cache-path: ${DATA_CACHE_PATH}
   --bf16: true
   --log-memory-to-tensorboard: true
+  --async-save: true
 TEST_TYPE: ckpt-resume

tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_sequence_parallel/model_config.yaml

@@ -52,4 +52,5 @@ MODEL_ARGS:
   --bf16: true
   --attention-backend: unfused
   --log-memory-to-tensorboard: true
+  --async-save: true
 TEST_TYPE: regular

tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_untie_embeddings_and_outputs/model_config.yaml

@@ -50,4 +50,5 @@ MODEL_ARGS:
   --bf16: true
   --attention-backend: unfused
   --log-memory-to-tensorboard: true
+  --async-save: true
 TEST_TYPE: regular

tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1/model_config.yaml

@@ -52,4 +52,5 @@ MODEL_ARGS:
   --bf16: true
   --attention-backend: unfused
   --log-memory-to-tensorboard: true
+  --async-save: true
 TEST_TYPE: regular

tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_calculate_per_token_loss/model_config.yaml

@@ -51,4 +51,5 @@ MODEL_ARGS:
   --bf16: true
   --attention-backend: unfused
   --log-memory-to-tensorboard: true
+  --async-save: true
 TEST_TYPE: regular

tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce/model_config.yaml

@@ -52,4 +52,5 @@ MODEL_ARGS:
   --bf16: true
   --attention-backend: unfused
   --log-memory-to-tensorboard: true
+  --async-save: true
 TEST_TYPE: regular

tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather/model_config.yaml

@@ -57,4 +57,5 @@ MODEL_ARGS:
   --bf16: true
   --attention-backend: unfused
   --log-memory-to-tensorboard: true
+  --async-save: true
 TEST_TYPE: regular

tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_untied/model_config.yaml

@@ -53,4 +53,5 @@ MODEL_ARGS:
   --bf16: true
   --attention-backend: unfused
   --log-memory-to-tensorboard: true
+  --async-save: true
 TEST_TYPE: regular

tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist/model_config.yaml

@@ -52,4 +52,5 @@ MODEL_ARGS:
   --data-cache-path: ${DATA_CACHE_PATH}
   --bf16: true
   --log-memory-to-tensorboard: true
+  --async-save: true
 TEST_TYPE: regular # Usually ckpt-resume, but as a WAR to #513 set to regular

tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss/model_config.yaml

@@ -52,4 +52,5 @@ MODEL_ARGS:
   --data-cache-path: ${DATA_CACHE_PATH}
   --bf16: true
   --log-memory-to-tensorboard: true
+  --async-save: true
 TEST_TYPE: ckpt-resume

tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/model_config.yaml

@@ -52,4 +52,5 @@ MODEL_ARGS:
   --data-cache-path: ${DATA_CACHE_PATH}
   --bf16: true
   --log-memory-to-tensorboard: true
+  --async-save: true
 TEST_TYPE: ckpt-resume

tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/model_config.yaml

@@ -58,4 +58,5 @@ MODEL_ARGS:
   --data-cache-path: ${DATA_CACHE_PATH}
   --bf16: true
   --log-memory-to-tensorboard: true
+  --async-save: true
 TEST_TYPE: regular # Usually ckpt-resume, but as a WAR to #513 set to regular

tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_untied/model_config.yaml

@@ -55,4 +55,5 @@ MODEL_ARGS:
   --data-cache-path: ${DATA_CACHE_PATH}
   --bf16: true
   --log-memory-to-tensorboard: true
+  --async-save: true
 TEST_TYPE: regular # Usually ckpt-resume, but as a WAR to #513 set to regular

tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_tunable_overlap/model_config.yaml

@@ -53,4 +53,5 @@ MODEL_ARGS:
   --data-cache-path: ${DATA_CACHE_PATH}
   --bf16: true
   --log-memory-to-tensorboard: true
+  --async-save: true
 TEST_TYPE: regular # Usually ckpt-resume, but as a WAR to #513 set to regular

tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_tunable_overlap/model_config.yaml

@@ -54,4 +54,5 @@ MODEL_ARGS:
   --bf16: true
   --attention-backend: unfused
   --log-memory-to-tensorboard: true
+  --async-save: true
 TEST_TYPE: regular

tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_uneven_pipeline/model_config.yaml

@@ -52,4 +52,5 @@ MODEL_ARGS:
   --decoder-last-pipeline-num-layers: 2
   --attention-backend: unfused
   --log-memory-to-tensorboard: true
+  --async-save: true
 TEST_TYPE: regular

tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp2_account_for_embedding_loss_in_pipeline_split/model_config.yaml

@@ -53,4 +53,5 @@ MODEL_ARGS:
   --account-for-loss-in-pipeline-split: true
   --attention-backend: unfused
   --log-memory-to-tensorboard: true
+  --async-save: true
 TEST_TYPE: regular

tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_cp2_nondeterministic/model_config.yaml

@@ -51,4 +51,5 @@ MODEL_ARGS:
   --bf16: true
   --attention-backend: flash
   --log-memory-to-tensorboard: true
+  --async-save: true
 TEST_TYPE: regular

tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_frozen_resume_torch_dist_cp2_nondeterministic/model_config.yaml

@@ -51,4 +51,5 @@ MODEL_ARGS:
   --bf16: true
   --attention-backend: flash
   --log-memory-to-tensorboard: true
+  --async-save: true
 TEST_TYPE: frozen-resume

tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_fsdp2_resume_torch_dist/model_config.yaml

@@ -52,4 +52,5 @@ MODEL_ARGS:
   --data-cache-path: ${DATA_CACHE_PATH}
   --bf16: true
   --log-memory-to-tensorboard: true
+  --async-save: true
 TEST_TYPE: ckpt-resume

tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_gdn/model_config.yaml

@@ -79,4 +79,5 @@ MODEL_ARGS:
   --bf16: true
   --attention-backend: unfused
   --log-memory-to-tensorboard: true
+  --async-save: true
 TEST_TYPE: ckpt-resume

tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_modelopt_distill_resume/model_config.yaml

@@ -66,4 +66,5 @@ MODEL_ARGS:
   --bf16: true
   --attention-backend: unfused
   --log-memory-to-tensorboard: true
+  --async-save: true
 TEST_TYPE: ckpt-resume

tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_multi_dist_optimizer_instances/model_config.yaml

@@ -53,4 +53,5 @@ MODEL_ARGS:
   --bf16: true
   --attention-backend: unfused
   --log-memory-to-tensorboard: true
+  --async-save: true
 TEST_TYPE: regular

tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_resume_torch_dist_cp2_nondeterministic/model_config.yaml

@@ -52,4 +52,5 @@ MODEL_ARGS:
   --bf16: true
   --attention-backend: flash
   --log-memory-to-tensorboard: true
+  --async-save: true
 TEST_TYPE: ckpt-resume

tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances/model_config.yaml

@@ -54,4 +54,5 @@ MODEL_ARGS:
   --data-cache-path: ${DATA_CACHE_PATH}
   --bf16: true
   --log-memory-to-tensorboard: true
+  --async-save: true
 TEST_TYPE: ckpt-resume

tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2/model_config.yaml

@@ -50,4 +50,5 @@ MODEL_ARGS:
   --bf16: true
   --attention-backend: unfused
   --log-memory-to-tensorboard: true
+  --async-save: true
 TEST_TYPE: regular

tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2/model_config.yaml

@@ -50,4 +50,5 @@ MODEL_ARGS:
   --bf16: true
   --attention-backend: flash
   --log-memory-to-tensorboard: true
+  --async-save: true
 TEST_TYPE: regular

tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_calculate_per_token_loss/model_config.yaml

@@ -50,4 +50,5 @@ MODEL_ARGS:
   --attention-backend: flash
   --log-memory-to-tensorboard: true
   --calculate-per-token-loss: true
+  --async-save: true
 TEST_TYPE: regular

tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_calculate_per_token_loss_nondeterministic/model_config.yaml

@@ -50,4 +50,5 @@ MODEL_ARGS:
   --attention-backend: flash
   --log-memory-to-tensorboard: true
   --calculate-per-token-loss: true
+  --async-save: true
 TEST_TYPE: regular

tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_etp4_calculate_per_token_loss_dp_last/model_config.yaml

@@ -53,4 +53,5 @@ MODEL_ARGS:
   --use-tp-pp-dp-mapping: true
   --expert-tensor-parallel-size: 4
   --calculate-per-token-loss: true
+  --async-save: true
 TEST_TYPE: regular

tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_etp4_calculate_per_token_loss_nondeterministic_dp_last/model_config.yaml

@@ -52,4 +52,5 @@ MODEL_ARGS:
   --use-tp-pp-dp-mapping: true
   --expert-tensor-parallel-size: 4
   --calculate-per-token-loss: true
+  --async-save: true
 TEST_TYPE: regular

tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_etp4_dp_last/model_config.yaml

@@ -52,4 +52,5 @@ MODEL_ARGS:
   --log-memory-to-tensorboard: true
   --use-tp-pp-dp-mapping: true
   --expert-tensor-parallel-size: 4
+  --async-save: true
 TEST_TYPE: regular

tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_etp4_nondeterministic_dp_last/model_config.yaml

@@ -52,4 +52,5 @@ MODEL_ARGS:
   --log-memory-to-tensorboard: true
   --use-tp-pp-dp-mapping: true
   --expert-tensor-parallel-size: 4
+  --async-save: true
 TEST_TYPE: regular

tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_nondeterministic/model_config.yaml

@@ -51,4 +51,5 @@ MODEL_ARGS:
   --bf16: true
   --attention-backend: flash
   --log-memory-to-tensorboard: true
+  --async-save: true
 TEST_TYPE: regular

tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cross_entropy_loss_fusion/model_config.yaml

@@ -46,4 +46,5 @@ MODEL_ARGS:
   --data-cache-path: ${DATA_CACHE_PATH}
   --bf16: true
   --log-memory-to-tensorboard: true
+  --async-save: true
 TEST_TYPE: regular