Check tf218 training efa test (#4504)

* check tf218 efa test

* enable efa tests

* change AWS Libfabric grep and exit message if error exists

* match EFA log change for p4d and p5

* add --fix-missing to fix cpu test

* revert toml

* revert toml

---------

Co-authored-by: Yadan Wei <yadanwei@amazon.com>

Author: Yadan-Wei

test/dlc_tests/container_tests/bin/efa/testEFA

@@ -29,10 +29,12 @@ fi
 validate_all_reduce_performance_logs(){
     grep "aws-ofi-nccl" ${TRAINING_LOG} || { echo "aws-ofi-nccl is not working, please check if it is installed correctly"; exit 1; }
     grep "NET/OFI Selected Provider is efa" ${TRAINING_LOG} || { echo "efa is not working, please check if it is installed correctly"; exit 1; }
-    grep "Using network AWS Libfabric" ${TRAINING_LOG}
+    # EFA 1.37.0 using "Using network Libfabric" instead of "Using network AWS Libfabric"
+    grep -E "Using network (AWS )?Libfabric" ${TRAINING_LOG} || { echo "efa is not working, please check if it is installed correctly"; exit 1; }
     if [[ ${INSTANCE_TYPE} == p4d* || ${INSTANCE_TYPE} == p5* ]]; then
         grep "Setting NCCL_TOPO_FILE environment variable to" ${TRAINING_LOG}
-        grep "NET/AWS Libfabric/0/GDRDMA" ${TRAINING_LOG}
+        # EFA 1.37.0 change from NET/AWS Libfabric/0/GDRDMA to NET/Libfabric/0/GDRDMA
+        grep -E "NET/(AWS )?Libfabric/0/GDRDMA" ${TRAINING_LOG}
     fi
 }
 

test/dlc_tests/ec2/test_efa.py

@@ -103,7 +103,7 @@ def test_pytorch_efa(
 @pytest.mark.multinode(2)
 @pytest.mark.parametrize("ec2_instance_type,region", EC2_EFA_GPU_INSTANCE_TYPE_AND_REGION)
 @pytest.mark.skipif(
-    is_pr_context() and not is_efa_dedicated(),
+    is_pr_context() and not is_efa_dedicated() and not are_heavy_instance_ec2_tests_enabled(),
     reason="Skip EFA test in PR context unless explicitly enabled",
 )
 @pytest.mark.team("frameworks")

test/test_utils/ec2.py

@@ -1666,7 +1666,7 @@ def install_python_in_instance(context, python_version="3.9"):
         (
             "sudo apt install -y make build-essential libssl-dev zlib1g-dev "
             "libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm "
-            "libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev"
+            "libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev --fix-missing"
         ),
         hide=True,
     )