reverted all changes and enabled logs of check_efa_nccl_all_reduce

Author: Bhanu Teja Goshikonda

dlc_developer_config.toml

@@ -37,16 +37,16 @@ deep_canary_mode = false
 [build]
 # Add in frameworks you would like to build. By default, builds are disabled unless you specify building an image.
 # available frameworks - ["base", "vllm", "autogluon", "huggingface_tensorflow", "huggingface_pytorch", "huggingface_tensorflow_trcomp", "huggingface_pytorch_trcomp", "pytorch_trcomp", "tensorflow", "pytorch", "stabilityai_pytorch"]
-build_frameworks = ["tensorflow"]
+build_frameworks = []
 
 
 # By default we build both training and inference containers. Set true/false values to determine which to build.
 build_training = true
-build_inference = false
+build_inference = true
 
 # Set do_build to "false" to skip builds and test the latest image built by this PR
 # Note: at least one build is required to set do_build to "false"
-do_build = false
+do_build = true
 
 [notify]
 ### Notify on test failures
@@ -57,24 +57,24 @@ notify_test_failures = false
 
 [test]
 ### On by default
-sanity_tests = false
-security_tests = false
+sanity_tests = true
+security_tests = true
   safety_check_test = false
   ecr_scan_allowlist_feature = false
-ecs_tests = false
-eks_tests = false
+ecs_tests = true
+eks_tests = true
 ec2_tests = true
 # Set it to true if you are preparing a Benchmark related PR
-ec2_benchmark_tests = true
+ec2_benchmark_tests = false
 
 ### Set ec2_tests_on_heavy_instances = true to be able to run any EC2 tests that use large/expensive instance types by
 ### default. If false, these types of tests will be skipped while other tests will run as usual.
 ### These tests are run in EC2 test jobs, so ec2_tests must be true if ec2_tests_on_heavy_instances is true.
 ### Off by default (set to false)
-ec2_tests_on_heavy_instances = true
+ec2_tests_on_heavy_instances = false
 ### SM specific tests
 ### On by default
-sagemaker_local_tests = false
+sagemaker_local_tests = true
 ### Set enable_ipv6 = true to run tests with IPv6-enabled resources
 ### Off by default (set to false)
 enable_ipv6 = false
@@ -92,13 +92,13 @@ enable_ipv6 = false
 ipv6_vpc_name = ""
 
 # run standard sagemaker remote tests from test/sagemaker_tests
-sagemaker_remote_tests = false
+sagemaker_remote_tests = true
 # run efa sagemaker tests
-sagemaker_efa_tests = true
+sagemaker_efa_tests = false
 # run release_candidate_integration tests
-sagemaker_rc_tests = true
+sagemaker_rc_tests = false
 # run sagemaker benchmark tests
-sagemaker_benchmark_tests = true
+sagemaker_benchmark_tests = false
 
 # SM remote EFA test instance type
 sagemaker_remote_efa_instance_type = ""
@@ -123,7 +123,7 @@ dlc-pr-base = ""
 
 # Standard Framework Training
 dlc-pr-pytorch-training = ""
-dlc-pr-tensorflow-2-training = "tensorflow/training/buildspec-2-18-sm.yml"
+dlc-pr-tensorflow-2-training = ""
 dlc-pr-autogluon-training = ""
 
 # ARM64 Training

tensorflow/training/docker/2.18/py3/cu125/Dockerfile.gpu

@@ -237,7 +237,7 @@ RUN mkdir /tmp/efa-ofi-nccl \
  && ./configure --with-libfabric=/opt/amazon/efa \
     --with-mpi=/opt/amazon/openmpi \
     --with-cuda=/usr/local/cuda \
-    --with-nccl=/usr \
+    --with-nccl=/usr/local \
     --prefix=/usr/local \
     --disable-tests \
  && make -j $(nproc) \

test/dlc_tests/container_tests/bin/efa/testEFA

@@ -89,7 +89,7 @@ check_efa_nccl_all_reduce(){
 
     RETURN_VAL=${PIPESTATUS[0]}
     # In case, if you would like see logs, uncomment below line
-    # RESULT=$(cat ${TRAINING_LOG})
+    RESULT=$(cat ${TRAINING_LOG})
 
     if [ ${RETURN_VAL} -eq 0 ]; then
         echo "***************************** check_efa_nccl_all_reduce passed *****************************"