diff --git a/.github/scripts/nova_dir.bash b/.github/scripts/nova_dir.bash
index 262a51c633..deb534a9b1 100644
--- a/.github/scripts/nova_dir.bash
+++ b/.github/scripts/nova_dir.bash
@@ -13,9 +13,6 @@ if [[ "$working_dir" == "$FBGEMM_REPO" ]]; then cd fbgemm_gpu || echo "Failed to
 ## Build clean/wheel will be done in pre-script. Set flag such that setup.py will skip these steps in Nova workflow
 export BUILD_FROM_NOVA=1
 
-## TODO (huydhn): Remove this when Nova is moving to manywheel 2.28
-export SET_GLIBCXX_PRELOAD=1
-
 ## Overwrite existing ENV VAR in Nova
 if [[ "$CONDA_ENV" != "" ]]; then export CONDA_RUN="conda run --no-capture-output -p ${CONDA_ENV}" && echo "$CONDA_RUN"; fi
 
diff --git a/.github/scripts/nova_prescript.bash b/.github/scripts/nova_prescript.bash
index 315befb009..1c333db954 100644
--- a/.github/scripts/nova_prescript.bash
+++ b/.github/scripts/nova_prescript.bash
@@ -42,13 +42,6 @@ runtime=$((end_time-start_time))
 start_time=${end_time}
 echo "[NOVA] Time taken to display GPU Info: ${runtime} seconds"
 
-# Install C/C++ Compilers
-install_cxx_compiler "${BUILD_ENV_NAME}"
-end_time=$(date +%s)
-runtime=$((end_time-start_time))
-start_time=${end_time}
-echo "[NOVA] Time taken to install C/C++ Compilers: ${runtime} seconds"
-
 # Install Build Tools
 install_build_tools "${BUILD_ENV_NAME}"
 end_time=$(date +%s)
@@ -64,13 +57,15 @@ start_time=${end_time}
 echo "[NOVA] Time taken to collect PyTorch environment information: ${runtime} seconds"
 
 if [[ $CU_VERSION = cu* ]]; then
-  # Extract the CUDA version number from CU_VERSION
-  cuda_version=$(echo "[NOVA] ${CU_VERSION}" | cut -c 3-)
-  install_cudnn "${BUILD_ENV_NAME}" "$(pwd)/build_only/cudnn" "${cuda_version}"
-  end_time=$(date +%s)
-  runtime=$((end_time-start_time))
-  start_time=${end_time}
-  echo "[NOVA] Time taken to install cudnn: ${runtime} seconds"
+  # shellcheck disable=SC2155
+  env_prefix=$(env_name_or_prefix "${BUILD_ENV_NAME}")
+
+  echo "[INSTALL] Set environment variables LD_LIBRARY_PATH ..."
+  # shellcheck disable=SC2086
+  print_exec conda env config vars set ${env_prefix} \
+    LD_LIBRARY_PATH="/usr/local/lib:${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}" \
+    CUDNN_INCLUDE_DIR="${CUDA_HOME}/include" \
+    CUDNN_LIBRARY="${CUDA_HOME}/lib64"
 
   echo "[NOVA] -------- Finding NVML_LIB_PATH -----------"
   if [[ ${NVML_LIB_PATH} == "" ]]; then