Merge pull request #5 from wheelos/cpu_docker

fix: fix deps missing in cpu docker image - add install_proj - add in…

Author: daohu527

README.md

@@ -100,9 +100,9 @@ Build the entire Apollo project with:
 To build a single module, use:
 
 ```bash
-./apollo.sh build <module_name>
+./apollo.sh build_cpu <module_name>
 # example:
-./apollo.sh build planning
+./apollo.sh build_cpu planning
 ```
 
 ---

docker/build/installers/install_libtorch.sh

@@ -21,58 +21,120 @@ set -e
 CURR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
 . ${CURR_DIR}/installer_base.sh
 
-# TODO(build): Docs on how to build libtorch on Jetson boards
-# References:
-#   https://github.com/ApolloAuto/apollo/blob/pre6/docker/build/installers/install_libtorch.sh
-#   https://github.com/dusty-nv/jetson-containers/blob/master/Dockerfile.pytorch
-#   https://forums.developer.nvidia.com/t/pytorch-for-jetson-version-1-6-0-now-available
-#   https://github.com/pytorch/pytorch/blob/master/docker/caffe2/ubuntu-16.04-cpu-all-options/Dockerfile
-bash ${CURR_DIR}/install_mkl.sh
+# --- Unified Version Control ---
+# Manage all Pytorch related component versions here
+PYTORCH_VERSION="2.6.0"
+CHECKSUM=""6887b5186e466a6d5ca044a51d083bb03c48cb1b4952059b7ca51a5398fbafcc""
+TORCHVISION_VERSION="0.18.0" # Note: Version must be compatible with Pytorch
+TORCHAUDIO_VERSION="2.4.0"  # Note: Version must be compatible with Pytorch
 
+# --- Environment Detection ---
 TARGET_ARCH="$(uname -m)"
+CUDA_SUPPORT=false
+CUDA_VERSION_STR="" # e.g., "11.8"
+CUDA_VERSION_TAG="" # e.g., "cu118"
 
-##============================================================##
-# libtorch_cpu
-
-if [[ "${TARGET_ARCH}" == "x86_64" ]]; then
-    # https://download.pytorch.org/libtorch/cpu/libtorch-cxx11-abi-shared-with-deps-1.5.0%2Bcpu.zip
-    VERSION="1.7.0-2"
-    CHECKSUM="02fd4f30e97ce8911ef933d0516660892392e95e6768b50f591f4727f6224390"
-elif [[ "${TARGET_ARCH}" == "aarch64" ]]; then
-    VERSION="1.6.0-1"
-    CHECKSUM="6d1fba522e746213c209fbf6275fa6bac68e360bcd11cbd4d3bdbddb657bee82"
-else
-    error "libtorch for ${TARGET_ARCH} not ready. Exiting..."
-    exit 1
+if [ "${TARGET_ARCH}" = "x86_64" ] && command -v nvcc >/dev/null 2>&1; then
+  # Get CUDA major and minor version (e.g., 11.8)
+  CUDA_VERSION_STR=$(nvcc --version | sed -n 's/.*release \([0-9]*\.[0-9]*\).*/\1/p')
+  if [ -n "${CUDA_VERSION_STR}" ]; then
+    # Format version for Pytorch whl URL (e.g., 11.8 -> cu118)
+    CUDA_VERSION_TAG="cu$(echo ${CUDA_VERSION_STR} | sed 's/\.//g')"
+    CUDA_SUPPORT=true
+    ok "Found CUDA ${CUDA_VERSION_STR}. PyTorch will be installed with GPU support."
+  else
+    warn "nvcc found, but could not determine CUDA version. Falling back to CPU."
+  fi
 fi
 
-PKG_NAME="libtorch_cpu-${VERSION}-linux-${TARGET_ARCH}.tar.gz"
-DOWNLOAD_LINK="https://apollo-system.cdn.bcebos.com/archive/6.0/${PKG_NAME}"
-download_if_not_cached "${PKG_NAME}" "${CHECKSUM}" "${DOWNLOAD_LINK}"
-
-tar xzf "${PKG_NAME}"
-mv "${PKG_NAME%.tar.gz}" /usr/local/libtorch_cpu
-rm -f "${PKG_NAME}"
-ok "Successfully installed libtorch_cpu ${VERSION}"
-
-##============================================================##
-# libtorch_gpu
-if [[ "${TARGET_ARCH}" == "x86_64" ]]; then
-    VERSION="1.7.0-2"
-    CHECKSUM="b64977ca4a13ab41599bac8a846e8782c67ded8d562fdf437f0e606cd5a3b588"
-    PKG_NAME="libtorch_gpu-${VERSION}-cu111-linux-x86_64.tar.gz"
-else # AArch64
-    VERSION="1.6.0-1"
-    PKG_NAME="libtorch_gpu-1.6.0-1-linux-aarch64.tar.gz"
-    CHECKSUM="eeb5a223d9dbe40fe96f16e6711c49a3777cea2c0a8da2445d63e117fdad0385"
-fi
+# --- Python PyTorch Installation ---
+function install_pytorch_python() {
+  info "Installing Python PyTorch ${PYTORCH_VERSION}..."
+
+  local INDEX_URL_FLAG=""
+  if [ "$CUDA_SUPPORT" = true ]; then
+    INDEX_URL_FLAG="--index-url https://download.pytorch.org/whl/${CUDA_VERSION_TAG}"
+  else
+    INDEX_URL_FLAG="--index-url https://download.pytorch.org/whl/cpu"
+  fi
+
+  pip3 install \
+    torch==${PYTORCH_VERSION} \
+    torchvision==${TORCHVISION_VERSION} \
+    torchaudio==${TORCHAUDIO_VERSION} \
+    ${INDEX_URL_FLAG}
+
+  # Verify installation
+  info "Verifying Python PyTorch installation..."
+  python3 -c "
+import torch
+print(f'PyTorch Version: {torch.__version__}')
+print(f'CUDA Available: {torch.cuda.is_available()}')
+if torch.cuda.is_available():
+  print(f'CUDA Version: {torch.version.cuda}')
+  print(f'GPU Name: {torch.cuda.get_device_name(0)}')
+"
+  ok "Python PyTorch installation successful."
+}
+
+# --- C++ LibTorch Installation ---
+function install_libtorch_cpp() {
+  info "Installing LibTorch C++ ${PYTORCH_VERSION}..."
+  local BASE_URL="https://download.pytorch.org/libtorch/cpu"
+  local ARCHIVE=""
+  local URL=""
+
+  if [ "${TARGET_ARCH}" = "x86_64" ]; then
+    if [ "$CUDA_SUPPORT" = true ]; then
+      ARCHIVE="libtorch-cxx11-abi-shared-with-deps-${PYTORCH_VERSION}+${CUDA_VERSION_TAG}.zip"
+      URL="${BASE_URL}/${CUDA_VERSION_TAG}/${ARCHIVE}"
+    else
+      ARCHIVE="libtorch-cxx11-abi-shared-with-deps-${PYTORCH_VERSION}%2Bcpu.zip"
+      URL="${BASE_URL}/cpu/${ARCHIVE}"
+    fi
+  elif [ "${TARGET_ARCH}" = "aarch64" ]; then
+    # WARNING: Official pre-compiled LibTorch C++ package is not available for aarch64.
+    # If needed, you must build from source. Skipping installation here.
+    warn "Official pre-compiled LibTorch C++ is not available for aarch64."
+    warn "Skipping LibTorch C++ installation. If required, you must build from source."
+    return 0 # Exit normally
+  else
+    error "Unsupported architecture: ${TARGET_ARCH}"
+    return 1
+  fi
+
+  local DOWNLOAD_DIR="/tmp/libtorch_download"
+  mkdir -p "${DOWNLOAD_DIR}"
+  pushd "${DOWNLOAD_DIR}" > /dev/null
+
+  info "Downloading from ${URL}"
+  download_if_not_cached "${ARCHIVE}" "${CHECKSUM}" "${URL}"
+  unzip -q "${ARCHIVE}"
+
+  # Install to system path
+  INSTALL_DIR="/usr/local/libtorch"
+  info "Installing LibTorch to ${INSTALL_DIR}..."
+  rm -rf "${INSTALL_DIR}" # Remove old version
+  mkdir -p "${INSTALL_DIR}"
+  # Use mv instead of cp -r for better efficiency
+  mv libtorch/* "${INSTALL_DIR}/"
+
+  # Clean up downloaded files
+  popd > /dev/null
+  rm -rf "${DOWNLOAD_DIR}"
+
+  # Optionally: update dynamic linker cache
+  ldconfig
 
-DOWNLOAD_LINK="https://apollo-system.cdn.bcebos.com/archive/6.0/${PKG_NAME}"
-download_if_not_cached "${PKG_NAME}" "${CHECKSUM}" "${DOWNLOAD_LINK}"
+  ok "LibTorch C++ ${PYTORCH_VERSION} installed successfully."
+}
 
-tar xzf "${PKG_NAME}"
-mv "${PKG_NAME%.tar.gz}" /usr/local/libtorch_gpu
+# --- Main Execution Flow ---
+main() {
+  # TODO(daohu527): For inference, no python version is required
+  # install_pytorch_python
+  install_libtorch_cpp
+  info "✅ All PyTorch components have been installed."
+}
 
-# Cleanup
-rm -f "${PKG_NAME}"
-ok "Successfully installed libtorch_gpu ${VERSION}"
+main "$@"

docker/build/installers/install_mkl.sh

@@ -15,6 +15,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 ###############################################################################
+
+# TODO(daohu527): Pytorch depends on mkl, but it has been packaged in torch and can be removed later.
+
 set -e
 
 cd "$(dirname "${BASH_SOURCE[0]}")"

docker/build/installers/install_ordinary_modules.sh

@@ -49,10 +49,10 @@ declare -a module_dreamview_deps=("install_dreamview_deps.sh")
 declare -a module_drivers_deps=("install_drivers_deps.sh")
 declare -a module_guardian_deps=()
 declare -a module_localization_deps=("install_proj.sh")
-declare -a module_map_deps=()
+declare -a module_map_deps=("install_proj.sh")
 declare -a module_monitor_deps=()
 declare -a module_perception_deps=("install_paddle_deps.sh" "install_ffmpeg.sh" "install_opencv.sh")
-declare -a module_planning_deps=("install_adolc.sh" "install_ipopt.sh")
+declare -a module_planning_deps=("install_adolc.sh" "install_ipopt.sh" "install_libtorch.sh" "install_opencv.sh")
 declare -a module_prediction_deps=("install_opencv.sh")
 declare -a module_routing_deps=()
 declare -a module_storytelling_deps=()

docker/build/installers/install_proj.sh

@@ -47,7 +47,7 @@ download_if_not_cached "$PKG_NAME" "$CHECKSUM" "$DOWNLOAD_LINK"
 
 tar xzf "${PKG_NAME}"
 pushd proj-${VERSION} >/dev/null
-mkdir build && cd build
+mkdir -p build && cd build
 cmake .. \
     -DBUILD_SHARED_LIBS=ON \
     -DBUILD_TESTING=OFF \

docker/build/installers/installer_base.sh

@@ -82,7 +82,7 @@ function apollo_environ_setup() {
     if [ ! -f "${APOLLO_LD_FILE}" ]; then
         echo "${SYSROOT_DIR}/lib" | tee -a "${APOLLO_LD_FILE}"
     fi
-    if [ ! -f "${APOLLO_PROFILE}" ]; then
+    if [ ! -f "${APOLLO_PROFILE}" ] && [ -f "/opt/apollo/rcfiles/apollo.sh.sample" ]; then
         cp -f /opt/apollo/rcfiles/apollo.sh.sample "${APOLLO_PROFILE}"
         echo "add_to_path ${SYSROOT_DIR}/bin" >> "${APOLLO_PROFILE}"
     fi

modules/planning/learning_based/model_inference/BUILD

@@ -27,7 +27,6 @@ cc_library(
         "//modules/common/util",
         "//modules/planning/common/util:math_util_lib",
         "//modules/planning/learning_based/img_feature_renderer:birdview_img_feature_renderer",
-        #"//third_party/libtorch",
         "@opencv//:core",
     ] + if_gpu(
         ["@libtorch_gpu"],

modules/planning/tools/BUILD

@@ -22,7 +22,6 @@ cc_binary(
     name = "inference_demo",
     srcs = ["inference_demo.cc"],
     deps = [
-        #"//third_party/libtorch",
         "@com_github_gflags_gflags//:gflags",
     ] + if_gpu(
         ["@libtorch_gpu"],

scripts/ci/apollo_build.sh

@@ -284,7 +284,8 @@ function run_bazel_build() {
     jobs_args="--jobs=${CUSTOM_JOBS}"
   fi
   # default set cpus number according to the total cpu cores
-  local cpus_args="--local_resources=cpu=$(nproc)"
+  local cpu_count=$(($(nproc) / 2))
+  local cpus_args="--local_resources=cpu=${cpu_count}"
   if [[ -n "${CUSTOM_CPUS}" ]]; then
     cpus_args="--local_resources=cpu=${CUSTOM_CPUS}"
   fi

third_party/libtorch/BUILD

@@ -72,4 +72,4 @@ install_src_files(
     src_dir = ["."],
     dest = "3rd-libtorch-gpu/src",
     filter = "*",
-)
+)