Developer Build: Add Open3D-ML to Open3D wheel (#2470)

ssheorey · germanros1987 · web-flow · commit 53bf72008bce · 2020-10-14T19:10:14.000-07:00
* Add Open3D-ML to Open3D wheel

* Install ML requirements for wheel test.

* Undo windows workflow changes

* Developer build from GHA CI input form

* Disable checking DLLs needed by the wheel.

Co-authored-by: germanros1987 &lt;38517452+germanros1987@users.noreply.github.com&gt;
diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml
@@ -2,6 +2,12 @@ name: macOS CI
 
 on:
   workflow_dispatch:
+    inputs:
+      developer_build:
+        description: 'Set to OFF for Release wheels'
+        required: false
+        default: 'ON'
+
   push:
     branches:
       - master
@@ -68,11 +74,20 @@ jobs:
     env:
       BUILD_CUDA_MODULE: OFF
       NPROC: 2
+      OPEN3D_ML_ROOT: ${{ github.workspace }}/Open3D-ML
+      DEVELOPER_BUILD: ${{ github.event.inputs.developer_build }}
     steps:
       - name: Checkout source code
         uses: actions/checkout@v2
         with:
           submodules: true
+
+      - name: Checkout Open3D-ML source code
+        uses: actions/checkout@v2
+        with:
+          repository: intel-isl/Open3D-ML
+          path: ${{ env.OPEN3D_ML_ROOT }}
+
       - name: Setup cache
         uses: actions/cache@v2
         with:
@@ -150,11 +165,19 @@ jobs:
       fail-fast: false
       matrix:
         python_version: [3.6, 3.7, 3.8]
+    env:
+      OPEN3D_ML_ROOT: ${{ github.workspace }}/Open3D-ML
     steps:
       - name: Checkout source code
         uses: actions/checkout@v2
         with:
           submodules: false
+      - name: Checkout Open3D-ML source code
+        uses: actions/checkout@v2
+        with:
+          repository: intel-isl/Open3D-ML
+          path: ${{ env.OPEN3D_ML_ROOT }}
+
       - name: Download wheels
         uses: actions/download-artifact@v2
         # See https://github.com/dawidd6/action-download-artifact for more
@@ -175,9 +198,10 @@ jobs:
           pi_tag=$(python -c "import sys; print(f'cp{sys.version_info.major}{sys.version_info.minor}')")
           python -m pip install ../open3d*-$pi_tag-*.whl
           echo
-          echo "Dynamic libraries used:"
-          DLL_PATH=$(dirname $(python -c "import open3d; print(open3d.cpu.pybind.__file__)"))/..
-          find "$DLL_PATH"/cpu/ -type f -execdir otool -L {} \;
+          # echo "Dynamic libraries used:"
+          # DLL_PATH=$(dirname $(python -c "import open3d; print(open3d.cpu.pybind.__file__)"))/..
+          # find "$DLL_PATH"/cpu/ -type f -execdir otool -L {} \;
+          python -m pip install -r "$OPEN3D_ML_ROOT/requirements.txt"
           echo
           test_wheel
       - name: Run Python unit tests (benchmarks)
diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml
@@ -2,6 +2,12 @@ name: Ubuntu CI
 
 on:
   workflow_dispatch:
+    inputs:
+      developer_build:
+        description: 'Set to OFF for Release wheels'
+        required: false
+        default: 'ON'
+
   push:
     branches:
       - master
@@ -67,11 +73,20 @@ jobs:
     env:
       BUILD_CUDA_MODULE: ON
       NPROC: 2
+      OPEN3D_ML_ROOT: ${{ github.workspace }}/Open3D-ML
+      DEVELOPER_BUILD: ${{ github.event.inputs.developer_build }}
     steps:
       - name: Checkout source code
         uses: actions/checkout@v2
         with:
           submodules: true
+
+      - name: Checkout Open3D-ML source code
+        uses: actions/checkout@v2
+        with:
+          repository: intel-isl/Open3D-ML
+          path: ${{ env.OPEN3D_ML_ROOT }}
+
       - name: Setup cache
         uses: actions/cache@v2
         with:
@@ -157,11 +172,19 @@ jobs:
       fail-fast: false
       matrix:
         python_version: [3.6, 3.7, 3.8]
+    env:
+      OPEN3D_ML_ROOT: ${{ github.workspace }}/Open3D-ML
     steps:
       - name: Checkout source code
         uses: actions/checkout@v2
         with:
           submodules: false
+      - name: Checkout Open3D-ML source code
+        uses: actions/checkout@v2
+        with:
+          repository: intel-isl/Open3D-ML
+          path: ${{ env.OPEN3D_ML_ROOT }}
+
       - name: Download wheels
         uses: actions/download-artifact@v2
         # See https://github.com/dawidd6/action-download-artifact for more
@@ -181,9 +204,10 @@ jobs:
           pi_tag=$(python -c "import sys; print(f'cp{sys.version_info.major}{sys.version_info.minor}')")
           python -m pip install ../open3d*-$pi_tag-*.whl
           echo
-          echo "Dynamic libraries used:"
-          DLL_PATH=$(dirname $(python -c "import open3d; print(open3d.cpu.pybind.__file__)"))/..
-          find "$DLL_PATH"/{cpu,cuda}/ -type f -print -execdir ldd {} \;
+          # echo "Dynamic libraries used:"
+          # DLL_PATH=$(dirname $(python -c "import open3d; print(open3d.cpu.pybind.__file__)"))/..
+          # find "$DLL_PATH"/{cpu,cuda}/ -type f -print -execdir ldd {} \;
+          python -m pip install -r "$OPEN3D_ML_ROOT/requirements.txt"
           echo
           test_wheel
       - name: Run Python unit tests (benchmarks)
diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
@@ -2,6 +2,12 @@ name: Windows CI
 
 on:
   workflow_dispatch:
+    inputs:
+      developer_build:
+        description: 'Set to OFF for Release wheels'
+        required: false
+        default: 'ON'
+
   push:
     branches:
       - master
@@ -72,6 +78,8 @@ jobs:
       fail-fast: false
       matrix:
         python_version: [3.6, 3.7, 3.8]
+    env:
+      DEVELOPER_BUILD: ${{ github.event.inputs.developer_build }}
     steps:
       - name: Checkout source code
         uses: actions/checkout@v2
@@ -87,7 +95,10 @@ jobs:
         run: |
           mkdir build
           cd build
-          cmake -G "Visual Studio 16 2019" -A x64 -DCMAKE_INSTALL_PREFIX="C:\Program Files\Open3D" -DBUILD_SHARED_LIBS=OFF -DSTATIC_WINDOWS_RUNTIME=ON ..
+          if ($DEVELOPER_BUILD -ne "OFF") {
+            $DEVELOPER_BUILD="ON"
+          }
+          cmake -G "Visual Studio 16 2019" -A x64 -DCMAKE_INSTALL_PREFIX="C:\Program Files\Open3D" -DBUILD_SHARED_LIBS=OFF -DSTATIC_WINDOWS_RUNTIME=ON -DDEVELOPER_BUILD=$DEVELOPER_BUILD ..
 
       - name: Build Python package
         run: |
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -20,6 +20,7 @@
 * Fast compression mode for PNG writing. (Issue #846)
 * Ubuntu 20.04 (Focal) support.
 * Added Line3D/Ray3D/Segment3D classes with plane, point, closest-distance, and AABB tests
+* Add Open3D-ML to Open3D wheel
 
 ## 0.9.0
 
diff --git a/util/ci_utils.sh b/util/ci_utils.sh
@@ -181,21 +181,28 @@ build_all() {
 build_wheel() {
 
     echo "Building Open3D wheel"
+
+    BUILD_FILAMENT_FROM_SOURCE=OFF
+    set +u
+    if [ -f "${OPEN3D_ML_ROOT}/set_open3d_ml_root.sh" ]; then
+        echo "Open3D-ML available at ${OPEN3D_ML_ROOT}. Bundling Open3D-ML in wheel."
+        BUNDLE_OPEN3D_ML=ON
+    else
+        BUNDLE_OPEN3D_ML=OFF
+    fi
+    if [[ "$DEVELOPER_BUILD" != "OFF" ]]; then # Validate input coming from GHA input field
+        DEVELOPER_BUILD="ON"
+    else
+        echo "Building for a new Open3D release"
+    fi
+    set -u
+
     echo
     echo Building with CPU only...
     mkdir -p build
     cd build # PWD=Open3D/build
-
-    # BUILD_FILAMENT_FROM_SOURCE if Linux and old glibc (Ubuntu 18.04)
-    BUILD_FILAMENT_FROM_SOURCE=OFF
-    #if [[ "$OSTYPE" == linux-gnu* ]]; then
-    #    glibc_version=$(ldd --version | grep -o -E '([0-9]+\.)+[0-9]+' | head -1)
-    #    if dpkg --compare-versions "$glibc_version" lt 2.31; then
-    #        BUILD_FILAMENT_FROM_SOURCE=ON
-    #    fi
-    #fi
-
     cmakeOptions=(-DBUILD_SHARED_LIBS=OFF
+        -DDEVELOPER_BUILD="$DEVELOPER_BUILD"
         -DBUILD_TENSORFLOW_OPS=ON
         -DBUILD_PYTORCH_OPS=ON
         -DBUILD_RPC_INTERFACE=ON
@@ -206,6 +213,7 @@ build_wheel() {
         -DCMAKE_BUILD_TYPE=Release
         -DBUILD_UNIT_TESTS=OFF
         -DBUILD_BENCHMARKS=OFF
+        -DBUNDLE_OPEN3D_ML="$BUNDLE_OPEN3D_ML"
     )
     cmake -DBUILD_CUDA_MODULE=OFF "${cmakeOptions[@]}" ..
     echo
