build public binary wheels (#381)

Summary:
Now live at https://dl.fbaipublicfiles.com/detectron2/wheels/index.html
Pull Request resolved: fairinternal/detectron2#381

Differential Revision: D19776395

Pulled By: ppwwyyxx

fbshipit-source-id: 363a7f5fa1499a1e1d91eccd055c9b5ca01cc160

Author: ppwwyyxx

.github/CONTRIBUTING.md

@@ -32,13 +32,14 @@ If some part is not as extensible, you can also bring up the issue to make it mo
 
 When sending a PR, please do:
 
-1. Fork the repo and create your branch from `master`.
+1. If a PR contains multiple orthogonal changes, split it to several PRs.
 2. If you've added code that should be tested, add tests.
-3. If APIs are changed, update the documentation.
-4. Ensure the test suite passes.
-5. Make sure your code lints with `./dev/linter.sh`.
-6. If a PR contains multiple orthogonal changes, split it to several PRs.
-7. If you haven't already, complete the Contributor License Agreement ("CLA").
+3. For PRs that need experiments (e.g. adding a new model), you don't need to update model zoo,
+   but do provide experiment results in the description of the PR.
+4. If APIs are changed, update the documentation.
+5. Ensure the test suite passes.
+6. Make sure your code lints with `./dev/linter.sh`.
+
 
 ## Contributor License Agreement ("CLA")
 In order to accept your pull request, we need you to submit a CLA. You only need

.gitignore

@@ -16,6 +16,7 @@ _ext
 detectron2.egg-info/
 build/
 dist/
+wheels/
 
 # pytorch/python/numpy formats
 *.pth

INSTALL.md

@@ -12,12 +12,11 @@ also installs detectron2 with a few simple commands.
 	You can install them together at [pytorch.org](https://pytorch.org) to make sure of this.
 - OpenCV, optional, needed by demo and visualization
 - pycocotools: `pip install cython; pip install 'git+https://github.com/cocodataset/cocoapi.git#subdirectory=PythonAPI'`
-- gcc & g++ ≥ 4.9
 
 
-### Build and Install Detectron2
+### Build Detectron2 from Source
 
-After having the above dependencies, run:
+After having the above dependencies and gcc & g++ ≥ 4.9, run:
 ```
 pip install 'git+https://github.com/facebookresearch/detectron2.git'
 # (add --user if you don't have permission)
@@ -33,8 +32,21 @@ cd detectron2 && pip install -e .
 To __rebuild__ detectron2 that's built from a local clone, `rm -rf build/ **/*.so` then `pip install -e .`.
 You often need to rebuild detectron2 after reinstalling PyTorch.
 
+### Install Pre-Built Detectron2
+```
+# for CUDA 10.1:
+pip install detectron2 -f \
+	https://dl.fbaipublicfiles.com/detectron2/wheels/cu101/index.html
+```
+You can replace cu101 with "cu{100,92}" or "cpu".
+
+Note that such installation has to be used with the latest official PyTorch release (currently 1.4).
+It will not work with your custom build of PyTorch.
+
 ### Common Installation Issues
 
+If you met issues using the pre-built detectron2, please uninstall it and try building it from source.
+
 Click each issue for its solutions:
 
 <details>

dev/packaging/README.md

@@ -0,0 +1,17 @@
+
+## To build a cu101 wheel for release:
+
+```
+$ nvidia-docker run -it --storage-opt "size=20GB" --name pt  pytorch/manylinux-cuda101
+# inside the container:
+# git clone https://github.com/facebookresearch/detectron2/
+# cd detectron2
+# export CU_VERSION=cu101 D2_VERSION_SUFFIX= PYTHON_VERSION=3.7 PYTORCH_VERSION=1.4
+# ./dev/packaging/build_wheel.sh
+```
+
+## To build all wheels for `CUDA {9.2,10.0,10.1}` x `Python {3.6,3.7,3.8}`:
+```
+./dev/packaging/build_all_wheels.sh
+./dev/packaging/gen_wheel_index.sh /path/to/wheels
+```

dev/packaging/build_all_wheels.sh

@@ -0,0 +1,56 @@
+#!/bin/bash -e
+
+PYTORCH_VERSION=1.4
+
+build_for_one_cuda() {
+  cu=$1
+
+  case "$cu" in
+    cu*)
+      container_name=manylinux-cuda${cu/cu/}
+      ;;
+    cpu)
+      container_name=manylinux-cuda101
+      ;;
+    *)
+      echo "Unrecognized cu=$cu"
+      exit 1
+      ;;
+  esac
+
+  echo "Launching container $container_name ..."
+
+  for py in 3.6 3.7 3.8; do
+    docker run -itd \
+      --name $container_name \
+      --mount type=bind,source="$(pwd)",target=/detectron2 \
+      pytorch/$container_name
+
+    cat <<EOF | docker exec -i $container_name sh
+      export CU_VERSION=$cu D2_VERSION_SUFFIX=+$cu PYTHON_VERSION=$py
+      export PYTORCH_VERSION=$PYTORCH_VERSION
+      cd /detectron2 && ./dev/packaging/build_wheel.sh
+EOF
+
+    if [[ "$cu" == "cu101" ]]; then
+      # build wheel without local version
+      cat <<EOF | docker exec -i $container_name sh
+        export CU_VERSION=$cu D2_VERSION_SUFFIX= PYTHON_VERSION=$py
+        export PYTORCH_VERSION=$PYTORCH_VERSION
+        cd /detectron2 && ./dev/packaging/build_wheel.sh
+EOF
+    fi
+
+    docker exec -i $container_name rm -rf /detectron2/build/$cu
+    docker container stop $container_name
+    docker container rm $container_name
+  done
+}
+
+if [[ -n "$1" ]]; then
+  build_for_one_cuda "$1"
+else
+  for cu in cu101 cu100 cu92 cpu; do
+    build_for_one_cuda "$cu"
+  done
+fi

dev/packaging/build_wheel.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+set -ex
+
+ldconfig  # https://github.com/NVIDIA/nvidia-docker/issues/854
+
+script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+. "$script_dir/pkg_helpers.bash"
+
+echo "Build Settings:"
+echo "CU_VERSION: $CU_VERSION"                 # e.g. cu100
+echo "D2_VERSION_SUFFIX: $D2_VERSION_SUFFIX"   # e.g. +cu100 or ""
+echo "PYTHON_VERSION: $PYTHON_VERSION"         # e.g. 3.6
+echo "PYTORCH_VERSION: $PYTORCH_VERSION"       # e.g. 1.4
+
+setup_cuda
+setup_wheel_python
+
+export TORCH_VERSION_SUFFIX="+$CU_VERSION"
+if [[ "$CU_VERSION" == "cu101" ]]; then
+	export TORCH_VERSION_SUFFIX=""
+fi
+pip_install pip numpy -U
+pip_install "torch==$PYTORCH_VERSION$TORCH_VERSION_SUFFIX" \
+	-f https://download.pytorch.org/whl/$CU_VERSION/torch_stable.html
+
+# use separate directories to allow parallel build
+BASE_BUILD_DIR=build/$CU_VERSION/$PYTHON_VERSION
+python setup.py \
+  build -b $BASE_BUILD_DIR \
+  bdist_wheel -b $BASE_BUILD_DIR/build_dist -d wheels/$CU_VERSION

dev/packaging/gen_wheel_index.sh

@@ -0,0 +1,24 @@
+#!/bin/bash -e
+
+
+root=$1
+if [[ -z "$root" ]]; then
+  echo "Usage: ./gen_wheel_index.sh /path/to/wheels"
+  exit
+fi
+
+index=$root/index.html
+
+cd "$root"
+for cu in cpu cu92 cu100 cu101; do
+  cd $cu
+  for whl in *.whl; do
+    echo "<a href=\"$whl\">$whl</a><br>"
+  done > index.html
+  cd "$root"
+done
+
+for whl in $(find . -type f -name '*.whl' -printf '%P\n' | sort); do
+  echo "<a href=\"$whl\">$whl</a><br>"
+done > "$index"
+

dev/packaging/pkg_helpers.bash

@@ -0,0 +1,52 @@
+#!/bin/bash -e
+
+# Function to retry functions that sometimes timeout or have flaky failures
+retry () {
+    $*  || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*)
+}
+# Install with pip a bit more robustly than the default
+pip_install() {
+  retry pip install --progress-bar off "$@"
+}
+
+
+setup_cuda() {
+  # Now work out the CUDA settings
+  # Like other torch domain libraries, we choose common GPU architectures only.
+  export FORCE_CUDA=1
+  case "$CU_VERSION" in
+    cu101)
+      export CUDA_HOME=/usr/local/cuda-10.1/
+      export TORCH_CUDA_ARCH_LIST="3.5;3.7;5.0;5.2;6.0+PTX;6.1+PTX;7.0+PTX;7.5+PTX"
+      ;;
+    cu100)
+      export CUDA_HOME=/usr/local/cuda-10.0/
+      export TORCH_CUDA_ARCH_LIST="3.5;3.7;5.0;5.2;6.0+PTX;6.1+PTX;7.0+PTX;7.5+PTX"
+      ;;
+    cu92)
+      export CUDA_HOME=/usr/local/cuda-9.2/
+      export TORCH_CUDA_ARCH_LIST="3.5;3.7;5.0;5.2;6.0+PTX;6.1+PTX;7.0+PTX"
+      ;;
+    cpu)
+      unset FORCE_CUDA
+      export CUDA_VISIBLE_DEVICES=
+      ;;
+    *)
+      echo "Unrecognized CU_VERSION=$CU_VERSION"
+      exit 1
+      ;;
+  esac
+}
+
+setup_wheel_python() {
+  case "$PYTHON_VERSION" in
+    3.6) python_abi=cp36-cp36m ;;
+    3.7) python_abi=cp37-cp37m ;;
+    3.8) python_abi=cp38-cp38 ;;
+    *)
+      echo "Unrecognized PYTHON_VERSION=$PYTHON_VERSION"
+      exit 1
+      ;;
+  esac
+  export PATH="/opt/python/$python_abi/bin:$PATH"
+}

docs/notes/changelog.md

@@ -1,7 +1,10 @@
 # Change Log
 
+### Releases
+See release log at
+[https://github.com/facebookresearch/detectron2/releases](https://github.com/facebookresearch/detectron2/releases)
 
-### Notable Changes:
+### Notable Backward Incompatible Changes:
 
 * 2019-11-11: `detectron2.data.detection_utils.read_image` transposes images with exif information.
 * 2019-10-10: initial release.

setup.py

@@ -20,9 +20,10 @@ def get_version():
     version_line = [l.strip() for l in init_py if l.startswith("__version__")][0]
     version = version_line.split("=")[-1].strip().strip("'\"")
 
-    # Used by CI to build nightly packages. Users should never use it.
-    # To build a nightly wheel, run:
-    # FORCE_CUDA=1 BUILD_NIGHTLY=1 TORCH_CUDA_ARCH_LIST=All python setup.py bdist_wheel
+    # The following is used to build release packages.
+    # Users should never use it.
+    suffix = os.getenv("D2_VERSION_SUFFIX", "")
+    version = version + suffix
     if os.getenv("BUILD_NIGHTLY", "0") == "1":
         from datetime import datetime
 
@@ -52,7 +53,9 @@ def get_extensions():
     extra_compile_args = {"cxx": []}
     define_macros = []
 
-    if (torch.cuda.is_available() and CUDA_HOME is not None) or os.getenv("FORCE_CUDA", "0") == "1":
+    if (
+        torch.cuda.is_available() and CUDA_HOME is not None and os.path.isdir(CUDA_HOME)
+    ) or os.getenv("FORCE_CUDA", "0") == "1":
         extension = CUDAExtension
         sources += source_cuda
         define_macros += [("WITH_CUDA", None)]
@@ -95,18 +98,20 @@ def get_model_zoo_configs() -> List[str]:
         path.dirname(path.realpath(__file__)), "detectron2", "model_zoo", "configs"
     )
     # Symlink the config directory inside package to have a cleaner pip install.
-    if path.exists(destination):
-        # Remove stale symlink/directory from a previous build.
+
+    # Remove stale symlink/directory from a previous build.
+    if path.exists(source_configs_dir):
         if path.islink(destination):
             os.unlink(destination)
-        else:
+        elif path.isdir(destination):
             shutil.rmtree(destination)
 
-    try:
-        os.symlink(source_configs_dir, destination)
-    except OSError:
-        # Fall back to copying if symlink fails: ex. on Windows.
-        shutil.copytree(source_configs_dir, destination)
+    if not path.exists(destination):
+        try:
+            os.symlink(source_configs_dir, destination)
+        except OSError:
+            # Fall back to copying if symlink fails: ex. on Windows.
+            shutil.copytree(source_configs_dir, destination)
 
     config_paths = glob.glob("configs/**/*.yaml", recursive=True)
     return config_paths
@@ -124,7 +129,7 @@ def get_model_zoo_configs() -> List[str]:
     python_requires=">=3.6",
     install_requires=[
         "termcolor>=1.1",
-        "Pillow==6.2.2",  # torchvision currently does not work with Pillow 7
+        "Pillow",  # you can also use pillow-simd for better performance
         "yacs>=0.1.6",
         "tabulate",
         "cloudpickle",