[pull] F#SRC-2333_wait_for_tcp_joint_sub_melodic from ros-planning:melodic-devel

.docker/README.md

@@ -1,3 +1,3 @@
-# MoveIt! Docker Containers
+# MoveIt Docker Containers
 
-For more information see [Continuous Integration and Docker](http://moveit.ros.org/documentation/contributing/continuous_integration.html) documentation.
+For more information see the pages [Continuous Integration and Docker](http://moveit.ros.org/documentation/contributing/continuous_integration.html) and [Using Docker Containers with MoveIt](https://moveit.ros.org/install/docker/).

.docker/ci-testing/Dockerfile

@@ -0,0 +1,15 @@
+# moveit/moveit:melodic-ci-testing
+# Sets up a base image to use for running Continuous Integration on Travis
+
+FROM moveit/moveit:melodic-ci
+MAINTAINER Robert Haschke rhaschke@techfak.uni-bielefeld.de
+
+# Switch to ros-testing
+RUN echo "deb http://packages.ros.org/ros-testing/ubuntu `lsb_release -cs` main" | tee /etc/apt/sources.list.d/ros1-latest.list
+
+# Upgrade packages to ros-testing and clean apt-cache within one RUN command
+RUN apt-get -qq update && \
+    apt-get -qq -y dist-upgrade && \
+    #
+    # Clear apt-cache to reduce image size
+    rm -rf /var/lib/apt/lists/*

.docker/ci/Dockerfile

@@ -2,48 +2,48 @@
 # Sets up a base image to use for running Continuous Integration on Travis
 
 FROM ros:melodic-ros-base
-MAINTAINER Dave Coleman dave@picknik.ai
+MAINTAINER Robert Haschke rhaschke@techfak.uni-bielefeld.de
 
 ENV TERM xterm
 
 # Setup (temporary) ROS workspace
 WORKDIR /root/ws_moveit
 
+# Copy MoveIt sources from docker context
+COPY . src/moveit
+
 # Commands are combined in single RUN statement with "apt/lists" folder removal to reduce image size
 # https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#minimize-the-number-of-layers
 RUN \
-    mkdir src && \
-    cd src && \
-    #
-    # Download moveit source, so that we can get necessary dependencies
-    wstool init --shallow . https://raw.githubusercontent.com/ros-planning/moveit/${ROS_DISTRO}-devel/moveit.rosinstall && \
-    #
     # Update apt package list as previous containers clear the cache
-    apt-get -qq update && \
-    apt-get -qq dist-upgrade && \
+    apt-get -q update && \
+    apt-get -q -y dist-upgrade && \
     #
     # Install some base dependencies
-    apt-get -qq install -y \
-        # Some source builds require a package.xml be downloaded via wget from an external location
-        wget \
-        # Required for rosdep command
-        sudo \
+    apt-get -q install --no-install-recommends -y \
+        # Some basic requirements
+        wget git sudo \
         # Preferred build tools
-        python-catkin-tools \
+        python$(test "${ROS_DISTRO}" = "noetic" && echo 3)-catkin-tools \
         clang clang-format-10 clang-tidy clang-tools \
         ccache && \
     #
-    # Download all dependencies of MoveIt!
-    rosdep update && \
-    rosdep install -y --from-paths . --ignore-src --rosdistro ${ROS_DISTRO} --as-root=apt:false && \
+    # Download MoveIt sources, so that we can fetch all necessary dependencies
+    wstool init --shallow src src/moveit/.github/workflows/upstream.rosinstall && \
+    git clone --depth 1 --branch ${ROS_DISTRO}-devel https://github.com/ros-planning/moveit_resources src/moveit_resources && \
     #
+    # Download all dependencies of MoveIt
+    rosdep update && \
+    DEBIAN_FRONTEND=noninteractive \
+    rosdep install -y --from-paths src --ignore-src --rosdistro ${ROS_DISTRO} --as-root=apt:false && \
     # Remove the source code from this container
-    # TODO: in the future we may want to keep this here for further optimization of later containers
-    cd .. && \
-    rm -rf src/ && \
+    rm -rf src && \
     #
     # Clear apt-cache to reduce image size
-    rm -rf /var/lib/apt/lists/*
+    rm -rf /var/lib/apt/lists/* && \
+    # Globally disable git security
+    # https://github.blog/2022-04-12-git-security-vulnerability-announced
+    git config --global --add safe.directory "*"
 
 # Continous Integration Setting
 ENV IN_DOCKER 1

.docker/release/Dockerfile

@@ -1,10 +1,11 @@
 # moveit/moveit:melodic-release
-# Full debian-based install of MoveIt! using apt-get
+# Full debian-based install of MoveIt using apt-get
 
 FROM ros:melodic-ros-base
 MAINTAINER Dave Coleman dave@picknik.ai
 
 # Commands are combined in single RUN statement with "apt/lists" folder removal to reduce image size
-RUN apt-get update && \
+RUN apt-get update -q && \
+    apt-get dist-upgrade -q -y && \
     apt-get install -y ros-${ROS_DISTRO}-moveit-* && \
     rm -rf /var/lib/apt/lists/*

.docker/source/Dockerfile

@@ -1,30 +1,35 @@
+# syntax = docker/dockerfile:1.3
+
 # moveit/moveit:melodic-source
 # Downloads the moveit source code and install remaining debian dependencies
 
-FROM moveit/moveit:melodic-ci-shadow-fixed
-MAINTAINER Dave Coleman dave@picknik.ai
+FROM moveit/moveit:melodic-ci-testing
+MAINTAINER Robert Haschke rhaschke@techfak.uni-bielefeld.de
 
+ENV PYTHONIOENCODING UTF-8
+# Export ROS_UNDERLAY for downstream docker containers
 ENV ROS_UNDERLAY /root/ws_moveit/install
-WORKDIR $ROS_UNDERLAY/../src
+# Environment variable used in instructions on moveit.ros.org website for running clang-tidy
+ENV CATKIN_WS $(realpath $ROS_UNDERLAY/..)
+WORKDIR $ROS_UNDERLAY/..
+
+# Copy MoveIt sources from docker context
+COPY . src/moveit
 
 # Commands are combined in single RUN statement with "apt/lists" folder removal to reduce image size
 # https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#minimize-the-number-of-layers
-RUN \
-    # Download moveit source so that we can get necessary dependencies
-    wstool init . https://raw.githubusercontent.com/ros-planning/moveit/${ROS_DISTRO}-devel/moveit.rosinstall && \
-    #
-    # Update apt package list as cache is cleared in previous container
-    # Usually upgrading involves a few packages only (if container builds became out-of-sync)
-    apt-get -qq update && \
-    apt-get -qq dist-upgrade && \
+RUN --mount=type=cache,target=/root/.ccache/ \
+    # Enable ccache
+    PATH=/usr/lib/ccache:$PATH && \
+    # Fetch required upstream sources for building
+    wstool init --shallow src src/moveit/.github/workflows/upstream.rosinstall && \
+    git clone --depth 1 --branch ${ROS_DISTRO}-devel https://github.com/ros-planning/moveit_resources src/moveit_resources && \
     #
-    rosdep update && \
-    rosdep install -y --from-paths . --ignore-src --rosdistro ${ROS_DISTRO} --as-root=apt:false && \
-    rm -rf /var/lib/apt/lists/*
-
-ENV PYTHONIOENCODING UTF-8
-RUN cd .. && \
-    catkin config --extend /opt/ros/$ROS_DISTRO --install --cmake-args -DCMAKE_BUILD_TYPE=Release && \
+    catkin config --extend /opt/ros/$ROS_DISTRO --install --cmake-args -DCMAKE_BUILD_TYPE=Release -DCMAKE_EXPORT_COMPILE_COMMANDS=ON && \
     # Status rate is limited so that just enough info is shown to keep Docker from timing out,
     # but not too much such that the Docker log gets too long (another form of timeout)
-    catkin build --limit-status-rate 0.001 --no-notify
+    catkin build --limit-status-rate 0.001 --no-notify && \
+    ccache -s && \
+    #
+    # Update /ros_entrypoint.sh to source our new workspace
+    sed -i "s#/opt/ros/\$ROS_DISTRO/setup.bash#$ROS_UNDERLAY/setup.sh#g" /ros_entrypoint.sh

.dockerignore

@@ -0,0 +1,37 @@
+# ignore everything
+*
+# but include these:
+!.github/workflows/upstream.rosinstall
+!**/package.xml
+!**/CATKIN_IGNORE
+
+# https://github.com/moby/moby/issues/42788
+!moveit_plugins/moveit_plugins/package.xml
+!moveit_plugins/moveit_ros_control_interface/package.xml
+!moveit_plugins/moveit_simple_controller_manager/package.xml
+!moveit_plugins/moveit_fake_controller_manager/package.xml
+!moveit_kinematics/package.xml
+!moveit_setup_assistant/package.xml
+!moveit_core/package.xml
+!moveit_commander/package.xml
+!moveit_planners/ompl/package.xml
+!moveit_planners/chomp/chomp_motion_planner/package.xml
+!moveit_planners/chomp/chomp_interface/package.xml
+!moveit_planners/chomp/chomp_optimizer_adapter/package.xml
+!moveit_planners/pilz_industrial_motion_planner_testutils/package.xml
+!moveit_planners/pilz_industrial_motion_planner/package.xml
+!moveit_planners/moveit_planners/package.xml
+!moveit_runtime/package.xml
+!moveit/package.xml
+!moveit_ros/warehouse/package.xml
+!moveit_ros/moveit_servo/package.xml
+!moveit_ros/occupancy_map_monitor/package.xml
+!moveit_ros/perception/package.xml
+!moveit_ros/move_group/package.xml
+!moveit_ros/robot_interaction/package.xml
+!moveit_ros/visualization/package.xml
+!moveit_ros/manipulation/package.xml
+!moveit_ros/planning/package.xml
+!moveit_ros/planning_interface/package.xml
+!moveit_ros/benchmarks/package.xml
+!moveit_ros/moveit_ros/package.xml

.github/ISSUE_TEMPLATE.md

@@ -3,8 +3,8 @@
 Overview of your issue here.
 
 ### Your environment
-* ROS Distro: [Indigo|Jade|Kinetic]
-* OS Version: e.g. Ubuntu 16.04
+* ROS Distro: [Kinetic|Melodic|Noetic]
+* OS Version: e.g. Ubuntu 18.04
 * Source or Binary build?
 * If binary, which release version?
 * If source, which git commit or tag?

.github/workflows/ci.yaml

@@ -0,0 +1,152 @@
+# This config uses industrial_ci (https://github.com/ros-industrial/industrial_ci.git).
+# For troubleshooting, see readme (https://github.com/ros-industrial/industrial_ci/blob/master/README.rst)
+
+name: CI
+
+on:
+  workflow_dispatch:
+  pull_request:
+  push:
+    branches:
+      - melodic-devel
+
+jobs:
+  default:
+    strategy:
+      fail-fast: false
+      matrix:
+        env:
+          - IMAGE: melodic-ci
+            CCOV: true
+          - IMAGE: melodic-ci-testing
+            IKFAST_TEST: true
+            CATKIN_LINT: true
+            CLANG_TIDY: pedantic
+    env:
+      CXXFLAGS: >-
+        -Wall -Wextra -Wwrite-strings -Wunreachable-code -Wpointer-arith -Wredundant-decls
+        -Wno-unused-parameter
+        ${{ matrix.env.CLANG_TIDY && '-Wno-c++17-compat -Wno-overloaded-virtual -Wno-unused-function' || '-Wno-unused-but-set-parameter' }}
+      CLANG_TIDY_ARGS: --fix --fix-errors
+      DOCKER_IMAGE: moveit/moveit:${{ matrix.env.IMAGE }}
+      UPSTREAM_WORKSPACE: .github/workflows/upstream.rosinstall
+      TARGET_WORKSPACE: $TARGET_REPO_PATH github:ros-planning/moveit_resources#master
+      DOWNSTREAM_WORKSPACE: .github/workflows/downstream.rosinstall
+      # Pull any updates to the upstream workspace (after restoring it from cache)
+      AFTER_SETUP_UPSTREAM_WORKSPACE: vcs pull $BASEDIR/upstream_ws/src
+      AFTER_SETUP_DOWNSTREAM_WORKSPACE: vcs pull $BASEDIR/downstream_ws/src
+      # Clear the ccache stats before and log the stats after the build
+      AFTER_SETUP_CCACHE: ccache --zero-stats --max-size=10.0G
+      AFTER_BUILD_TARGET_WORKSPACE: ccache --show-stats
+      AFTER_BUILD_DOWNSTREAM_WORKSPACE: ccache --show-stats
+      # Compile CCOV with Debug. Enable -Werror (except CLANG_TIDY=pedantic, which makes the clang-tidy step fail on warnings)
+      TARGET_CMAKE_ARGS: >
+        -DCMAKE_BUILD_TYPE=${{ matrix.env.CCOV && 'Debug' || 'Release'}}
+        -DCMAKE_CXX_FLAGS="${{ matrix.env.CLANG_TIDY != 'pedantic' && '-Werror ' || '' }}$CXXFLAGS${{ matrix.env.CCOV && ' --coverage -O2 -fno-omit-frame-pointer'}}"
+      UPSTREAM_CMAKE_ARGS: "-DCMAKE_CXX_FLAGS=''"
+      DOWNSTREAM_CMAKE_ARGS: -DCMAKE_CXX_FLAGS="-Wall -Wextra"
+      CCACHE_DIR: ${{ github.workspace }}/.ccache
+      BASEDIR: ${{ github.workspace }}/.work
+      CLANG_TIDY_BASE_REF: ${{ github.event_name != 'workflow_dispatch' && (github.base_ref || github.ref) || '' }}
+      BEFORE_CLANG_TIDY_CHECKS: |
+        # Show list of applied checks
+        (cd $TARGET_REPO_PATH; clang-tidy --list-checks)
+        # Disable clang-tidy for ikfast plugins as we cannot fix the generated code
+        find $BASEDIR/target_ws/build -iwholename "*_ikfast_plugin/compile_commands.json" -exec rm {} \;
+      CC: ${{ matrix.env.CLANG_TIDY && 'clang' }}
+      CXX: ${{ matrix.env.CLANG_TIDY && 'clang++' }}
+
+    name: "${{ matrix.env.IMAGE }}${{ matrix.env.CATKIN_LINT && ' + catkin_lint' || ''}}${{ matrix.env.CCOV && ' + ccov' || ''}}${{ matrix.env.IKFAST_TEST && ' + ikfast' || ''}}${{ matrix.env.CLANG_TIDY && (github.event_name != 'workflow_dispatch' && ' + clang-tidy (delta)' || ' + clang-tidy (all)') || '' }}"
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Free up disk space"
+        if: matrix.env.CCOV
+        run: |
+          sudo apt-get -qq purge build-essential "ghc*"
+          sudo apt-get clean
+          # cleanup docker images not used by us
+          docker system prune -af
+          # shift ccache folder to /mnt on a separate disk
+          sudo mkdir /mnt/ccache
+          mkdir ${{ env.CCACHE_DIR }}
+          sudo mount --bind ${{ env.CCACHE_DIR }} /mnt/ccache
+          # free up a lot of stuff from /usr/local
+          sudo rm -rf /usr/local
+          df -h
+      - uses: actions/checkout@v2
+      - name: Cache upstream workspace
+        uses: pat-s/always-upload-cache@v2.1.5
+        with:
+          path: ${{ env.BASEDIR }}/upstream_ws
+          key: ${{ env.CACHE_PREFIX }}-${{ github.run_id }}
+          restore-keys: ${{ env.CACHE_PREFIX }}
+        env:
+          CACHE_PREFIX: upstream_ws-${{ matrix.env.IMAGE }}-${{ hashFiles('.github/workflows/upstream.rosinstall', '.github/workflows/ci.yaml') }}
+      - name: Cache downstream workspace
+        uses: pat-s/always-upload-cache@v2.1.5
+        with:
+          path: ${{ env.BASEDIR }}/downstream_ws
+          key: ${{ env.CACHE_PREFIX }}-${{ github.run_id }}
+          restore-keys: ${{ env.CACHE_PREFIX }}
+        env:
+          CACHE_PREFIX: downstream_ws-${{ matrix.env.IMAGE }}-${{ hashFiles('.github/workflows/downstream.rosinstall', '.github/workflows/ci.yaml') }}
+      # The target directory cache doesn't include the source directory because
+      # that comes from the checkout.  See "prepare target_ws for cache" task below
+      - name: Cache target workspace
+        if: "!matrix.env.CCOV"
+        uses: pat-s/always-upload-cache@v2.1.5
+        with:
+          path: ${{ env.BASEDIR }}/target_ws
+          key: ${{ env.CACHE_PREFIX }}-${{ github.run_id }}
+          restore-keys: ${{ env.CACHE_PREFIX }}
+        env:
+          CACHE_PREFIX: target_ws${{ matrix.env.CCOV && '-ccov' || '' }}-${{ matrix.env.IMAGE }}-${{ hashFiles('**/CMakeLists.txt', '**/package.xml', '.github/workflows/ci.yaml') }}
+      - name: Cache ccache
+        uses: pat-s/always-upload-cache@v2.1.5
+        with:
+          path: ${{ env.CCACHE_DIR }}
+          key: ${{ env.CACHE_PREFIX }}-${{ github.sha }}-${{ github.run_id }}
+          restore-keys: |
+            ${{ env.CACHE_PREFIX }}-${{ github.sha }}
+            ${{ env.CACHE_PREFIX }}
+        env:
+          CACHE_PREFIX: ccache-${{ matrix.env.IMAGE }}${{ matrix.env.CCOV && '-ccov' || '' }}
+
+      - name: Generate ikfast packages
+        if: matrix.env.IKFAST_TEST
+        run: moveit_kinematics/test/test_ikfast_plugins.sh
+      - id: ici
+        name: Run industrial_ci
+        uses: ros-industrial/industrial_ci@master
+
+      - name: Upload test artifacts (on failure)
+        uses: actions/upload-artifact@v2
+        if: failure() && (steps.ici.outputs.run_target_test || steps.ici.outputs.target_test_results)
+        with:
+          name: test-results-${{ matrix.env.IMAGE }}
+          path: ${{ env.BASEDIR }}/target_ws/**/test_results/**/*.xml
+      - name: Generate codecov report
+        uses: rhaschke/lcov-action@main
+        if: always() && matrix.env.CCOV && steps.ici.outputs.target_test_results == '0'
+        with:
+          docker: $DOCKER_IMAGE
+          workdir: ${{ env.BASEDIR }}/target_ws
+          ignore: '"*/target_ws/build/*" "*/target_ws/install/*" "*/test/*"'
+      - name: Upload codecov report
+        uses: codecov/codecov-action@v2
+        if: always() && matrix.env.CCOV && steps.ici.outputs.target_test_results == '0'
+        with:
+          files: ${{ env.BASEDIR }}/target_ws/coverage.info
+      - name: Upload clang-tidy changes
+        uses: rhaschke/upload-git-patch-action@main
+        if: always() && matrix.env.CLANG_TIDY
+        with:
+          name: clang-tidy
+          path: ${{ env.BASEDIR }}/target_ws/src/$(basename $(pwd))
+      - name: Prepare target_ws for cache
+        if: always() && !matrix.env.CCOV
+        run: |
+          du -sh ${{ env.BASEDIR }}/target_ws
+          sudo find ${{ env.BASEDIR }}/target_ws -wholename '*/test_results/*' -delete
+          sudo rm -rf ${{ env.BASEDIR }}/target_ws/src
+          du -sh ${{ env.BASEDIR }}/target_ws