CMake: Add packaging support (#96)

alessandrogario · web-flow · commit eea7a544d789 · 2021-02-18T20:37:25.000-05:00
* CMake: Add packaging support

* CI: Upload artifacts, auto-create releases from tags

* Build: Lock the cxx-common version
diff --git a/.github/workflows/vcpkg_ci.yml b/.github/workflows/vcpkg_ci.yml
@@ -31,12 +31,14 @@ jobs:
 
     steps:
       - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
       - name: Install utility tools
         shell: bash
         run: |
           # TODO some of these should probably live in the Docker build image
           apt-get update
-          apt-get install -y pixz xz-utils make
+          apt-get install -y pixz xz-utils make rpm
 
       - name: Build with build script
         shell: bash
@@ -54,6 +56,33 @@ jobs:
           CLANG_BIN="$(find ../../pre-built-llvm-${{ matrix.llvm }} -name clang | grep "bin/clang")"
           ../scripts/roundtrip.py "./tools/rellic-decomp-${{ matrix.llvm }}.0" ../tests/tools/decomp "${CLANG_BIN}"
 
+      - name: Locate the packages
+        id: package_names
+        shell: bash
+        working-directory: rellic-build
+        run: |
+          echo ::set-output name=DEB_PACKAGE_PATH::rellic-build/$(ls *.deb)
+          echo ::set-output name=RPM_PACKAGE_PATH::rellic-build/$(ls *.rpm)
+          echo ::set-output name=TGZ_PACKAGE_PATH::rellic-build/$(ls *.tar.gz)
+
+      - name: Store the DEB package
+        uses: actions/upload-artifact@v1
+        with:
+          name: ${{ matrix.image.name }}-${{ matrix.image.tag }}_llvm${{ matrix.llvm }}_deb_package
+          path: ${{ steps.package_names.outputs.DEB_PACKAGE_PATH }}
+
+      - name: Store the RPM package
+        uses: actions/upload-artifact@v1
+        with:
+          name: ${{ matrix.image.name }}-${{ matrix.image.tag }}_llvm${{ matrix.llvm }}_rpm_package
+          path: ${{ steps.package_names.outputs.RPM_PACKAGE_PATH }}
+
+      - name: Store the TGZ package
+        uses: actions/upload-artifact@v1
+        with:
+          name: ${{ matrix.image.name }}-${{ matrix.image.tag }}_llvm${{ matrix.llvm }}_tgz_package
+          path: ${{ steps.package_names.outputs.TGZ_PACKAGE_PATH }}
+
   build_mac:
     strategy:
       fail-fast: false
@@ -69,6 +98,8 @@ jobs:
 
     steps:
       - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
       - name: Build with build script
         shell: bash
         run: |
@@ -84,3 +115,114 @@ jobs:
           # Run tests manually (kinda ugly, so use CMake way)
           CLANG_BIN="$(find ../../pre-built-llvm-${{ matrix.llvm }} -name clang | grep "bin/clang")"
           ../scripts/roundtrip.py "./tools/rellic-decomp-${{ matrix.llvm }}.0" ../tests/tools/decomp "${CLANG_BIN}"
+      - name: Locate the packages
+        id: package_names
+        shell: bash
+        working-directory: rellic-build
+        run: |
+          echo ::set-output name=TGZ_PACKAGE_PATH::rellic-build/$(ls *.tar.gz)
+
+      - name: Store the TGZ package
+        uses: actions/upload-artifact@v1
+        with:
+          name: macos-11.0_llvm${{ matrix.llvm }}_tgz_package
+          path: ${{ steps.package_names.outputs.TGZ_PACKAGE_PATH }}
+
+
+  release_linux:
+    # Do not run the release procedure if any of the builds has failed
+    needs: [ build_linux, build_mac ]
+    runs-on: ubuntu-20.04
+    if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
+
+    steps:
+      - name: Download all artifacts
+        uses: actions/download-artifact@v2
+
+      - name: Draft the new release
+        id: create_release
+        uses: actions/create-release@v1
+
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+        with:
+          tag_name: ${{ github.ref }}
+          release_name: Version ${{ github.ref }}
+          draft: true
+          prerelease: true
+
+      - name: Group the packages by platform
+        run: |
+          zip -r9 rellic_ubuntu-18.04_packages.zip \
+                  ubuntu-18.04*
+
+          zip -r9 rellic_ubuntu-20.04_packages.zip \
+                  ubuntu-20.04*
+
+      - name: Upload the Ubuntu 18.04 packages
+        uses: actions/upload-release-asset@v1
+
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: rellic_ubuntu-18.04_packages.zip
+          asset_name: rellic_ubuntu-18.04_packages.zip
+          asset_content_type: application/gzip
+
+      - name: Upload the Ubuntu 20.04 packages
+        uses: actions/upload-release-asset@v1
+
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: rellic_ubuntu-20.04_packages.zip
+          asset_name: rellic_ubuntu-20.04_packages.zip
+          asset_content_type: application/gzip
+
+
+
+
+  release_macos:
+    # Do not run the release procedure if any of the builds has failed
+    needs: [ build_linux, build_mac ]
+    runs-on: 'macos-11.0'
+    if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
+
+    steps:
+      - name: Download all artifacts
+        uses: actions/download-artifact@v2
+
+      - name: Draft the new release
+        id: create_release
+        uses: actions/create-release@v1
+
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+        with:
+          tag_name: ${{ github.ref }}
+          release_name: Version ${{ github.ref }}
+          draft: true
+          prerelease: true
+
+      - name: Group the packages by platform
+        run: |
+          zip -r9 rellic_macos-11.0_packages.zip \
+                  macos-11.0*
+
+      - name: Upload the macOS 11.0 packages
+        uses: actions/upload-release-asset@v1
+
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: rellic_macos-11.0_packages.zip
+          asset_name: rellic_macos-11.0_packages.zip
+          asset_content_type: application/gzip
diff --git a/packaging/README.md b/packaging/README.md
@@ -0,0 +1,17 @@
+# rellic packaging scripts
+
+## How to generate packages
+
+1. Configure and build rellic
+2. Set the **DESTDIR** variable to a new folder
+3. Run the packaging script, passing the **DESTDIR** folder
+
+Example:
+
+```sh
+rellic_version=$(git describe --always)
+
+cpack -D RELLIC_DATA_PATH="/path/to/install/directory" \
+      -R ${rellic_version} \
+      --config "packaging/main.cmake"
+```
diff --git a/packaging/cmake/dispatcher.cmake b/packaging/cmake/dispatcher.cmake
@@ -0,0 +1,23 @@
+#
+# Copyright (c) 2021-present, Trail of Bits, Inc.
+# All rights reserved.
+#
+# This source code is licensed in accordance with the terms specified in
+# the LICENSE file found in the root directory of this source tree.
+#
+
+set(CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION}-${CMAKE_SYSTEM_PROCESSOR}")
+set(CPACK_INSTALLED_DIRECTORIES "${RELLIC_DATA_PATH};.")
+
+string(TOLOWER "${CMAKE_SYSTEM_NAME}" system_name)
+if(system_name STREQUAL "darwin")
+  set(system_name "macos")
+endif()
+
+set(common_include "${CMAKE_CURRENT_LIST_DIR}/system/${system_name}/common.cmake")
+if(EXISTS "${common_include}")
+  include("${common_include}")
+endif()
+
+string(TOLOWER "${CPACK_GENERATOR}" cpack_generator)
+include("${CMAKE_CURRENT_LIST_DIR}/system/${system_name}/generators/${cpack_generator}.cmake")
diff --git a/packaging/cmake/system/linux/generators/deb.cmake b/packaging/cmake/system/linux/generators/deb.cmake
@@ -0,0 +1,12 @@
+#
+# Copyright (c) 2021-present, Trail of Bits, Inc.
+# All rights reserved.
+#
+# This source code is licensed in accordance with the terms specified in
+# the LICENSE file found in the root directory of this source tree.
+#
+
+set(CPACK_DEBIAN_PACKAGE_PRIORITY "extra")
+set(CPACK_DEBIAN_PACKAGE_SECTION "default")
+set(CPACK_DEBIAN_PACKAGE_HOMEPAGE "${CPACK_PACKAGE_HOMEPAGE_URL}")
+
diff --git a/packaging/cmake/system/linux/generators/rpm.cmake b/packaging/cmake/system/linux/generators/rpm.cmake
@@ -0,0 +1,13 @@
+#
+# Copyright (c) 2021-present, Trail of Bits, Inc.
+# All rights reserved.
+#
+# This source code is licensed in accordance with the terms specified in
+# the LICENSE file found in the root directory of this source tree.
+#
+
+set(CPACK_RPM_PACKAGE_RELEASE "${CPACK_PACKAGE_VERSION}")
+set(CPACK_RPM_FILE_NAME "RPM-DEFAULT")
+set(CPACK_RPM_PACKAGE_DESCRIPTION "${CPACK_PACKAGE_DESCRIPTION}")
+set(CPACK_RPM_PACKAGE_GROUP "default")
+set(CPACK_RPM_PACKAGE_LICENSE "GNU Affero General Public License v3.0")
diff --git a/packaging/cmake/system/linux/generators/tgz.cmake b/packaging/cmake/system/linux/generators/tgz.cmake
@@ -0,0 +1,10 @@
+#
+# Copyright (c) 2021-present, Trail of Bits, Inc.
+# All rights reserved.
+#
+# This source code is licensed in accordance with the terms specified in
+# the LICENSE file found in the root directory of this source tree.
+#
+
+set(CPACK_SET_DESTDIR ON)
+set(CPACK_INCLUDE_TOPLEVEL_DIRECTORY 0)
diff --git a/packaging/cmake/system/macos/generators/tgz.cmake b/packaging/cmake/system/macos/generators/tgz.cmake
@@ -0,0 +1,10 @@
+#
+# Copyright (c) 2021-present, Trail of Bits, Inc.
+# All rights reserved.
+#
+# This source code is licensed in accordance with the terms specified in
+# the LICENSE file found in the root directory of this source tree.
+#
+
+set(CPACK_SET_DESTDIR ON)
+set(CPACK_INCLUDE_TOPLEVEL_DIRECTORY 0)
diff --git a/packaging/main.cmake b/packaging/main.cmake
@@ -0,0 +1,30 @@
+#
+# Copyright (c) 2021-present, Trail of Bits, Inc.
+# All rights reserved.
+#
+# This source code is licensed in accordance with the terms specified in
+# the LICENSE file found in the root directory of this source tree.
+#
+
+if("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux")
+  set(CPACK_GENERATOR "TGZ;DEB;RPM")
+
+elseif("${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin")
+  set(CPACK_GENERATOR "TGZ")
+endif()
+
+if(RELLIC_DATA_PATH STREQUAL "")
+  message(FATAL_ERROR "The RELLIC_DATA_PATH variable was not set")
+endif()
+
+if(RELLIC_PACKAGE_VERSION STREQUAL "")
+  message(FATAL_ERROR "The RELLIC_PACKAGE_VERSION variable was not set")
+endif()
+
+set(CPACK_PROJECT_CONFIG_FILE "${CMAKE_CURRENT_LIST_DIR}/cmake/dispatcher.cmake")
+
+set(CPACK_PACKAGE_DESCRIPTION "Rellic produces goto-free C output from LLVM bitcode")
+set(CPACK_PACKAGE_NAME "rellic")
+set(CPACK_PACKAGE_VENDOR "Trail of Bits")
+set(CPACK_PACKAGE_CONTACT "opensource@trailofbits.com")
+set(CPACK_PACKAGE_HOMEPAGE_URL "https://github.com/lifting-bits/rellic")
diff --git a/scripts/build.sh b/scripts/build.sh
@@ -28,6 +28,7 @@ ARCH_VERSION=unknown
 BUILD_FLAGS=
 USE_HOST_COMPILER=0
 LIBRARIES="${BUILD_DIR}/libraries"
+CXX_COMMON_VERSION="v0.1.1"
 
 Z3_ARCHIVE=z3-
 Z3_VERSION=4.7.1
@@ -102,7 +103,7 @@ function GetArchVersion
 function DownloadCxxCommon
 {
   local GITHUB_LIBS="${LIBRARY_VERSION}.tar.xz"
-  local URL="https://github.com/trailofbits/cxx-common/releases/latest/download/${GITHUB_LIBS}"
+  local URL="https://github.com/trailofbits/cxx-common/releases/download/${CXX_COMMON_VERSION}/${GITHUB_LIBS}"
 
   echo "Fetching: ${URL}"
   if ! curl -LO "${URL}"; then
diff --git a/scripts/build_with_vcpkg.sh b/scripts/build_with_vcpkg.sh
@@ -28,6 +28,7 @@ LLVM_VERSION=llvm-9
 OS_VERSION=unknown
 ARCH_VERSION=unknown
 BUILD_FLAGS=
+CXX_COMMON_VERSION="v0.1.1"
 
 # There are pre-build versions of various libraries for specific
 # Ubuntu releases.
@@ -102,7 +103,7 @@ function GetArchVersion
 function DownloadVcpkgLibraries
 {
   local GITHUB_LIBS="${LIBRARY_VERSION}.tar.xz"
-  local URL="https://github.com/trailofbits/cxx-common/releases/latest/download/${GITHUB_LIBS}"
+  local URL="https://github.com/trailofbits/cxx-common/releases/download/${CXX_COMMON_VERSION}/${GITHUB_LIBS}"
 
   mkdir -p "${DOWNLOAD_DIR}"
   pushd "${DOWNLOAD_DIR}" || return 1
@@ -264,6 +265,41 @@ function Build
   return $?
 }
 
+# Create the packages
+function Package
+{
+  tag_count=$(cd "${SRC_DIR}" && git tag | wc -l)
+  if [[ ${tag_count} == 0 ]]; then
+    echo "WARNING: No tag found, marking this release as 0.0.0"
+    rellic_tag="v0.0.0"
+  else
+    rellic_tag=$(cd "${SRC_DIR}" && git describe --tags --always --abbrev=0)
+  fi
+
+  rellic_commit=$(cd "${SRC_DIR}" && git rev-parse HEAD | cut -c1-7)
+  rellic_version="${rellic_tag:1}.${rellic_commit}"
+
+  (
+    set -x
+
+    if [[ -d "install" ]]; then
+      rm -rf "install"
+    fi
+
+    mkdir "install"
+    export DESTDIR="$(pwd)/install"
+
+    cmake --build . \
+      --target install
+
+    cpack -D RELLIC_DATA_PATH="${DESTDIR}" \
+      -R ${rellic_version} \
+      --config "${SRC_DIR}/packaging/main.cmake"
+  ) || return $?
+
+  return $?
+}
+
 # Get a LLVM version name for the build. This is used to find the version of
 # cxx-common to download.
 function GetLLVMVersion
@@ -375,7 +411,7 @@ function main
   mkdir -p "${BUILD_DIR}"
   cd "${BUILD_DIR}" || exit 1
 
-  if ! (DownloadLibraries && Configure && Build); then
+  if ! (DownloadLibraries && Configure && Build && Package); then
     echo "[x] Build aborted."
     exit 1
   fi
