Add scip-clang code intelligence indexing via GitHub Actions (#48) #340
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: validate-migration | |
| on: [ push ] | |
| concurrency: | |
| group: ${{ github.ref }} | |
| cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} | |
| env: | |
| # Setup: Constants for downloading and compiling dependencies | |
| P4_DOWNLOAD_URL: "https://filehost.perforce.com/perforce/r23.1/bin.linux26x86_64/p4api-glibc2.3-openssl1.1.1.tgz" | |
| P4_DOWNLOAD_DIR: "/tmp/p4-download" | |
| OPENSSL_DOWNLOAD_URL: "https://www.openssl.org/source/openssl-1.1.1w.tar.gz" | |
| OPENSSL_SOURCE_DIR: "/tmp/openssl-src" | |
| OPENSSL_INSTALL_DIR: "/tmp/openssl-install" | |
| OPENSSL_DOWNLOAD_DIR: "/tmp/openssl-download" | |
| LLVM_PROJECT_DOWNLOAD_URL: "https://github.com/llvm/llvm-project/archive/refs/tags/llvmorg-16.0.4.tar.gz" | |
| LLVM_PROJECT_DOWNLOAD_DIR: "/tmp/llvm-project-download" | |
| LLVM_PROJECT_SOURCE_DIR: "/tmp/llvm-project-src" | |
| LLVM_PROJECT_INSTALL_DIR: "/tmp/llvm-project-install" | |
| # Setup: Perforce credentials | |
| P4PORT: "ssl:perforce.sgdev.org:1666" # the address of the Perforce server to connect to | |
| P4USER: "admin" # the name of the Perforce user | |
| P4PASSWD: ${{ secrets.P4PASSWD }} # the ticket for the Perforce user | |
| # Setup: Enable local sccache for faster builds. Do not make compilation | |
| # depend on the availability of GitHub's remote artifact cache. | |
| CMAKE_C_COMPILER_LAUNCHER: "sccache" | |
| CMAKE_CXX_COMPILER_LAUNCHER: "sccache" | |
| # Setup: Sanitizer flags | |
| UBSAN_OPTIONS: "print_stacktrace=1" # print stack trace on undefined behavior | |
| ASAN_OPTIONS: "check_initialization_order=1" # check initialization order of globals | |
| # Test: Input parameters to the validate-migration test script | |
| DEPOT_NAME: "source/src-cli" # the name of the Perforce depot to validate the git migration against | |
| jobs: | |
| validate-migration: | |
| strategy: | |
| fail-fast: false | |
| # The shared Perforce server only has two client license slots available | |
| # for validation jobs. | |
| max-parallel: 2 | |
| matrix: | |
| tool: # Note: If you add or change the name of tools here, you must also adjust the logic in the "Prepare environment for specific tool" step | |
| - addressAndUndefinedSanitizer # https://github.com/google/sanitizers/wiki/AddressSanitizer and https://clang.llvm.org/docs/UndefinedBehaviorSanitizer.html | |
| - threadSanitizer # https://github.com/google/sanitizers/wiki/ThreadSanitizerCppManual | |
| - valgrindMemcheck # https://valgrind.org/info/tools.html#memcheck | |
| - valgrindHelgrind # https://valgrind.org/info/tools.html#helgrind | |
| - noTool | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: "Prepare environment for specific tool" | |
| run: | | |
| # Set environment variables for future steps based on the tool being used. We can't refer to the matrix variables in the env section | |
| # of the job, so we have to do it here. | |
| # | |
| # See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-environment-variable | |
| # for more information on the method to set environment variables. | |
| bash <<"EOF" | |
| set -euxo pipefail | |
| case ${{ matrix.tool }} in | |
| noTool) | |
| CMAKE_CXX_FLAGS="-stdlib=libc++" | |
| ;; | |
| addressAndUndefinedSanitizer) | |
| BUILD_LIBCXX="true" | |
| OPENSSL_FLAGS="-fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all -nostdinc++ -nostdlib++ -stdlib=libc++ -fPIE -DPEDANTIC" | |
| CMAKE_CXX_FLAGS="-fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all -nostdinc++ -nostdlib++ -stdlib=libc++ -fPIE" | |
| CMAKE_EXE_LINKER_FLAGS="-lc++ -lc++abi -lunwind -pie" | |
| LLVM_USE_SANITIZER="Address;Undefined" | |
| ;; | |
| threadSanitizer) | |
| BUILD_LIBCXX="true" | |
| OPENSSL_FLAGS="-fsanitize=thread -fno-sanitize-recover=all -nostdinc++ -nostdlib++ -fPIE -stdlib=libc++" | |
| CMAKE_CXX_FLAGS="-fsanitize=thread -fno-sanitize-recover=all -nostdinc++ -nostdlib++ -fPIE -stdlib=libc++" | |
| CMAKE_EXE_LINKER_FLAGS="-lc++ -lc++abi -lunwind -pie" | |
| LLVM_USE_SANITIZER="Thread" | |
| ;; | |
| valgrindMemcheck) | |
| OPENSSL_FLAGS="-DPURIFY" # See https://github.com/openssl/openssl/blob/OpenSSL_1_1_1-stable/Configure#L113-L124 | |
| VALGRIND_FLAGS="--tool=memcheck --leak-check=full --show-leak-kinds=all --track-origins=yes" | |
| # Use GCC instead of Clang for Valgrind since Ubuntu ships with suppression files for glibc | |
| USE_GCC="true" | |
| ;; | |
| valgrindHelgrind) | |
| OPENSSL_FLAGS="-DPURIFY" # See https://github.com/openssl/openssl/blob/OpenSSL_1_1_1-stable/Configure#L113-L124 | |
| VALGRIND_FLAGS="--tool=helgrind --track-lockorders=yes --history-level=full" | |
| # Use GCC instead of Clang for Valgrind since Ubuntu ships with suppression files for glibc | |
| USE_GCC="true" | |
| ;; | |
| *) | |
| echo "Unknown tool: ${{ matrix.tool }}" | |
| exit 1 | |
| ;; | |
| esac | |
| { | |
| # Set the flags for compiling C++ | |
| # | |
| # Note: Valgrind's documentation recommends compiling with -O0 to avoid compiler optimizations that can | |
| # lead to inaccurate line numbers. However, we've found that compiling with -O1 is much faster (~2x) and | |
| # "should work fairly well" according to the Valgrind documentation. | |
| # | |
| # If you're getting inaccurate line numbers, try tweaking the optimization level to -O0 to see | |
| # that helps. | |
| # | |
| # See https://valgrind.org/docs/manual/quick-start.html#quick-start.prepare for more information. | |
| echo "CMAKE_CXX_FLAGS=-fno-omit-frame-pointer -g -O1 ${CMAKE_CXX_FLAGS:-""}" | |
| echo "CMAKE_EXE_LINKER_FLAGS=${CMAKE_EXE_LINKER_FLAGS:-""}" | |
| # Set the number of parallel jobs to use when compiling | |
| # See https://cmake.org/cmake/help/latest/envvar/CMAKE_BUILD_PARALLEL_LEVEL.html for more information | |
| echo "CMAKE_BUILD_PARALLEL_LEVEL=$(($(nproc) + 1))" | |
| echo "LLVM_USE_SANITIZER=${LLVM_USE_SANITIZER:-""}" | |
| # Set the flags for compiling Valgrind | |
| echo "VALGRIND_FLAGS=${VALGRIND_FLAGS:-""}" | |
| # Set the flags for compiling OpenSSL | |
| echo "OPENSSL_FLAGS=${OPENSSL_FLAGS:-""}" | |
| # Indicate whether we should build libc++ or not | |
| echo "BUILD_LIBCXX=${BUILD_LIBCXX:-"false"}" | |
| # Configure the C/C++ compilers | |
| if [[ "${USE_GCC:-"false"}" == "true" ]]; then | |
| echo 'CC=sccache gcc' | |
| echo 'CXX=sccache g++' | |
| echo 'CMAKE_C_COMPILER=gcc' | |
| echo 'CMAKE_CXX_COMPILER=g++' | |
| else | |
| # use clang | |
| echo 'CC=sccache clang-16' | |
| echo 'CXX=sccache clang++-16' | |
| echo 'CMAKE_C_COMPILER=clang-16' | |
| echo 'CMAKE_CXX_COMPILER=clang++-16' | |
| fi | |
| # Set the temporary Perforce client name to use while validating the git migration | |
| # P4Clients may not contain slashes, so eliminate them first. | |
| normalized_ref="$(echo "${{ github.ref_name }}" | sed 's/\//-/g')" | |
| echo "P4CLIENT=validate-migration-${normalized_ref}-${{ github.sha }}-${{ github.run_number }}-${{ matrix.tool }}" | |
| } >>"$GITHUB_ENV" | |
| EOF | |
| - uses: actions/checkout@v2 | |
| - name: "Run sccache" | |
| uses: mozilla-actions/sccache-action@v0.0.10 | |
| - name: "Install Perforce CLI" | |
| uses: perforce/setup-p4@1.0.2 | |
| with: | |
| command: "trust -y -f" | |
| p4_version: 23.1 | |
| - name: "Install APT package dependencies" | |
| run: | | |
| bash <<"EOF" | |
| set -euxo pipefail | |
| # Add the LLVM apt repository | |
| wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - | |
| sudo apt-add-repository "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-16 main" | |
| PACKAGES=( | |
| # C++ compilation dependencies | |
| git | |
| make | |
| cmake | |
| gcc | |
| g++ | |
| libc6-dbg | |
| ninja-build | |
| valgrind | |
| # Clang 16 dependencies | |
| clang-16 | |
| libc++-16-dev | |
| libc++abi-16-dev | |
| # For llvm-symbolizer for symbolizing stack traces | |
| llvm-16 | |
| # For envsubst in the run-test.sh script | |
| gettext # for envsubst | |
| ) | |
| sudo apt-get update | |
| sudo apt-get install --yes "${PACKAGES[@]}" | |
| EOF | |
| - name: "Cache LLVM project gzip" | |
| if : ${{ env.BUILD_LIBCXX == 'true' }} | |
| id: cache-llvm-project-source | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.LLVM_PROJECT_DOWNLOAD_DIR }} | |
| key: ${{ runner.os }}-llvm-project-gzip-${{ env.LLVM_PROJECT_DOWNLOAD_URL }} | |
| - name: "Download LLVM project" | |
| if: ${{ env.BUILD_LIBCXX == 'true' && steps.cache-llvm-project-source.outputs.cache-hit != 'true' }} | |
| run: | | |
| mkdir -p "${LLVM_PROJECT_DOWNLOAD_DIR}" || true | |
| pushd "${LLVM_PROJECT_DOWNLOAD_DIR}" | |
| curl -L "${LLVM_PROJECT_DOWNLOAD_URL}" --output llvm-project.tar.gz | |
| popd | |
| - name: "Unpack LLVM project" | |
| if: ${{ env.BUILD_LIBCXX == 'true' }} | |
| run: | | |
| mkdir -p "${LLVM_PROJECT_SOURCE_DIR}" || true | |
| tar -C "${LLVM_PROJECT_SOURCE_DIR}" -xzf "${LLVM_PROJECT_DOWNLOAD_DIR}/llvm-project.tar.gz" --strip-components 1 | |
| - name: "Install LibCXX" | |
| if: ${{ env.BUILD_LIBCXX == 'true' }} | |
| run: | | |
| bash <<"EOF" | |
| set -euxo pipefail | |
| pushd "${LLVM_PROJECT_SOURCE_DIR}" | |
| cmake -G Ninja -S runtimes -B build \ | |
| -DCMAKE_INSTALL_PREFIX="${LLVM_PROJECT_INSTALL_DIR}" \ | |
| -DCMAKE_BUILD_TYPE=Release \ | |
| -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \ | |
| -DCMAKE_C_COMPILER="${CMAKE_C_COMPILER}" \ | |
| -DCMAKE_CXX_COMPILER="${CMAKE_CXX_COMPILER}" \ | |
| -DLLVM_USE_SANITIZER="${LLVM_USE_SANITIZER:-""}" \ | |
| -DCMAKE_C_COMPILER_LAUNCHER=sccache \ | |
| -DCMAKE_CXX_COMPILER_LAUNCHER=sccache | |
| ninja --verbose -C build cxx cxxabi unwind | |
| ninja --verbose -C build install-cxx install-cxxabi install-unwind | |
| popd | |
| EOF | |
| - name: "Cache Perforce API headers gzip" | |
| id: cache-p4api | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.P4_DOWNLOAD_DIR }} | |
| key: ${{ runner.os }}-p4api-gzip-${{ env.P4_DOWNLOAD_URL }} | |
| - name: "Download Perforce API headers gzip" | |
| if: ${{ steps.cache-p4api.outputs.cache-hit != 'true' }} | |
| run: | | |
| mkdir -p "${P4_DOWNLOAD_DIR}" || true | |
| pushd "${P4_DOWNLOAD_DIR}" | |
| curl -L "${P4_DOWNLOAD_URL}" --output p4.tgz | |
| popd | |
| - name: "Unpack Perforce API headers" | |
| run: | | |
| mkdir -p vendor/helix-core-api/linux | |
| tar -C vendor/helix-core-api/linux -xzf "${P4_DOWNLOAD_DIR}/p4.tgz" --strip 1 | |
| - name: "Cache OpenSSL gzip" | |
| id: cache-openssl-source | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.OPENSSL_DOWNLOAD_DIR }} | |
| key: ${{ runner.os }}-openssl-source-gzip-v2-${{ env.OPENSSL_DOWNLOAD_URL }} | |
| - name: "Download OpenSSL gzip" | |
| if : ${{ steps.cache-openssl-source.outputs.cache-hit != 'true' }} | |
| run: | | |
| mkdir -p "${OPENSSL_DOWNLOAD_DIR}" || true | |
| pushd "${OPENSSL_DOWNLOAD_DIR}" | |
| curl -L "${OPENSSL_DOWNLOAD_URL}" --output openssl.tgz | |
| popd | |
| - name: "Compile OpenSSL" | |
| run: | | |
| bash <<"EOF" | |
| set -euxo pipefail | |
| # unpack the OpenSSL source | |
| pushd "${OPENSSL_DOWNLOAD_DIR}" | |
| mkdir -p "${OPENSSL_SOURCE_DIR}" || true | |
| tar -C "${OPENSSL_SOURCE_DIR}" -xzf openssl.tgz --strip-components 1 | |
| popd | |
| pushd "${OPENSSL_SOURCE_DIR}" | |
| CONFIG_ARGS=( | |
| --prefix="${OPENSSL_INSTALL_DIR}" | |
| --openssldir="${OPENSSL_INSTALL_DIR}" | |
| ) | |
| read -r -a array <<<"${OPENSSL_FLAGS:-}" | |
| for flag in "${array[@]}"; do | |
| if [[ -n "${flag}" ]]; then | |
| CONFIG_ARGS+=("$flag") | |
| fi | |
| done | |
| ./config "${CONFIG_ARGS[@]}" | |
| make --jobs="$(($(nproc) + 1))" | |
| sudo make install | |
| popd | |
| EOF | |
| - name: "Build P4 Fusion and run validate migration script" | |
| run: | | |
| bash <<"EOF" | |
| set -euxo pipefail | |
| export NUM_NETWORK_THREADS="$(($(nproc) + 1))" # use one more than the number of cores | |
| timeout --verbose 30m .github/workflows/validate-migration-scripts/build-p4-fusion-and-run-validate-migration.sh | |
| EOF | |
| env: | |
| OPENSSL_ROOT_DIR: ${{ env.OPENSSL_INSTALL_DIR }} | |
| USE_VALGRIND: ${{ env.VALGRIND_FLAGS != '' }} | |
| LIBCXX_INSTALL_DIR: ${{ env.LLVM_PROJECT_INSTALL_DIR }} | |
| - name: "Ensure p4 client is deleted" | |
| if: always() | |
| run: | | |
| timeout --verbose 30s .github/workflows/validate-migration-scripts/delete-perforce-client.sh |