Add scip-clang code intelligence indexing via GitHub Actions #339

Workflow file for this run

.github/workflows/validate-migration.yaml at 107a50d

	name: validate-migration

	on: [ push ]

	concurrency:
	group: ${{ github.ref }}
	cancel-in-progress: ${{ github.ref != 'refs/heads/master' }}

	env:
	# Setup: Constants for downloading and compiling dependencies
	P4_DOWNLOAD_URL: "https://filehost.perforce.com/perforce/r23.1/bin.linux26x86_64/p4api-glibc2.3-openssl1.1.1.tgz"
	P4_DOWNLOAD_DIR: "/tmp/p4-download"

	OPENSSL_DOWNLOAD_URL: "https://www.openssl.org/source/openssl-1.1.1w.tar.gz"
	OPENSSL_SOURCE_DIR: "/tmp/openssl-src"
	OPENSSL_INSTALL_DIR: "/tmp/openssl-install"
	OPENSSL_DOWNLOAD_DIR: "/tmp/openssl-download"

	LLVM_PROJECT_DOWNLOAD_URL: "https://github.com/llvm/llvm-project/archive/refs/tags/llvmorg-16.0.4.tar.gz"
	LLVM_PROJECT_DOWNLOAD_DIR: "/tmp/llvm-project-download"
	LLVM_PROJECT_SOURCE_DIR: "/tmp/llvm-project-src"
	LLVM_PROJECT_INSTALL_DIR: "/tmp/llvm-project-install"

	# Setup: Perforce credentials
	P4PORT: "ssl:perforce.sgdev.org:1666" # the address of the Perforce server to connect to
	P4USER: "admin" # the name of the Perforce user
	P4PASSWD: ${{ secrets.P4PASSWD }} # the ticket for the Perforce user

	# Setup: Enable local sccache for faster builds. Do not make compilation
	# depend on the availability of GitHub's remote artifact cache.
	CMAKE_C_COMPILER_LAUNCHER: "sccache"
	CMAKE_CXX_COMPILER_LAUNCHER: "sccache"

	# Setup: Sanitizer flags
	UBSAN_OPTIONS: "print_stacktrace=1" # print stack trace on undefined behavior
	ASAN_OPTIONS: "check_initialization_order=1" # check initialization order of globals

	# Test: Input parameters to the validate-migration test script
	DEPOT_NAME: "source/src-cli" # the name of the Perforce depot to validate the git migration against

	jobs:
	validate-migration:
	strategy:
	fail-fast: false
	# The shared Perforce server only has two client license slots available
	# for validation jobs.
	max-parallel: 2
	matrix:
	tool: # Note: If you add or change the name of tools here, you must also adjust the logic in the "Prepare environment for specific tool" step
	- addressAndUndefinedSanitizer # https://github.com/google/sanitizers/wiki/AddressSanitizer and https://clang.llvm.org/docs/UndefinedBehaviorSanitizer.html
	- threadSanitizer # https://github.com/google/sanitizers/wiki/ThreadSanitizerCppManual
	- valgrindMemcheck # https://valgrind.org/info/tools.html#memcheck
	- valgrindHelgrind # https://valgrind.org/info/tools.html#helgrind
	- noTool

	runs-on: ubuntu-22.04

	steps:
	- name: "Prepare environment for specific tool"
	run: \|
	# Set environment variables for future steps based on the tool being used. We can't refer to the matrix variables in the env section
	# of the job, so we have to do it here.
	#
	# See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-environment-variable
	# for more information on the method to set environment variables.

	bash <<"EOF"

	set -euxo pipefail

	case ${{ matrix.tool }} in

	noTool)
	CMAKE_CXX_FLAGS="-stdlib=libc++"
	;;

	addressAndUndefinedSanitizer)
	BUILD_LIBCXX="true"
	OPENSSL_FLAGS="-fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all -nostdinc++ -nostdlib++ -stdlib=libc++ -fPIE -DPEDANTIC"
	CMAKE_CXX_FLAGS="-fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all -nostdinc++ -nostdlib++ -stdlib=libc++ -fPIE"
	CMAKE_EXE_LINKER_FLAGS="-lc++ -lc++abi -lunwind -pie"
	LLVM_USE_SANITIZER="Address;Undefined"
	;;

	threadSanitizer)
	BUILD_LIBCXX="true"
	OPENSSL_FLAGS="-fsanitize=thread -fno-sanitize-recover=all -nostdinc++ -nostdlib++ -fPIE -stdlib=libc++"
	CMAKE_CXX_FLAGS="-fsanitize=thread -fno-sanitize-recover=all -nostdinc++ -nostdlib++ -fPIE -stdlib=libc++"
	CMAKE_EXE_LINKER_FLAGS="-lc++ -lc++abi -lunwind -pie"
	LLVM_USE_SANITIZER="Thread"
	;;

	valgrindMemcheck)
	OPENSSL_FLAGS="-DPURIFY" # See https://github.com/openssl/openssl/blob/OpenSSL_1_1_1-stable/Configure#L113-L124
	VALGRIND_FLAGS="--tool=memcheck --leak-check=full --show-leak-kinds=all --track-origins=yes"

	# Use GCC instead of Clang for Valgrind since Ubuntu ships with suppression files for glibc
	USE_GCC="true"
	;;

	valgrindHelgrind)
	OPENSSL_FLAGS="-DPURIFY" # See https://github.com/openssl/openssl/blob/OpenSSL_1_1_1-stable/Configure#L113-L124
	VALGRIND_FLAGS="--tool=helgrind --track-lockorders=yes --history-level=full"

	# Use GCC instead of Clang for Valgrind since Ubuntu ships with suppression files for glibc
	USE_GCC="true"
	;;

	*)
	echo "Unknown tool: ${{ matrix.tool }}"
	exit 1
	;;

	esac

	{
	# Set the flags for compiling C++
	#
	# Note: Valgrind's documentation recommends compiling with -O0 to avoid compiler optimizations that can
	# lead to inaccurate line numbers. However, we've found that compiling with -O1 is much faster (~2x) and
	# "should work fairly well" according to the Valgrind documentation.
	#
	# If you're getting inaccurate line numbers, try tweaking the optimization level to -O0 to see
	# that helps.
	#
	# See https://valgrind.org/docs/manual/quick-start.html#quick-start.prepare for more information.
	echo "CMAKE_CXX_FLAGS=-fno-omit-frame-pointer -g -O1 ${CMAKE_CXX_FLAGS:-""}"

	echo "CMAKE_EXE_LINKER_FLAGS=${CMAKE_EXE_LINKER_FLAGS:-""}"

	# Set the number of parallel jobs to use when compiling
	# See https://cmake.org/cmake/help/latest/envvar/CMAKE_BUILD_PARALLEL_LEVEL.html for more information
	echo "CMAKE_BUILD_PARALLEL_LEVEL=$(($(nproc) + 1))"

	echo "LLVM_USE_SANITIZER=${LLVM_USE_SANITIZER:-""}"

	# Set the flags for compiling Valgrind
	echo "VALGRIND_FLAGS=${VALGRIND_FLAGS:-""}"

	# Set the flags for compiling OpenSSL
	echo "OPENSSL_FLAGS=${OPENSSL_FLAGS:-""}"

	# Indicate whether we should build libc++ or not
	echo "BUILD_LIBCXX=${BUILD_LIBCXX:-"false"}"

	# Configure the C/C++ compilers

	if [[ "${USE_GCC:-"false"}" == "true" ]]; then
	echo 'CC=sccache gcc'
	echo 'CXX=sccache g++'
	echo 'CMAKE_C_COMPILER=gcc'
	echo 'CMAKE_CXX_COMPILER=g++'
	else
	# use clang
	echo 'CC=sccache clang-16'
	echo 'CXX=sccache clang++-16'
	echo 'CMAKE_C_COMPILER=clang-16'
	echo 'CMAKE_CXX_COMPILER=clang++-16'
	fi

	# Set the temporary Perforce client name to use while validating the git migration
	# P4Clients may not contain slashes, so eliminate them first.
	normalized_ref="$(echo "${{ github.ref_name }}" \| sed 's/\//-/g')"
	echo "P4CLIENT=validate-migration-${normalized_ref}-${{ github.sha }}-${{ github.run_number }}-${{ matrix.tool }}"
	} >>"$GITHUB_ENV"

	EOF

	- uses: actions/checkout@v2

	- name: "Run sccache"
	uses: mozilla-actions/sccache-action@v0.0.10

	- name: "Install Perforce CLI"
	uses: perforce/setup-p4@1.0.2
	with:
	command: "trust -y -f"
	p4_version: 23.1

	- name: "Install APT package dependencies"
	run: \|
	bash <<"EOF"

	set -euxo pipefail

	# Add the LLVM apt repository
	wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key \| sudo apt-key add -
	sudo apt-add-repository "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-16 main"

	PACKAGES=(
	# C++ compilation dependencies
	git
	make
	cmake
	gcc
	g++
	libc6-dbg
	ninja-build
	valgrind

	# Clang 16 dependencies
	clang-16
	libc++-16-dev
	libc++abi-16-dev

	# For llvm-symbolizer for symbolizing stack traces
	llvm-16

	# For envsubst in the run-test.sh script
	gettext # for envsubst
	)

	sudo apt-get update
	sudo apt-get install --yes "${PACKAGES[@]}"

	EOF

	- name: "Cache LLVM project gzip"
	if : ${{ env.BUILD_LIBCXX == 'true' }}
	id: cache-llvm-project-source
	uses: actions/cache@v4
	with:
	path: ${{ env.LLVM_PROJECT_DOWNLOAD_DIR }}
	key: ${{ runner.os }}-llvm-project-gzip-${{ env.LLVM_PROJECT_DOWNLOAD_URL }}

	- name: "Download LLVM project"
	if: ${{ env.BUILD_LIBCXX == 'true' && steps.cache-llvm-project-source.outputs.cache-hit != 'true' }}
	run: \|
	mkdir -p "${LLVM_PROJECT_DOWNLOAD_DIR}" \|\| true
	pushd "${LLVM_PROJECT_DOWNLOAD_DIR}"

	curl -L "${LLVM_PROJECT_DOWNLOAD_URL}" --output llvm-project.tar.gz
	popd

	- name: "Unpack LLVM project"
	if: ${{ env.BUILD_LIBCXX == 'true' }}
	run: \|
	mkdir -p "${LLVM_PROJECT_SOURCE_DIR}" \|\| true
	tar -C "${LLVM_PROJECT_SOURCE_DIR}" -xzf "${LLVM_PROJECT_DOWNLOAD_DIR}/llvm-project.tar.gz" --strip-components 1

	- name: "Install LibCXX"
	if: ${{ env.BUILD_LIBCXX == 'true' }}
	run: \|
	bash <<"EOF"

	set -euxo pipefail

	pushd "${LLVM_PROJECT_SOURCE_DIR}"

	cmake -G Ninja -S runtimes -B build \
	-DCMAKE_INSTALL_PREFIX="${LLVM_PROJECT_INSTALL_DIR}" \
	-DCMAKE_BUILD_TYPE=Release \
	-DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \
	-DCMAKE_C_COMPILER="${CMAKE_C_COMPILER}" \
	-DCMAKE_CXX_COMPILER="${CMAKE_CXX_COMPILER}" \
	-DLLVM_USE_SANITIZER="${LLVM_USE_SANITIZER:-""}" \
	-DCMAKE_C_COMPILER_LAUNCHER=sccache \
	-DCMAKE_CXX_COMPILER_LAUNCHER=sccache

	ninja --verbose -C build cxx cxxabi unwind
	ninja --verbose -C build install-cxx install-cxxabi install-unwind

	popd

	EOF

	- name: "Cache Perforce API headers gzip"
	id: cache-p4api
	uses: actions/cache@v4
	with:
	path: ${{ env.P4_DOWNLOAD_DIR }}
	key: ${{ runner.os }}-p4api-gzip-${{ env.P4_DOWNLOAD_URL }}

	- name: "Download Perforce API headers gzip"
	if: ${{ steps.cache-p4api.outputs.cache-hit != 'true' }}
	run: \|
	mkdir -p "${P4_DOWNLOAD_DIR}" \|\| true

	pushd "${P4_DOWNLOAD_DIR}"
	curl -L "${P4_DOWNLOAD_URL}" --output p4.tgz

	popd

	- name: "Unpack Perforce API headers"
	run: \|
	mkdir -p vendor/helix-core-api/linux
	tar -C vendor/helix-core-api/linux -xzf "${P4_DOWNLOAD_DIR}/p4.tgz" --strip 1

	- name: "Cache OpenSSL gzip"
	id: cache-openssl-source
	uses: actions/cache@v4
	with:
	path: ${{ env.OPENSSL_DOWNLOAD_DIR }}
	key: ${{ runner.os }}-openssl-source-gzip-v2-${{ env.OPENSSL_DOWNLOAD_URL }}

	- name: "Download OpenSSL gzip"
	if : ${{ steps.cache-openssl-source.outputs.cache-hit != 'true' }}
	run: \|
	mkdir -p "${OPENSSL_DOWNLOAD_DIR}" \|\| true
	pushd "${OPENSSL_DOWNLOAD_DIR}"

	curl -L "${OPENSSL_DOWNLOAD_URL}" --output openssl.tgz
	popd

	- name: "Compile OpenSSL"
	run: \|
	bash <<"EOF"

	set -euxo pipefail

	# unpack the OpenSSL source
	pushd "${OPENSSL_DOWNLOAD_DIR}"

	mkdir -p "${OPENSSL_SOURCE_DIR}" \|\| true
	tar -C "${OPENSSL_SOURCE_DIR}" -xzf openssl.tgz --strip-components 1

	popd

	pushd "${OPENSSL_SOURCE_DIR}"

	CONFIG_ARGS=(
	--prefix="${OPENSSL_INSTALL_DIR}"
	--openssldir="${OPENSSL_INSTALL_DIR}"
	)

	read -r -a array <<<"${OPENSSL_FLAGS:-}"
	for flag in "${array[@]}"; do
	if [[ -n "${flag}" ]]; then
	CONFIG_ARGS+=("$flag")
	fi
	done

	./config "${CONFIG_ARGS[@]}"
	make --jobs="$(($(nproc) + 1))"

	sudo make install

	popd

	EOF

	- name: "Build P4 Fusion and run validate migration script"
	run: \|
	bash <<"EOF"

	set -euxo pipefail

	export NUM_NETWORK_THREADS="$(($(nproc) + 1))" # use one more than the number of cores

	timeout --verbose 30m .github/workflows/validate-migration-scripts/build-p4-fusion-and-run-validate-migration.sh

	EOF
	env:
	OPENSSL_ROOT_DIR: ${{ env.OPENSSL_INSTALL_DIR }}
	USE_VALGRIND: ${{ env.VALGRIND_FLAGS != '' }}
	LIBCXX_INSTALL_DIR: ${{ env.LLVM_PROJECT_INSTALL_DIR }}

	- name: "Ensure p4 client is deleted"
	if: always()
	run: \|
	timeout --verbose 30s .github/workflows/validate-migration-scripts/delete-perforce-client.sh

Navigation Menu

Uh oh!

Add scip-clang code intelligence indexing via GitHub Actions #339

Workflow file

Add scip-clang code intelligence indexing via GitHub Actions #339

Uh oh!

Workflow file for this run