From 9ca498efdd165f3340a02667b8275562f356c407 Mon Sep 17 00:00:00 2001 From: egpast Date: Tue, 28 Jun 2022 01:11:16 -0700 Subject: [PATCH] Fix dev branch CircleCI Currently, all dev publish_docs jobs are failing the Java 11 requirement (https://github.com/facebook/buck/blob/dev/build.xml#L365-L367 ). A lot of things need to be changed to pass that requirement: - Install OpenJDK 11 instead of 8 - Install new Android command-line tools. The old SDK tools don't support JDK 8 (https://stackoverflow.com/a/65782803 ). Oddly, the *new* tools don't install to the right location (https://stackoverflow.com/a/67413427 ), so they need to be moved around a little before they'll work. - Install Python 3.9.4 (required by Ant) - Build Buck! Release-branch builds require Java 8, so we can't use those and hope to pass the version check. - Pass down our local Buck executable to docs/publish.sh and docs/soyweb-prod.sh. By default, these scripts will try to use the Ant-bootstrapped Java 8 Buck instead of our locally built Java 11 Buck. This has been tested by adding a couple tweaks in a separate branch (https://github.com/egpast/buck/commits/dev-force-circleci ) to force publish_docs to run on commits and stop it from actually publishing, and then confirming that CircleCI passes (https://app.circleci.com/pipelines/github/egpast/buck/25/workflows/b049df7e-9e0c-49a8-95d5-eee82aeb40bc/jobs/17 ). --- .circleci/config.yml | 69 +++++++++++++++++++++++++++++--------------- docs/publish.sh | 18 ++++++++---- docs/soyweb-prod.sh | 24 ++++++++++++++- 3 files changed, 81 insertions(+), 30 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 35950fdc796..54eee755569 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3,14 +3,15 @@ version: 2.1 orbs: win: circleci/windows@2.2.0 -install_openjdk8: &install_openjdk8 - name: Install OpenJDK8 +install_openjdk11: &install_openjdk11 + name: Install OpenJDK11 command: | if [ "${PLATFORM}" == "linux" ]; then - sudo apt-get update && sudo apt-get install openjdk-8-jdk - sudo update-java-alternatives -s java-1.8.0-openjdk-amd64 + sudo apt-get update && sudo apt-get install openjdk-11-jdk + sudo update-java-alternatives -s java-1.11.0-openjdk-amd64 elif [ "${PLATFORM}" == "macos" ]; then - brew cask install adoptopenjdk8 + brew install openjdk@11 + export PATH="/usr/local/opt/openjdk@11/bin:$PATH" fi java -version @@ -19,14 +20,15 @@ install_android_sdk: &install_android_sdk command: | sdk_os="linux" if [ "${PLATFORM}" == "macos" ]; then - sdk_os="darwin" + sdk_os="mac" fi - sdk_zip_filename="sdk-tools-${sdk_os}-4333796.zip" - mkdir -p "${ANDROID_SDK}" - cd "${ANDROID_SDK}" + sdk_zip_filename="commandlinetools-${sdk_os}-8512546_latest.zip" + mkdir -p "${ANDROID_SDK}/cmdline-tools" + cd "${ANDROID_SDK}/cmdline-tools" curl -O "https://dl.google.com/android/repository/${sdk_zip_filename}" unzip "${sdk_zip_filename}" - export PATH="${ANDROID_SDK}/tools/bin:${PATH}" + mv cmdline-tools latest + export PATH="${ANDROID_SDK}/cmdline-tools/latest/bin:${PATH}" echo 'y' |sdkmanager --install tools echo 'y' |sdkmanager --install platform-tools echo 'y' |sdkmanager --install "build-tools;28.0.0" @@ -40,16 +42,39 @@ install_android_sdk: &install_android_sdk fi install_python: &install_python - name: Install Python 3.6.2 + name: Install Python 3.9.4 command: | if [ "${PLATFORM}" == "macos" ]; then brew install pyenv fi - pyenv install -s 3.6.2 - pyenv global 3.6.2 system + pyenv install -s 3.9.4 + pyenv global 3.9.4 system export PATH="$(pyenv root)/shims:${PATH}" python --version +run_ant_build: &run_ant_build + name: Run Ant Build + command: | + if [ "${PLATFORM}" == "macos" ]; then + # The latest ant depends on JDK13, install 1.9 instead. + brew install ant@1.9 + export PATH="/usr/local/opt/ant@1.9/bin:${PATH}" + fi + cd "${BUCKROOT}" + set -ex + export ANT_OPTS='-Xmx1000m' + ant + +run_buck_build: &run_buck_build + name: Run Buck Build + command: | + cd "${BUCKROOT}" + echo '-Xmx1024m' > .buckjavaargs.local + export PATH="${ANDROID_SDK}/tools/bin:${PATH}" + export PATH="$(pyenv root)/shims:${PATH}" + set -ex + ./bin/buck build buck --out "./${BUCK_PEX_LOCATION}" || { cat "buck-out/log/buck-0.log"; exit 1; } + linux_environment: &linux_environment # Use string constant for values, no environment variables PLATFORM: "linux" @@ -57,7 +82,7 @@ linux_environment: &linux_environment ANDROID_SDK: "/home/circleci/android-sdk" TERM: "dumb" BUCK_NUM_THREADS: 3 - BUCK_PEX_LOCATION: "./new_buck.pex" + BUCK_PEX_LOCATION: "new_buck.pex" jobs: publish_docs: @@ -69,26 +94,24 @@ jobs: steps: - checkout - run: - <<: *install_openjdk8 + <<: *install_openjdk11 - run: # android_sdk needed to build java docs. <<: *install_android_sdk - run: <<: *install_python - run: - # We do not want to build buck, install the latest release instead. - name: Install Buck - command: | - url=`curl -sH "Authorization: token ${GITHUB_TOKEN}" https://api.github.com/repos/facebook/buck/releases/latest |grep "browser_download_url.*deb" |awk '{gsub("\"", "", $2); print $2}'` - curl -L -O $url - filename=`basename ${url}` - sudo dpkg -i ${filename} || echo "Warning: Buck installed without dependencies." + <<: *run_ant_build + - restore_cache: + key: v-{{ .Environment.CACHE_VERSION }}-buck-build-{{ .Branch }} + - run: + <<: *run_buck_build - run: name: Publish docs command: | export ANDROID_HOME="${ANDROID_SDK}" cd docs - ./publish.sh --start-soyweb + ./publish.sh --buck-location "${BUCKROOT}/${BUCK_PEX_LOCATION}" --start-soyweb workflows: version: 2.1 diff --git a/docs/publish.sh b/docs/publish.sh index 61bf554a217..6e371b2c79e 100755 --- a/docs/publish.sh +++ b/docs/publish.sh @@ -18,7 +18,7 @@ set -e start_soyweb() { echo "Starting soyweb-prod.sh" >&2 - ./soyweb-prod.sh >&2 & + ./soyweb-prod.sh --buck-location "${BUCK_LOCATION}" >&2 & local soyweb_pid=$! sleep 2 if ! kill -0 "$soyweb_pid" >/dev/null 2>&1; then @@ -31,10 +31,11 @@ start_soyweb() { show_help() { cat <<-EOF -Usage: publish.sh [--start-soyweb] [--keep-files] - --start-soyweb Start soyweb and shut it down when the script is finished - --keep-files Keep temporary files after attempting to publish - --help Show this help +Usage: publish.sh [--buck-location BUCK_LOCATION] [--start-soyweb] [--keep-files] + --buck-location Sets the location of the Buck executable to use + --start-soyweb Start soyweb and shut it down when the script is finished + --keep-files Keep temporary files after attempting to publish + --help Show this help Set the environment variables GIT_USER and GITHUB_TOKEN if you want to publish using another GitHub account. These variables are already set in CircleCI to automate publishing. @@ -42,12 +43,17 @@ EOF exit 1 } +BUCK_LOCATION="buck" START_SOYWEB=0 KEEP_FILES=0 SOYWEB_PID=0 for arg do shift case $arg in + --buck-location) + BUCK_LOCATION="$1" + shift + ;; --start-soyweb) START_SOYWEB=1 ;; --keep-files) KEEP_FILES=1 ;; --help) show_help ;; @@ -66,7 +72,7 @@ STATIC_FILES_DIR=$(mktemp -d) # Always run this from the the docs dir cd "$DOCS_DIR" -buck run //docs:generate_buckconfig_aliases +"${BUCK_LOCATION}" run //docs:generate_buckconfig_aliases if ( [[ -n $IS_GIT ]] && ! git diff --quiet ) || hg status | grep -q .; then echo "Repository is not clean; refusing to publish" diff --git a/docs/soyweb-prod.sh b/docs/soyweb-prod.sh index 112c2de4025..a07b121d397 100755 --- a/docs/soyweb-prod.sh +++ b/docs/soyweb-prod.sh @@ -20,6 +20,28 @@ # Remove any residual files that could derail build and publication. # +show_help() { + cat <<-EOF +Usage: soyweb-prod.sh [--buck-location BUCK_LOCATION] + --buck-location Sets the location of the Buck executable to use + --help Show this help +EOF + exit 1 +} + +BUCK_LOCATION="buck" +for arg do + shift + case $arg in + --buck-location) + BUCK_LOCATION="$1" + shift + ;; + --help) show_help ;; + *) set -- "$@" "$arg" ;; + esac +done + DOCS_DIR=$(dirname "$0") BUCK_DIR=$(realpath "$DOCS_DIR/..") DOCS_DIR=$(realpath "$DOCS_DIR") @@ -28,6 +50,6 @@ cd "$BUCK_DIR" || exit ant clean cd "$BUCK_DIR/docs" || exit -buck run //docs:generate_buckconfig_aliases +"${BUCK_LOCATION}" run //docs:generate_buckconfig_aliases exec java -jar plovr-81ed862.jar soyweb --port 9814 --dir . --globals globals.json