Build (3.1) #15672
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Licensed to the Apache Software Foundation (ASF) under one | |
| # or more contributor license agreements. See the NOTICE file | |
| # distributed with this work for additional information | |
| # regarding copyright ownership. The ASF licenses this file | |
| # to you under the Apache License, Version 2.0 (the | |
| # "License"); you may not use this file except in compliance | |
| # with the License. You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, | |
| # software distributed under the License is distributed on an | |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| # KIND, either express or implied. See the License for the | |
| # specific language governing permissions and limitations | |
| # under the License. | |
| name: Build (3.1) | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| force_build: | |
| description: "Force run build job when manually triggered" | |
| required: false | |
| default: "true" | |
| schedule: | |
| - cron: '*/30 * * * *' | |
| # A full third party build runs for hours while the schedule fires every 30 | |
| # minutes. Without a concurrency group the runs stack up and race each other on | |
| # the same release tag. | |
| concurrency: | |
| group: ${{ github.workflow }} | |
| cancel-in-progress: false | |
| jobs: | |
| prerelease: | |
| name: Prerelease | |
| # ubuntu-latest ships GNU md5sum. On a macOS runner download-thirdparty.sh | |
| # finds no md5sum, skips verification altogether and packs whatever it got | |
| # into the source tarball, including the 0-byte files a failed download | |
| # leaves behind. The build jobs then fail on a checksum the prerelease job | |
| # never looked at. | |
| runs-on: ubuntu-latest | |
| env: | |
| GH_REPO: ${{ github.repository }} | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| permissions: | |
| contents: write | |
| outputs: | |
| should_release: ${{ steps.check_diff.outputs.should_release }} | |
| doris_version: ${{ steps.check_diff.outputs.doris_version }} | |
| attempt: ${{ steps.check_diff.outputs.attempt }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v5 | |
| with: | |
| repository: 'apache/doris' | |
| ref: 'branch-3.1' | |
| fetch-depth: 0 | |
| - name: Check Diff | |
| id: check_diff | |
| run: | | |
| tag_name='automation-3.1' | |
| title="Apache Doris Third Party Prebuilt (${tag_name/automation-/})" | |
| max_attempts=3 | |
| if [[ -z "$(gh release list)" ]] || | |
| ! gh release list | awk -F "\t" '{ print $3 }' | grep "${tag_name}" >/dev/null; then | |
| gh release create -t "${title}" "${tag_name}" | |
| fi | |
| # The release note is the only state this pipeline keeps. Read it once | |
| # so the three fields below cannot come from three different revisions. | |
| note="$(gh release view "${tag_name}")" | |
| last_version="$(echo "${note}" | sed -n -E 's/Doris Version: \*(.*)\*.*/\1/p')" | |
| last_status="$(echo "${note}" | sed -n -E 's/Status: \*(.*)\*.*/\1/p')" | |
| last_attempt="$(echo "${note}" | sed -n -E 's/Attempts: \*([0-9]+)\*.*/\1/p')" | |
| # A note without a usable counter must not enable retries: treating it | |
| # as attempt 0 is what turns every scheduled run into a full rebuild. | |
| [[ "${last_attempt}" =~ ^[0-9]+$ ]] || last_attempt="${max_attempts}" | |
| current_version="$(git log -1 --format='%H')" | |
| echo "Last Version: ${last_version}" | |
| echo "Last Status: ${last_status}" | |
| echo "Last Attempt: ${last_attempt}" | |
| echo "Current Version: ${current_version}" | |
| should_release=false | |
| attempt=1 | |
| if [[ -z "${last_version}" ]]; then | |
| echo "The first release was detected." | |
| should_release=true | |
| elif [[ "${last_version}" != "${current_version}" ]]; then | |
| cmd="git diff --name-only ${last_version} ${current_version} | grep -E '^thirdparty/'" | |
| echo "Execute: ${cmd}" | |
| content="$(eval "${cmd}")" || true | |
| if [[ -n "${content}" ]]; then | |
| echo -e "Detect changes:\n${content}" | |
| should_release=true | |
| fi | |
| elif [[ "${last_status}" == 'FAILURE' ]] && [[ "${last_attempt}" -lt "${max_attempts}" ]]; then | |
| # Downloads from third party mirrors fail often enough that a single | |
| # red run should not park the branch until the next thirdparty/ | |
| # commit lands. Retry a bounded number of times, never forever. | |
| attempt=$((last_attempt + 1)) | |
| echo "Previous build failed, retrying (attempt ${attempt}/${max_attempts})." | |
| should_release=true | |
| fi | |
| if "${should_release}"; then | |
| echo -ne "Update Time: *$(date)*\nDoris Version: *${current_version}*\nStatus: *BUILDING*\nAttempts: *${attempt}*" >release_note.md | |
| else | |
| gh release view "${tag_name}" | sed -n '/--/,$p' | awk '{ if (NR > 1) print $0 }' | sed "{ | |
| s/Update Time:.*/Update Time: *$(date)*/ | |
| s/Doris Version:.*/Doris Version: *${current_version}*/ | |
| }" >release_note.md | |
| fi | |
| gh release edit -F release_note.md "${tag_name}" | |
| echo "should_release=${should_release}" >> $GITHUB_OUTPUT | |
| echo "doris_version=${current_version}" >> $GITHUB_OUTPUT | |
| echo "attempt=${attempt}" >> $GITHUB_OUTPUT | |
| - name: Download Source and Upload | |
| if: steps.check_diff.outputs.should_release == 'true' | |
| run: | | |
| tag_name='automation-3.1' | |
| cd thirdparty | |
| # Pre-download packages whose mirror in branch-3.1's vars.sh is blocked | |
| # or gone on GH Actions runners: | |
| # lzo: fossies.org returns 410 Gone | |
| # libuuid: nchc.dl.sourceforge.net (TW mirror) is unreachable | |
| # Both are fixed upstream in apache/doris; this keeps the pipeline | |
| # green until that lands and is a no-op once it does, because the | |
| # checksums are identical. | |
| mkdir -p src | |
| curl -fL "https://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz" \ | |
| -o src/lzo-2.10.tar.gz | |
| curl -fL "https://downloads.sourceforge.net/project/libuuid/libuuid-1.0.3.tar.gz" \ | |
| -o src/libuuid-1.0.3.tar.gz | |
| sed '/# unpacking thirdpart archives/,$d' download-thirdparty.sh | bash - | |
| tar -zcvf doris-thirdparty-source.tgz src | |
| gh release delete-asset "${tag_name}" doris-thirdparty-source.tgz --yes || true | |
| gh release upload --clobber "${tag_name}" doris-thirdparty-source.tgz | |
| build: | |
| name: Build | |
| needs: prerelease | |
| if: needs.prerelease.outputs.should_release == 'true' || (github.event_name == 'workflow_dispatch' && github.event.inputs.force_build == 'true') | |
| strategy: | |
| # One platform going red must not cancel the others: a cancelled matrix | |
| # leg reports nothing, so a single failure used to hide the real state of | |
| # the other two platforms. | |
| fail-fast: false | |
| matrix: | |
| config: | |
| - name: macOS-x86_64 | |
| os: macos-15-intel | |
| # Intel macOS runners are on their way out; do not let them block | |
| # the arm64 and Linux artifacts. | |
| continue-on-error: true | |
| packages: >- | |
| 'm4' | |
| 'automake' | |
| 'autoconf' | |
| 'libtool' | |
| 'pkg-config' | |
| 'texinfo' | |
| 'coreutils' | |
| 'gnu-getopt' | |
| 'python@3' | |
| 'ninja' | |
| 'ccache' | |
| 'bison' | |
| 'byacc' | |
| 'gettext' | |
| 'wget' | |
| 'pcre' | |
| 'openjdk@11' | |
| 'maven' | |
| 'node' | |
| 'llvm@16' | |
| - name: macOS-arm64 | |
| os: macos-14 | |
| packages: >- | |
| 'm4' | |
| 'automake' | |
| 'autoconf' | |
| 'libtool' | |
| 'pkg-config' | |
| 'texinfo' | |
| 'coreutils' | |
| 'gnu-getopt' | |
| 'python@3' | |
| 'ninja' | |
| 'ccache' | |
| 'bison' | |
| 'byacc' | |
| 'gettext' | |
| 'wget' | |
| 'pcre' | |
| 'openjdk@11' | |
| 'maven' | |
| 'node' | |
| 'llvm@16' | |
| - name: Linux | |
| os: ubuntu-22.04 | |
| packages: >- | |
| 'build-essential' | |
| 'automake' | |
| 'autoconf' | |
| 'libtool-bin' | |
| 'pkg-config' | |
| 'cmake=3.22.1-1ubuntu1.22.04.2' | |
| 'ninja-build' | |
| 'ccache' | |
| 'python-is-python3' | |
| 'bison' | |
| 'byacc' | |
| 'flex' | |
| 'binutils-dev' | |
| 'libiberty-dev' | |
| 'curl' | |
| 'git' | |
| 'zip' | |
| 'unzip' | |
| 'autopoint' | |
| 'openjdk-8-jdk' | |
| 'openjdk-8-jdk-headless' | |
| 'maven' | |
| runs-on: ${{ matrix.config.os }} | |
| continue-on-error: ${{ matrix.config.continue-on-error == true }} | |
| env: | |
| GH_REPO: ${{ github.repository }} | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| steps: | |
| # easimon/maximize-build-space is deliberately not used here. It hands the | |
| # reclaimed space to an LVM mounted on the workspace but reserves only | |
| # root-reserve-mb on /, and gcc writes its assembler temporaries to /tmp on | |
| # /. That is how the 2.0 pipeline died on "No space left on device" while | |
| # building aws-sdk-cpp. The stock runner disk is what master and 4.0 build | |
| # on today. | |
| - name: Checkout | |
| uses: actions/checkout@v5 | |
| with: | |
| repository: 'apache/doris' | |
| ref: 'branch-3.1' | |
| fetch-depth: 0 | |
| - name: Download | |
| run: | | |
| tag_name='automation-3.1' | |
| cd thirdparty | |
| curl -L "https://github.com/${{ github.repository }}/releases/download/${tag_name}/doris-thirdparty-source.tgz" \ | |
| -o doris-thirdparty-source.tgz | |
| tar -zxvf doris-thirdparty-source.tgz | |
| - name: Prepare for ${{ matrix.config.os }} | |
| run: | | |
| if [[ "${{ matrix.config.name }}" =~ macOS-* ]]; then | |
| # Install packages except cmake | |
| brew install ${{ matrix.config.packages }} || true | |
| # Install specific version of cmake | |
| brew unlink cmake || true | |
| wget https://github.com/Kitware/CMake/releases/download/v3.22.1/cmake-3.22.1-macos-universal.tar.gz | |
| tar -xzf cmake-3.22.1-macos-universal.tar.gz | |
| sudo cp -r cmake-3.22.1-macos-universal/CMake.app/Contents/* /usr/local/ | |
| cmake --version | |
| else | |
| export DEFAULT_DIR='/opt/doris' | |
| export PATH="${DEFAULT_DIR}/ldb-toolchain/bin:${PATH}" | |
| sudo apt update | |
| sudo apt-cache policy cmake | |
| sudo DEBIAN_FRONTEND=noninteractive apt install --yes ${{ matrix.config.packages }} | |
| mkdir -p "${DEFAULT_DIR}" | |
| wget https://github.com/amosbird/ldb_toolchain_gen/releases/download/v0.18/ldb_toolchain_gen.sh \ | |
| -q -O /tmp/ldb_toolchain_gen.sh | |
| bash /tmp/ldb_toolchain_gen.sh "${DEFAULT_DIR}/ldb-toolchain" | |
| fi | |
| - name: Build and Upload | |
| run: | | |
| tag_name='automation-3.1' | |
| if [[ "${{ matrix.config.name }}" == 'Linux' ]]; then | |
| export DEFAULT_DIR='/opt/doris' | |
| export PATH="${DEFAULT_DIR}/ldb-toolchain/bin:${PATH}" | |
| export PATH="$(find /usr/lib/jvm/java-8-openjdk* -maxdepth 1 -type d -name 'bin'):${PATH}" | |
| export JAVA_HOME="$(find /usr/lib/jvm/java-8-openjdk* -maxdepth 0)" | |
| export DORIS_TOOLCHAIN=gcc | |
| else | |
| export MACOSX_DEPLOYMENT_TARGET=12.0 | |
| fi | |
| export CMAKE_POLICY_VERSION_MINIMUM=3.10 | |
| export CUSTOM_CMAKE="/usr/local/bin/cmake" | |
| cd thirdparty | |
| # Repair packages that may be empty if the source tarball was built | |
| # before the mirror fixes: | |
| # lzo: fossies.org returns 410 Gone | |
| # libuuid: nchc.dl.sourceforge.net (TW mirror) is unreachable | |
| if [[ ! -s src/lzo-2.10.tar.gz ]]; then | |
| curl -fL "https://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz" \ | |
| -o src/lzo-2.10.tar.gz | |
| fi | |
| if [[ ! -s src/libuuid-1.0.3.tar.gz ]]; then | |
| curl -fL "https://downloads.sourceforge.net/project/libuuid/libuuid-1.0.3.tar.gz" \ | |
| -o src/libuuid-1.0.3.tar.gz | |
| fi | |
| ./build-thirdparty.sh -j "$(nproc)" | |
| kernel="$(uname -s | awk '{print tolower($0)}')" | |
| arch="$(uname -m)" | |
| rm -rf "doris-thirdparty-prebuilt-${kernel}-${arch}.tar.xz" | |
| tar -cf - installed | xz -z -T0 - >"doris-thirdparty-prebuilt-${kernel}-${arch}.tar.xz" | |
| # Drop the old asset first: `gh release upload --clobber` intermittently | |
| # exits 1 with "release not found" while deleting the asset it is | |
| # replacing, even though the upload itself went through. | |
| gh release delete-asset "${tag_name}" "doris-thirdparty-prebuilt-${kernel}-${arch}.tar.xz" --yes || true | |
| gh release upload --clobber "${tag_name}" "doris-thirdparty-prebuilt-${kernel}-${arch}.tar.xz" | |
| success: | |
| name: Success | |
| needs: [prerelease, build] | |
| if: needs.prerelease.outputs.should_release == 'true' || (github.event_name == 'workflow_dispatch' && github.event.inputs.force_build == 'true') | |
| runs-on: ubuntu-latest | |
| env: | |
| GH_REPO: ${{ github.repository }} | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| DORIS_VERSION: ${{ needs.prerelease.outputs.doris_version }} | |
| permissions: | |
| contents: write | |
| steps: | |
| - name: Update Checksums | |
| run: | | |
| tag_name='automation-3.1' | |
| gh release download "${tag_name}" | |
| # Write the version this run actually built rather than parsing it back | |
| # out of the note, so the terminal state never depends on the note | |
| # having survived intact. | |
| echo -ne "Update Time: *$(date)*\nDoris Version: *${DORIS_VERSION}*\nStatus: *SUCCESS*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md | |
| gh release edit -F release_note.md "${tag_name}" | |
| failure: | |
| name: Failure | |
| needs: [prerelease, build] | |
| # Only report on a build that actually ran. If prerelease itself failed there | |
| # is no version to record, and writing an empty one is exactly what used to | |
| # make the next scheduled run rebuild from scratch, forever. | |
| if: always() && failure() && needs.prerelease.result == 'success' | |
| runs-on: ubuntu-latest | |
| env: | |
| GH_REPO: ${{ github.repository }} | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| DORIS_VERSION: ${{ needs.prerelease.outputs.doris_version }} | |
| ATTEMPT: ${{ needs.prerelease.outputs.attempt }} | |
| permissions: | |
| contents: write | |
| steps: | |
| - name: Update Checksums | |
| run: | | |
| tag_name='automation-3.1' | |
| gh release download "${tag_name}" | |
| # Keep Doris Version: the prerelease job keys off it, and dropping it | |
| # made every scheduled run look like a first release. | |
| echo -ne "Update Time: *$(date)*\nDoris Version: *${DORIS_VERSION}*\nStatus: *FAILURE*\nAttempts: *${ATTEMPT}*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md | |
| gh release edit -F release_note.md "${tag_name}" |