ci(github): build third-party images independently per OS version to prevent a single OS build failure from breaking the entire build

.github/workflows/build-push-thirdparty.yml

@@ -0,0 +1,185 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: BuildThirdpartyDocker - build and publish thirdparty
+
+on:
+  workflow_call:
+    inputs:
+      osversion:
+        required: true
+        type: string
+
+jobs:
+  build_push_src_docker_images:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USER }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Build and push
+        uses: docker/[email protected]
+        with:
+          context: .
+          file: ./docker/thirdparties-src/Dockerfile
+          push: true
+          tags: |
+            apache/pegasus:thirdparties-src-${{ inputs.osversion }}-${{ github.ref_name }}
+          build-args: |
+            GITHUB_BRANCH=${{ github.ref_name }}
+            OS_VERSION=${{ inputs.osversion }}
+            HADOOP_BIN_PATH=hadoop-bin
+            ZOOKEEPER_BIN_PATH=zookeeper-bin
+
+  build_push_bin_docker_images:
+    runs-on: ubuntu-latest
+    env:
+      # The glibc version on ubuntu1804 is lower than the node20 required, so
+      # we need to force the node version to 16.
+      # See more details: https://github.com/actions/checkout/issues/1809
+      ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION: node16
+      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
+    needs: build_push_src_docker_images
+    steps:
+      # The glibc version on ubuntu1804 is lower than the actions/checkout@v4 required, so
+      # we need to force to use actions/checkout@v3.
+      - uses: actions/checkout@v3
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USER }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Build and push for production
+        uses: docker/[email protected]
+        with:
+          context: .
+          file: ./docker/thirdparties-bin/Dockerfile
+          push: true
+          tags: |
+            apache/pegasus:thirdparties-bin-${{ inputs.osversion }}-${{ github.ref_name }}
+          build-args: |
+            GITHUB_BRANCH=${{ github.ref_name }}
+            OS_VERSION=${{ inputs.osversion }}
+            HADOOP_BIN_PATH=hadoop-bin
+            ZOOKEEPER_BIN_PATH=zookeeper-bin
+
+  build_push_bin_jemalloc_docker_images:
+    runs-on: ubuntu-latest
+    env:
+      # The glibc version on ubuntu1804 is lower than the node20 required, so
+      # we need to force the node version to 16.
+      # See more details: https://github.com/actions/checkout/issues/1809
+      ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION: node16
+      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
+    needs: build_push_src_docker_images
+    steps:
+      # The glibc version on ubuntu1804 is lower than the actions/checkout@v4 required, so
+      # we need to force to use actions/checkout@v3.
+      - uses: actions/checkout@v3
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USER }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Build and push for production with jemalloc
+        uses: docker/[email protected]
+        with:
+          context: .
+          file: ./docker/thirdparties-bin/Dockerfile
+          push: true
+          tags: |
+            apache/pegasus:thirdparties-bin-jemallc-${{ inputs.osversion }}-${{ github.ref_name }}
+          build-args: |
+            GITHUB_BRANCH=${{ github.ref_name }}
+            OS_VERSION=${{ inputs.osversion }}
+            USE_JEMALLOC=ON
+            HADOOP_BIN_PATH=hadoop-bin
+            ZOOKEEPER_BIN_PATH=zookeeper-bin
+
+  build_push_bin_test_docker_images:
+    runs-on: ubuntu-latest
+    needs: build_push_src_docker_images
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USER }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Build and push for test
+        uses: docker/[email protected]
+        with:
+          context: .
+          file: ./docker/thirdparties-bin/Dockerfile
+          push: true
+          tags: |
+            apache/pegasus:thirdparties-bin-test-${{ inputs.osversion }}-${{ github.ref_name }}
+          build-args: |
+            GITHUB_BRANCH=${{ github.ref_name }}
+            OS_VERSION=${{ inputs.osversion }}
+            ROCKSDB_PORTABLE=1
+            HADOOP_BIN_PATH=hadoop-bin
+            ZOOKEEPER_BIN_PATH=zookeeper-bin
+
+  build_push_bin_test_jemalloc_docker_images:
+    runs-on: ubuntu-latest
+    needs: build_push_src_docker_images
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USER }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Build and push for test with jemalloc
+        uses: docker/[email protected]
+        with:
+          context: .
+          file: ./docker/thirdparties-bin/Dockerfile
+          push: true
+          tags: |
+            apache/pegasus:thirdparties-bin-test-jemallc-${{ inputs.osversion }}-${{ github.ref_name }}
+          build-args: |
+            GITHUB_BRANCH=${{ github.ref_name }}
+            OS_VERSION=${{ inputs.osversion }}
+            ROCKSDB_PORTABLE=1
+            USE_JEMALLOC=ON
+            HADOOP_BIN_PATH=hadoop-bin
+            ZOOKEEPER_BIN_PATH=zookeeper-bin

.github/workflows/thirdparty-regular-push.yml

@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 ---
-name:  BuildThirdpartyDockerRegularly - build and publish thirdparty every week
+name: BuildThirdpartyDockerRegularly - build and publish thirdparty every week
 
 on:
   push:
@@ -40,8 +40,7 @@ on:
     - cron:  '0 18 * * 1'
 
 jobs:
-  build_push_src_docker_images:
-    runs-on: ubuntu-latest
+  build_push_thirdparty_docker_images:
     strategy:
       fail-fast: false
       matrix:
@@ -50,185 +49,7 @@ jobs:
           - ubuntu2004
           - ubuntu2204
           - rockylinux9
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-      - name: Login to DockerHub
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_USER }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Build and push
-        uses: docker/[email protected]
-        with:
-          context: .
-          file: ./docker/thirdparties-src/Dockerfile
-          push: true
-          tags: |
-            apache/pegasus:thirdparties-src-${{ matrix.osversion }}-${{ github.ref_name }}
-          build-args: |
-            GITHUB_BRANCH=${{ github.ref_name }}
-            OS_VERSION=${{ matrix.osversion }}
-            HADOOP_BIN_PATH=hadoop-bin
-            ZOOKEEPER_BIN_PATH=zookeeper-bin
-
-  build_push_bin_docker_images:
-    runs-on: ubuntu-latest
-    env:
-      # The glibc version on ubuntu1804 is lower than the node20 required, so
-      # we need to force the node version to 16.
-      # See more details: https://github.com/actions/checkout/issues/1809
-      ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION: node16
-      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
-    needs: build_push_src_docker_images
-    strategy:
-      fail-fast: false
-      matrix:
-        osversion:
-          - ubuntu1804
-          - ubuntu2004
-          - ubuntu2204
-          - rockylinux9
-    steps:
-      # The glibc version on ubuntu1804 is lower than the actions/checkout@v4 required, so
-      # we need to force to use actions/checkout@v3.
-      - uses: actions/checkout@v3
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-      - name: Login to DockerHub
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_USER }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Build and push for production
-        uses: docker/[email protected]
-        with:
-          context: .
-          file: ./docker/thirdparties-bin/Dockerfile
-          push: true
-          tags: |
-            apache/pegasus:thirdparties-bin-${{ matrix.osversion }}-${{ github.ref_name }}
-          build-args: |
-            GITHUB_BRANCH=${{ github.ref_name }}
-            OS_VERSION=${{ matrix.osversion }}
-            HADOOP_BIN_PATH=hadoop-bin
-            ZOOKEEPER_BIN_PATH=zookeeper-bin
-
-  build_push_bin_jemalloc_docker_images:
-    runs-on: ubuntu-latest
-    env:
-      # The glibc version on ubuntu1804 is lower than the node20 required, so
-      # we need to force the node version to 16.
-      # See more details: https://github.com/actions/checkout/issues/1809
-      ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION: node16
-      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
-    needs: build_push_src_docker_images
-    strategy:
-      fail-fast: false
-      matrix:
-        osversion:
-          - ubuntu1804
-          - ubuntu2004
-          - ubuntu2204
-          - rockylinux9
-    steps:
-      # The glibc version on ubuntu1804 is lower than the actions/checkout@v4 required, so
-      # we need to force to use actions/checkout@v3.
-      - uses: actions/checkout@v3
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-      - name: Login to DockerHub
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_USER }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Build and push for production with jemalloc
-        uses: docker/[email protected]
-        with:
-          context: .
-          file: ./docker/thirdparties-bin/Dockerfile
-          push: true
-          tags: |
-            apache/pegasus:thirdparties-bin-jemallc-${{ matrix.osversion }}-${{ github.ref_name }}
-          build-args: |
-            GITHUB_BRANCH=${{ github.ref_name }}
-            OS_VERSION=${{ matrix.osversion }}
-            USE_JEMALLOC=ON
-            HADOOP_BIN_PATH=hadoop-bin
-            ZOOKEEPER_BIN_PATH=zookeeper-bin
-
-  build_push_bin_test_docker_images:
-    runs-on: ubuntu-latest
-    needs: build_push_src_docker_images
-    strategy:
-      fail-fast: false
-      matrix:
-        osversion:
-          - ubuntu2204
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-      - name: Login to DockerHub
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_USER }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Build and push for test
-        uses: docker/[email protected]
-        with:
-          context: .
-          file: ./docker/thirdparties-bin/Dockerfile
-          push: true
-          tags: |
-            apache/pegasus:thirdparties-bin-test-${{ matrix.osversion }}-${{ github.ref_name }}
-          build-args: |
-            GITHUB_BRANCH=${{ github.ref_name }}
-            OS_VERSION=${{ matrix.osversion }}
-            ROCKSDB_PORTABLE=1
-            HADOOP_BIN_PATH=hadoop-bin
-            ZOOKEEPER_BIN_PATH=zookeeper-bin
-
-  build_push_bin_test_jemalloc_docker_images:
-    runs-on: ubuntu-latest
-    needs: build_push_src_docker_images
-    strategy:
-      fail-fast: false
-      matrix:
-        osversion:
-          - ubuntu2204
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-      - name: Login to DockerHub
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_USER }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Build and push for test with jemalloc
-        uses: docker/[email protected]
-        with:
-          context: .
-          file: ./docker/thirdparties-bin/Dockerfile
-          push: true
-          tags: |
-            apache/pegasus:thirdparties-bin-test-jemallc-${{ matrix.osversion }}-${{ github.ref_name }}
-          build-args: |
-            GITHUB_BRANCH=${{ github.ref_name }}
-            OS_VERSION=${{ matrix.osversion }}
-            ROCKSDB_PORTABLE=1
-            USE_JEMALLOC=ON
-            HADOOP_BIN_PATH=hadoop-bin
-            ZOOKEEPER_BIN_PATH=zookeeper-bin
+    uses: "./.github/workflows/build-push-thirdparty.yml"
+    with:
+      osversion: ${{ matrix.osversion }}
+    secrets: inherit