make errors from stalled stream protection retryable (#6501) #2963
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and publish Docker images | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - main | |
| - release-0.9 | |
| paths: | |
| - "quickwit/**" | |
| tags: | |
| - airmail | |
| - happy-plazza | |
| - qw* | |
| - v* | |
| permissions: | |
| contents: read | |
| env: | |
| REGISTRY_IMAGE: quickwit/quickwit | |
| jobs: | |
| build-lambda: | |
| name: Build Lambda ARM64 | |
| # Only build a fresh lambda for tag-triggered builds (v*, qw*, etc.). | |
| # Branch/edge builds and workflow_dispatch fall back to the pinned URL in build.rs. | |
| if: github.ref_type == 'tag' | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write | |
| actions: write | |
| outputs: | |
| artifact-name: ${{ steps.set-artifact-name.outputs.name }} | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: Extract asset version | |
| run: echo "ASSET_VERSION=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_ENV | |
| - name: Install rustup | |
| run: curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain none -y | |
| - name: Install cross | |
| run: cargo install cross | |
| - name: Retrieve and export commit date, hash, and tags | |
| run: | | |
| echo "QW_COMMIT_DATE=$(TZ=UTC0 git log -1 --format=%cd --date=format-local:%Y-%m-%dT%H:%M:%SZ)" >> $GITHUB_ENV | |
| echo "QW_COMMIT_HASH=$(git rev-parse HEAD)" >> $GITHUB_ENV | |
| echo "QW_COMMIT_TAGS=$(git tag --points-at HEAD | tr '\n' ',')" >> $GITHUB_ENV | |
| - name: Build Lambda binary | |
| run: cross build --release --features lambda-release --target aarch64-unknown-linux-gnu -p quickwit-lambda-server --bin quickwit-aws-lambda-leaf-search | |
| env: | |
| QW_COMMIT_DATE: ${{ env.QW_COMMIT_DATE }} | |
| QW_COMMIT_HASH: ${{ env.QW_COMMIT_HASH }} | |
| QW_COMMIT_TAGS: ${{ env.QW_COMMIT_TAGS }} | |
| working-directory: ./quickwit | |
| - name: Create Lambda zip | |
| run: | | |
| cd quickwit/target/aarch64-unknown-linux-gnu/release | |
| cp quickwit-aws-lambda-leaf-search bootstrap | |
| zip quickwit-aws-lambda-${{ env.ASSET_VERSION }}-aarch64.zip bootstrap | |
| mv quickwit-aws-lambda-${{ env.ASSET_VERSION }}-aarch64.zip ../../../../ | |
| - name: Set artifact name | |
| id: set-artifact-name | |
| run: echo "name=lambda-zip-${{ env.ASSET_VERSION }}" >> $GITHUB_OUTPUT | |
| - name: Upload Lambda zip as workflow artifact | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: ${{ steps.set-artifact-name.outputs.name }} | |
| path: quickwit-aws-lambda-${{ env.ASSET_VERSION }}-aarch64.zip | |
| retention-days: 3 | |
| docker: | |
| needs: [build-lambda] | |
| if: always() && (needs.build-lambda.result == 'success' || needs.build-lambda.result == 'skipped') | |
| strategy: | |
| matrix: | |
| include: | |
| - os: ubuntu-latest | |
| platform: linux/amd64 | |
| platform_suffix: amd64 | |
| - os: gh-ubuntu-arm64 | |
| platform: linux/arm64 | |
| platform_suffix: arm64 | |
| runs-on: ${{ matrix.os }} | |
| permissions: | |
| contents: read | |
| actions: write | |
| environment: | |
| name: production | |
| steps: | |
| - name: Cleanup Disk Space | |
| run: | | |
| df -h | |
| sudo rm -rf /opt/hostedtoolcache/CodeQL | |
| sudo rm -rf /usr/local/.ghcup | |
| sudo rm -rf /usr/local/lib/android | |
| sudo rm -rf /usr/share/dotnet | |
| sudo rm -rf /usr/share/swift | |
| df -h | |
| - name: Checkout | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 | |
| - name: Docker meta | |
| id: meta | |
| uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0 | |
| with: | |
| images: | | |
| ${{ env.REGISTRY_IMAGE }} | |
| labels: | | |
| org.opencontainers.image.title=Quickwit | |
| maintainer=Quickwit, Inc. <hello@quickwit.io> | |
| org.opencontainers.image.vendor=Quickwit, Inc. | |
| org.opencontainers.image.licenses=Apache-2.0 | |
| - name: Retrieve commit date, hash, and tags | |
| run: | | |
| echo "QW_COMMIT_DATE=$(TZ=UTC0 git log -1 --format=%cd --date=format-local:%Y-%m-%dT%H:%M:%SZ)" >> $GITHUB_ENV | |
| echo "QW_COMMIT_HASH=$(git rev-parse HEAD)" >> $GITHUB_ENV | |
| echo "QW_COMMIT_TAGS=$(git tag --points-at HEAD | tr '\n' ',')" >> $GITHUB_ENV | |
| if [[ "${{ github.event_name }}" == "push" && "${{ github.ref_type }}" == "tag" && "${GITHUB_REF#refs/tags/}" == *"jemprof"* ]]; then | |
| echo "CARGO_FEATURES=release-jemalloc-profiled" >> $GITHUB_ENV | |
| else | |
| echo "CARGO_FEATURES=release-feature-set" >> $GITHUB_ENV | |
| fi | |
| - name: Download Lambda artifact | |
| if: needs.build-lambda.result == 'success' | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: ${{ needs.build-lambda.outputs.artifact-name }} | |
| # Place inside quickwit/ so the Dockerfile's "COPY quickwit /quickwit" picks it up. | |
| path: quickwit/lambda-artifact | |
| - name: Set LAMBDA_ZIP_PATH build arg | |
| if: needs.build-lambda.result == 'success' | |
| run: | | |
| ZIP=$(ls quickwit/lambda-artifact/*.zip | xargs basename) | |
| echo "LAMBDA_ZIP_PATH=/quickwit/lambda-artifact/$ZIP" >> $GITHUB_ENV | |
| - name: Build and push image | |
| uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 | |
| id: build | |
| with: | |
| context: . | |
| platforms: ${{ matrix.platform }} | |
| build-args: | | |
| QW_COMMIT_DATE=${{ env.QW_COMMIT_DATE }} | |
| QW_COMMIT_HASH=${{ env.QW_COMMIT_HASH }} | |
| QW_COMMIT_TAGS=${{ env.QW_COMMIT_TAGS }} | |
| CARGO_FEATURES=${{ env.CARGO_FEATURES }} | |
| LAMBDA_ZIP_PATH=${{ env.LAMBDA_ZIP_PATH }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true | |
| - name: Export digest | |
| run: | | |
| mkdir -p /tmp/digests | |
| digest="${{ steps.build.outputs.digest }}" | |
| touch "/tmp/digests/${digest#sha256:}" | |
| - name: Upload digest | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: digest-${{ matrix.platform_suffix }} | |
| path: /tmp/digests/* | |
| if-no-files-found: error | |
| retention-days: 1 | |
| merge: | |
| runs-on: ubuntu-latest | |
| needs: [docker] | |
| permissions: | |
| contents: read | |
| actions: read | |
| environment: production | |
| steps: | |
| - name: Download digests | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| pattern: digest-* | |
| path: /tmp/digests | |
| merge-multiple: true | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 | |
| - name: Docker meta | |
| id: meta | |
| uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0 | |
| with: | |
| images: ${{ env.REGISTRY_IMAGE }} | |
| flavor: | | |
| latest=false | |
| tags: | | |
| type=edge,branch=main | |
| type=edge,branch=main,suffix=-slim-bookworm | |
| type=semver,pattern={{version}} | |
| type=semver,pattern={{version}},value=latest | |
| type=semver,pattern={{version}},suffix=-slim-bookworm | |
| type=ref,event=tag | |
| type=raw,value=v0.9.0-rc,enable=${{ github.ref == 'refs/heads/release-0.9' }} | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} | |
| - name: Create manifest list and push tags | |
| working-directory: /tmp/digests | |
| run: | | |
| docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ | |
| $(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *) | |
| - name: Inspect image | |
| run: | | |
| docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ steps.meta.outputs.version }} |