wip: splitting and caching instant queries for functions over range operators #1811
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and Push mimir-build-image | |
| # configure trigger by pull request | |
| on: | |
| pull_request: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number }} | |
| cancel-in-progress: true | |
| jobs: | |
| build_and_push: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| # Allow pushing to the GitHub repo for collaborators, forks should remain read-only | |
| contents: write | |
| # Allow PR modification for collaborators, forks should remain read-only | |
| pull-requests: write | |
| # Necessary to authenticate with Vault which happens in dockerhub-login | |
| id-token: write | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6 | |
| with: | |
| persist-credentials: false | |
| # We only want for this workflow to do anything when mimir-build-image/Dockerfile is modified. | |
| # However, we want for it to be able to run for all PRs, just so it can be made a required check | |
| # (i.e. so PRs can't be merged until it's done). | |
| - name: Check if mimir-build-image/Dockerfile was modified | |
| id: check_dockerfile | |
| run: | | |
| if gh pr diff ${{ github.event.pull_request.number }} --name-only | grep -q '^mimir-build-image/Dockerfile$'; then | |
| echo "modified=true" >> $GITHUB_OUTPUT | |
| echo "Dockerfile was modified" | |
| else | |
| echo "modified=false" >> $GITHUB_OUTPUT | |
| echo "Dockerfile was not modified" | |
| fi | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| # Retrieve GitHub App credentials to get a token with org member read permissions. | |
| # GITHUB_TOKEN doesn't have org-level permissions, so we use the mimir-github-bot GitHub App. | |
| - name: Retrieve GitHub App Credentials from Vault | |
| if: steps.check_dockerfile.outputs.modified == 'true' | |
| id: get-secrets | |
| uses: grafana/shared-workflows/actions/get-vault-secrets@28361cdb22223e5f1e34358c86c20908e7248760 # v1.1.0 | |
| with: | |
| repo_secrets: | | |
| APP_ID=mimir-github-bot:app_id | |
| PRIVATE_KEY=mimir-github-bot:private_key | |
| - name: Generate GitHub App Token for Membership | |
| if: steps.check_dockerfile.outputs.modified == 'true' | |
| id: app-token-for-membership | |
| uses: actions/create-github-app-token@29824e69f54612133e76f7eaac726eef6c875baf # v2 | |
| with: | |
| app-id: ${{ env.APP_ID }} | |
| private-key: ${{ env.PRIVATE_KEY }} | |
| owner: ${{ github.repository_owner }} | |
| # Check if PR author is a Grafana organization member using the GitHub API. | |
| # This is more reliable than author_association which depends on public membership visibility. | |
| # Uses the GitHub App token because GITHUB_TOKEN doesn't have org-level permissions. | |
| - name: Check if PR author is org member | |
| if: steps.check_dockerfile.outputs.modified == 'true' | |
| id: check_membership | |
| run: | | |
| # Check membership, capturing the HTTP status code | |
| status_code=$(gh api \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "X-GitHub-Api-Version: 2022-11-28" \ | |
| "/orgs/grafana/members/${PR_AUTHOR}" \ | |
| --include 2>&1 | head -1 | awk '{print $2}') | |
| if [ "$status_code" = "204" ]; then | |
| echo "is_member=true" >> $GITHUB_OUTPUT | |
| echo "✓ User ${PR_AUTHOR} is a Grafana organization member" | |
| else | |
| echo "is_member=false" >> $GITHUB_OUTPUT | |
| echo "✗ User ${PR_AUTHOR} is not a Grafana organization member (HTTP $status_code)" | |
| fi | |
| env: | |
| GITHUB_TOKEN: ${{ steps.app-token-for-membership.outputs.token }} | |
| PR_AUTHOR: ${{ github.event.pull_request.user.login }} | |
| # We want to allow running github actions for all contributors, but don't want all contributors to be able to | |
| # publish new build images just by sending the PR. Only Grafana org members and Renovate can proceed. | |
| # This step sets an 'authorized' output that all subsequent privileged steps depend on. | |
| - name: Check authorization | |
| if: steps.check_dockerfile.outputs.modified == 'true' | |
| id: authorize | |
| run: | | |
| if [ "${{ steps.check_membership.outputs.is_member }}" = "true" ] || [ "${PR_AUTHOR}" = "renovate-sh-app[bot]" ]; then | |
| echo "authorized=true" >> $GITHUB_OUTPUT | |
| echo "✓ User ${PR_AUTHOR} is authorized to build images" | |
| else | |
| echo "authorized=false" >> $GITHUB_OUTPUT | |
| # We want to fail the workflow here because we only get here if the build image Dockerfile | |
| # has changed, and such a PR shouldn't pass without a successful build and push. | |
| echo "::error::Only Grafana organization members and Renovate can trigger build image updates" | |
| exit 1 | |
| fi | |
| env: | |
| PR_AUTHOR: ${{ github.event.pull_request.user.login }} | |
| - name: Checkout Pull Request Branch | |
| if: steps.check_dockerfile.outputs.modified == 'true' && steps.authorize.outputs.authorized == 'true' | |
| run: gh pr checkout ${{ github.event.pull_request.number }} | |
| env: | |
| GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} | |
| - name: Setup QEMU | |
| if: steps.check_dockerfile.outputs.modified == 'true' && steps.authorize.outputs.authorized == 'true' | |
| uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3.7.0 | |
| - name: Set up Docker Buildx | |
| if: steps.check_dockerfile.outputs.modified == 'true' && steps.authorize.outputs.authorized == 'true' | |
| uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 | |
| - name: Login to Docker Hub | |
| if: steps.check_dockerfile.outputs.modified == 'true' && steps.authorize.outputs.authorized == 'true' | |
| uses: grafana/shared-workflows/actions/dockerhub-login@13fb504e3bfe323c1188bf244970d94b2d336e86 # v1.0.1 | |
| - name: Prepare Variables | |
| if: steps.check_dockerfile.outputs.modified == 'true' && steps.authorize.outputs.authorized == 'true' | |
| id: prepare | |
| run: | | |
| echo "path=mimir-build-image/Dockerfile" >> $GITHUB_OUTPUT | |
| main_build_image=$(make print-build-image) | |
| main_image_tag=$(echo $main_build_image | cut -d ':' -f 2) | |
| image_name=$(echo $main_build_image | cut -d ':' -f 1) | |
| echo "image=$image_name" >> $GITHUB_OUTPUT | |
| echo "main_image_tag=$main_image_tag" >> $GITHUB_OUTPUT | |
| - name: Compute Image Tag | |
| if: steps.check_dockerfile.outputs.modified == 'true' && steps.authorize.outputs.authorized == 'true' | |
| id: compute_hash | |
| run: | | |
| current_hash=$(md5sum ${{ steps.prepare.outputs.path }} | awk '{print substr($1, 0, 10)}') | |
| echo "the file path is ${{ steps.prepare.outputs.path }}" | |
| echo "build tag is $current_hash" | |
| tag="pr${{ github.event.pull_request.number }}-$current_hash" | |
| echo "tag=$tag" >> $GITHUB_OUTPUT | |
| - name: Check Should Build Image | |
| if: steps.check_dockerfile.outputs.modified == 'true' && steps.authorize.outputs.authorized == 'true' | |
| id: check_build | |
| run: | | |
| echo "Checking if image should be built" | |
| if skopeo inspect --raw "docker://${{ steps.prepare.outputs.image }}:${{ steps.compute_hash.outputs.tag }}" >/dev/null 2>&1; then | |
| echo "build=false" >> $GITHUB_OUTPUT | |
| echo "Tag ${{ steps.compute_hash.outputs.tag }} exists" | |
| else | |
| echo "Tag ${{ steps.compute_hash.outputs.tag }} does not exist" | |
| echo "build=true" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Add Comment to the PR | |
| if: steps.check_dockerfile.outputs.modified == 'true' && steps.authorize.outputs.authorized == 'true' | |
| id: notification | |
| run: | | |
| if [ ${{ steps.check_build.outputs.build }} == 'true' ]; then | |
| gh pr comment $PR_NUMBER --body "**Building new version of mimir-build-image**. After image is built and pushed to Docker Hub, \ | |
| a new commit will automatically be added to this PR with new image version \`$IMAGE:$TAG\`. This can take up to 1 hour." | |
| else | |
| echo "This PR will not trigger a build of mimir-build-image" | |
| gh pr comment $PR_NUMBER --body "**Not building new version of mimir-build-image**. This PR modifies \`mimir-build-image/Dockerfile\`, but the image \`$IMAGE:$TAG\` already exists." | |
| fi | |
| env: | |
| PR_NUMBER: ${{ github.event.pull_request.number }} | |
| GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} | |
| TAG: ${{ steps.compute_hash.outputs.tag }} | |
| IMAGE: ${{ steps.prepare.outputs.image }} | |
| - name: Build and Push Docker Image | |
| if: steps.check_dockerfile.outputs.modified == 'true' && steps.authorize.outputs.authorized == 'true' && steps.check_build.outputs.build == 'true' | |
| run: | | |
| echo "Building and Pushing Docker Image" | |
| make push-multiarch-build-image IMAGE_TAG=${{ steps.compute_hash.outputs.tag }} | |
| - name: Compare built tag with Makefile | |
| if: steps.check_dockerfile.outputs.modified == 'true' && steps.authorize.outputs.authorized == 'true' | |
| id: compare_tag | |
| run: | | |
| echo "Comparing built tag with Makefile" | |
| if [ ${{ steps.compute_hash.outputs.tag }} == "$(make print-build-image)" ]; then | |
| echo "Built tag is the same as the one in Makefile" | |
| echo "isDifferent=false" >> $GITHUB_OUTPUT | |
| else | |
| echo "Built tag is different from the one in Makefile" | |
| echo "isDifferent=true" >> $GITHUB_OUTPUT | |
| fi | |
| # Tokens expire after 1h. Since building the image can take a long time, we create a fresh App Token for the last | |
| # step of the build. (https://github.com/actions/create-github-app-token/issues/121#issuecomment-2027574184) | |
| - name: Generate GitHub App Token | |
| if: steps.check_dockerfile.outputs.modified == 'true' | |
| id: app-token-for-commit | |
| uses: actions/create-github-app-token@29824e69f54612133e76f7eaac726eef6c875baf # v2 | |
| with: | |
| app-id: ${{ env.APP_ID }} | |
| private-key: ${{ env.PRIVATE_KEY }} | |
| owner: ${{ github.repository_owner }} | |
| # This step uses "Mimir bot" token (generated at the start of the workflow) instead of | |
| # "github-actions bot" (secrets.GITHUB_TOKEN) because any events triggered by the latter | |
| # don't spawn GitHub actions. | |
| # Refer to https://docs.github.com/en/actions/security-for-github-actions/security-guides/automatic-token-authentication#using-the-github_token-in-a-workflow | |
| - name: Add commit to PR in order to update Build Image version | |
| if: steps.check_dockerfile.outputs.modified == 'true' && steps.authorize.outputs.authorized == 'true' && steps.compare_tag.outputs.isDifferent == 'true' | |
| run: | | |
| echo "Get current Build Image Version" | |
| echo "Current Build Image Version is $MAIN_TAG" | |
| echo "Built Image Version is $TAG" | |
| if [ "$MAIN_TAG" = "$TAG" ]; then | |
| echo "Build Image Version is already up to date" | |
| else | |
| echo "Build Image Version is not up to date" | |
| sed -i "s/$MAIN_TAG/${{ steps.compute_hash.outputs.tag }}/g" Makefile | |
| git config --global user.email "${GITHUB_LOGIN}@users.noreply.github.com" | |
| git config --global user.name "${GITHUB_LOGIN}" | |
| git config --global url.https://x-access-token:${GITHUB_TOKEN}@github.com/.insteadOf https://github.com/ | |
| git add Makefile | |
| git commit -m "Update build image version to ${{ steps.compute_hash.outputs.tag }}" | |
| git push origin HEAD | |
| fi | |
| env: | |
| GITHUB_TOKEN: ${{ steps.app-token-for-commit.outputs.token }} | |
| GITHUB_LOGIN: ${{ github.event.pull_request.user.login }} | |
| TAG: ${{ steps.compute_hash.outputs.tag }} | |
| MAIN_TAG: ${{ steps.prepare.outputs.main_image_tag }} |