diff --git a/.github/actions/build-bioconda-utils-container/action.yml b/.github/actions/build-bioconda-utils-container/action.yml new file mode 100644 index 00000000000..4e1428ff570 --- /dev/null +++ b/.github/actions/build-bioconda-utils-container/action.yml @@ -0,0 +1,106 @@ +name: Build bioconda-utils container +description: Build and test the multi-architecture bioconda-utils build environment. + +inputs: + tags: + description: Whitespace-separated tags to apply to the local manifest. + required: true + +outputs: + image: + description: Local multi-architecture manifest name. + value: ${{ steps.metadata.outputs.image }} + tags: + description: Whitespace-separated tags applied to the local manifest. + value: ${{ steps.metadata.outputs.tags }} + +runs: + using: composite + steps: + - name: Prepare metadata + id: metadata + shell: bash + env: + TAGS: ${{ inputs.tags }} + run: | + set -euo pipefail + + tags="$( xargs <<< "${TAGS}" )" + test -n "${tags}" + printf '%s\n' \ + 'image=bioconda-utils-build-env-cos7' \ + "tags=${tags}" \ + >> "${GITHUB_OUTPUT}" + + - name: Set up QEMU + shell: bash + run: | + set -euo pipefail + podman run --rm --privileged \ + docker.io/tonistiigi/binfmt --install arm64 + + - name: Install tools + shell: bash + run: | + set -euo pipefail + # jq is not installed in travier/podman-action. + dnf install -qy jq + + - name: Build amd64 image + uses: redhat-actions/buildah-build@v3 + with: + image: bioconda-utils-build-env-cos7-amd64 + arch: amd64 + build-args: | + BASE_IMAGE=quay.io/condaforge/linux-anvil-cos7-x86_64 + tags: ${{ steps.metadata.outputs.tags }} + containerfiles: ./Dockerfile + + - name: Build arm64 image + uses: redhat-actions/buildah-build@v3 + with: + image: bioconda-utils-build-env-cos7-arm64 + arch: arm64 + build-args: | + BASE_IMAGE=quay.io/condaforge/linux-anvil-aarch64 + tags: ${{ steps.metadata.outputs.tags }} + containerfiles: ./Dockerfile + + - name: Assemble multi-architecture manifest + shell: bash + env: + IMAGE: ${{ steps.metadata.outputs.image }} + TAGS: ${{ steps.metadata.outputs.tags }} + run: | + set -euo pipefail + + read -r -a tags <<< "${TAGS}" + for tag in "${tags[@]}" ; do + buildah manifest create "${IMAGE}:${tag}" + buildah manifest add \ + "${IMAGE}:${tag}" \ + "${IMAGE}-amd64:${tag}" + buildah manifest add \ + "${IMAGE}:${tag}" \ + "${IMAGE}-arm64:${tag}" + done + + - name: Test + shell: bash + env: + LOCAL_IMAGE: ${{ steps.metadata.outputs.image }} + LOCAL_TAGS: ${{ steps.metadata.outputs.tags }} + run: | + set -euo pipefail + + read -r -a tags <<< "${LOCAL_TAGS}" + for tag in "${tags[@]}" ; do + for architecture in amd64 arm64 ; do + podman run --rm --pull=never \ + --arch="${architecture}" \ + "${LOCAL_IMAGE}:${tag}" \ + bioconda-utils --version + done + done + + .github/scripts/test-local-container Dockerfile.test . diff --git a/.github/actions/publish-container/action.yml b/.github/actions/publish-container/action.yml new file mode 100644 index 00000000000..51bba7223ce --- /dev/null +++ b/.github/actions/publish-container/action.yml @@ -0,0 +1,129 @@ +name: Publish container +description: Protect immutable tags, push a local image, and verify the remote platforms. + +inputs: + image: + description: Local image or manifest name. + required: true + tags: + description: Whitespace-separated tags to publish. + required: true + mutable-tags: + description: Whitespace-separated tags that may be overwritten. + required: false + default: '' + registry: + description: Registry hostname and namespace. + required: true + username: + description: Registry username. + required: true + password: + description: Registry password or token. + required: true + +outputs: + test-image: + description: Immutable remote manifest reference to use for functional tests. + value: ${{ steps.verify.outputs.test-image }} + +runs: + using: composite + steps: + - name: Check immutable tags + shell: bash + env: + IMAGE: ${{ inputs.image }} + MUTABLE_TAGS: ${{ inputs.mutable-tags }} + REGISTRY: ${{ inputs.registry }} + TAGS: ${{ inputs.tags }} + run: | + set -euo pipefail + + existing_tags="$( + skopeo list-tags "docker://${REGISTRY}/${IMAGE}" \ + | jq -r '.Tags[]' + )" \ + || { + echo 'Could not list tags via skopeo.' + exit 1 + } + + for tag in ${TAGS} ; do + case " ${MUTABLE_TAGS} " in + *" ${tag} "* ) continue ;; + esac + if printf '%s\n' "${existing_tags}" | grep -qxF "${tag}" ; then + printf 'Tag %s already exists!\n' "${tag}" + exit 1 + fi + done + + - name: Push + id: push + uses: redhat-actions/push-to-registry@v3 + with: + image: ${{ inputs.image }} + tags: ${{ inputs.tags }} + registry: ${{ inputs.registry }} + username: ${{ inputs.username }} + password: ${{ inputs.password }} + + - name: Verify remote architectures + id: verify + shell: bash + env: + REGISTRY_PATHS: ${{ steps.push.outputs.registry-paths }} + run: | + set -euo pipefail + + jq -e 'type == "array" and length > 0' > /dev/null \ + <<< "${REGISTRY_PATHS}" + expected='amd64 arm64' + expected_digest='' + expected_repository='' + + while read -r image ; do + raw_manifest="$( skopeo inspect --raw "docker://${image}" )" + if jq -e 'has("manifests")' > /dev/null <<< "${raw_manifest}" ; then + actual="$( + jq -r \ + '.manifests[] | select(.platform.os == "linux") | .platform.architecture' \ + <<< "${raw_manifest}" \ + | sort -u \ + | xargs + )" + else + actual="$( + skopeo inspect "docker://${image}" \ + | jq -r '.Architecture' + )" + fi + + if [[ "${actual}" != "${expected}" ]] ; then + printf 'Unexpected architectures for %s: expected "%s", found "%s".\n' \ + "${image}" "${expected}" "${actual}" + exit 1 + fi + printf 'Verified remote architectures for %s: %s\n' "${image}" "${actual}" + + repository="${image%:*}" + digest="$( + skopeo inspect \ + --format '{{.Digest}}' \ + "docker://${image}" + )" + if [[ -z "${expected_digest}" ]] ; then + expected_digest="${digest}" + expected_repository="${repository}" + elif [[ "${repository}" != "${expected_repository}" \ + || "${digest}" != "${expected_digest}" ]] ; then + printf 'Pushed tag %s does not reference %s@%s.\n' \ + "${image}" "${expected_repository}" "${expected_digest}" + exit 1 + fi + done < <( jq -er '.[]' <<< "${REGISTRY_PATHS}" ) + + printf 'test-image=%s@%s\n' \ + "${expected_repository}" "${expected_digest}" \ + >> "${GITHUB_OUTPUT}" diff --git a/.github/scripts/test-local-container b/.github/scripts/test-local-container new file mode 100755 index 00000000000..04c5b9859df --- /dev/null +++ b/.github/scripts/test-local-container @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +# Functionally test both architectures of a local multi-architecture manifest. + +set -euo pipefail + +cleanup() { + buildah rmi --prune || true +} +trap cleanup EXIT + +if [[ "$#" -ne 2 ]]; then + echo 'Usage: test-local-container TEST_CONTAINERFILE CONTEXT' >&2 + exit 2 +fi +: "${LOCAL_IMAGE:?Set LOCAL_IMAGE to the local manifest name.}" +: "${LOCAL_TAGS:?Set LOCAL_TAGS to its whitespace-separated tags.}" + +test_containerfile="$1" +context="$2" +test -f "${test_containerfile}" +test -d "${context}" + +tags="$(xargs <<<"${LOCAL_TAGS}")" +read -r -a tag_list <<<"${tags}" +test "${#tag_list[@]}" -gt 0 + +for tag in "${tag_list[@]}"; do + actual="$( + buildah manifest inspect "${LOCAL_IMAGE}:${tag}" | + jq -r \ + '.manifests[] | select(.platform.os == "linux") | .platform.architecture' | + sort -u | + xargs + )" + if [[ "${actual}" != 'amd64 arm64' ]]; then + printf 'Unexpected architectures for %s:%s: expected "amd64 arm64", found "%s".\n' \ + "${LOCAL_IMAGE}" "${tag}" "${actual}" >&2 + exit 1 + fi +done + +base="${LOCAL_IMAGE}:${tag_list[0]}" +for architecture in amd64 arm64; do + printf 'Testing local %s image %s\n' "${architecture}" "${base}" + # The manifest under test is known to exist locally, but the test + # Containerfile may use additional base images in later stages. + buildah build \ + --pull=missing \ + --arch="${architecture}" \ + --build-arg=base="${base}" \ + --file="${test_containerfile}" \ + "${context}" +done diff --git a/.github/scripts/test-pushed-container b/.github/scripts/test-pushed-container new file mode 100755 index 00000000000..39850d8bcc9 --- /dev/null +++ b/.github/scripts/test-pushed-container @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +# Functionally test both architectures of an immutable remote manifest. + +set -euo pipefail + +cleanup() { + buildah rmi --prune || true +} +trap cleanup EXIT + +if [[ "$#" -ne 2 ]]; then + echo 'Usage: test-pushed-container TEST_CONTAINERFILE CONTEXT' >&2 + exit 2 +fi +: "${TEST_IMAGE:?Set TEST_IMAGE to the publish action test-image output.}" + +test_containerfile="$1" +context="$2" + +for architecture in amd64 arm64; do + printf 'Testing pushed %s image %s\n' "${architecture}" "${TEST_IMAGE}" + buildah build \ + --pull=always \ + --arch="${architecture}" \ + --build-arg=base="${TEST_IMAGE}" \ + --file="${test_containerfile}" \ + "${context}" +done diff --git a/.github/workflows/GithubActionTests.yml b/.github/workflows/GithubActionTests.yml index be1292b11b0..59cb13f1390 100644 --- a/.github/workflows/GithubActionTests.yml +++ b/.github/workflows/GithubActionTests.yml @@ -8,7 +8,7 @@ jobs: qc: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - uses: astral-sh/ruff-action@v4.0.0 name: ruff check - uses: astral-sh/ruff-action@v4.0.0 @@ -37,7 +37,7 @@ jobs: - long_running_2 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: fetch-depth: 0 @@ -71,7 +71,7 @@ jobs: name: OSX tests runs-on: macos-15-intel steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: fetch-depth: 0 @@ -99,7 +99,7 @@ jobs: name: autobump test runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - uses: prefix-dev/setup-pixi@v0 with: diff --git a/.github/workflows/base-glibc-busybox-bash.yaml b/.github/workflows/base-glibc-busybox-bash.yaml index d93aa5e015e..bbb56c2b053 100644 --- a/.github/workflows/base-glibc-busybox-bash.yaml +++ b/.github/workflows/base-glibc-busybox-bash.yaml @@ -29,7 +29,7 @@ jobs: DEBIAN_VERSION: '12.5' steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Set up QEMU run: | @@ -68,10 +68,11 @@ jobs: ${{ env.MAJOR_VERSION }}.${{ env.MINOR_VERSION }} latest ' + tags="$( printf '%s\n' "${tags}" | xargs )" printf %s\\n \ "image=${image_name}" \ - "tags=$( echo ${tags} )" \ - >> $GITHUB_OUTPUT + "tags=${tags}" \ + >> "${GITHUB_OUTPUT}" for tag in ${tags} ; do buildah manifest create "${image_name}:${tag}" @@ -126,82 +127,34 @@ jobs: done - name: Test + shell: bash + env: + LOCAL_IMAGE: ${{ steps.build.outputs.image }} + LOCAL_TAGS: ${{ steps.build.outputs.tags }} run: | - image='${{ steps.build.outputs.image }}' - ids="$( - for tag in ${{ steps.build.outputs.tags }} ; do - buildah manifest inspect "${image}:${tag}" \ - | jq -r '.manifests[]|.digest' \ - | while read id ; do - buildah images --format '{{.ID}}{{.Digest}}' \ - | sed -n "s/${id}//p" - done - done - )" - ids="$( printf %s "${ids}" | sort -u )" - for id in ${ids} ; do - podman history "${id}" - buildah bud \ - --build-arg=base="${id}" \ - --file=Dockerfile.test \ - "images/${image}" - done - buildah rmi --prune || true - - - if: ${{ github.ref == 'refs/heads/master' }} - name: Check Tags - run: | - # Quay.io does not support immutable images. - # => Check for duplicate tags to avoid overwriting existing images. - existing_tags="$( - skopeo list-tags docker://quay.io/bioconda/${{ steps.build.outputs.image }} \ - | jq -r '.Tags[]' - )" \ - || { - echo 'Could not list tags via skopeo.' - exit 1 - } - for tag in ${{ steps.build.outputs.tags }} ; do - case "${tag}" in - latest | '${{ env.MAJOR_VERSION }}' ) ;; - * ) - if printf %s "${existing_tags}" | grep -qxF "${tag}" ; then - printf 'Tag %s already exists!\n' "${tag}" - exit 1 - fi - esac - done + .github/scripts/test-local-container \ + 'images/${{ env.IMAGE_NAME }}/Dockerfile.test' \ + 'images/${{ env.IMAGE_NAME }}' - if: ${{ github.ref == 'refs/heads/master' }} - name: Push - uses: redhat-actions/push-to-registry@v2 + id: publish + name: Publish + uses: ./.github/actions/publish-container with: image: ${{ steps.build.outputs.image }} tags: ${{ steps.build.outputs.tags }} - registry: ${{ secrets.QUAY_BIOCONDA_REPO }} + mutable-tags: >- + latest + ${{ env.MAJOR_VERSION }} + registry: ${{ vars.QUAY_BIOCONDA_REPO }} username: ${{ secrets.QUAY_BIOCONDA_USERNAME }} password: ${{ secrets.QUAY_BIOCONDA_TOKEN }} - if: ${{ github.ref == 'refs/heads/master' }} name: Test Pushed + env: + TEST_IMAGE: ${{ steps.publish.outputs.test-image }} run: | - image='${{ env.IMAGE_NAME }}' - ids="$( - for tag in ${{ steps.build.outputs.tags }} ; do - buildah manifest inspect "${image}:${tag}" \ - | jq -r '.manifests[]|.digest' \ - | while read id ; do - buildah images --format '{{.ID}}{{.Digest}}' \ - | sed -n "s/${id}//p" - done - done - )" - ids="$( printf %s "${ids}" | sort -u )" - for id in ${ids} ; do - podman history "${id}" - buildah bud \ - --build-arg=base="${id}" \ - --file=Dockerfile.test \ - "images/${image}" - done - buildah rmi --prune || true + .github/scripts/test-pushed-container \ + 'images/${{ env.IMAGE_NAME }}/Dockerfile.test' \ + 'images/${{ env.IMAGE_NAME }}' diff --git a/.github/workflows/base-glibc-debian-bash.yaml b/.github/workflows/base-glibc-debian-bash.yaml index 38a3b4b27ff..e1b18a67e20 100644 --- a/.github/workflows/base-glibc-debian-bash.yaml +++ b/.github/workflows/base-glibc-debian-bash.yaml @@ -28,7 +28,7 @@ jobs: DEBIAN_VERSION: '12.5' steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Set up QEMU run: | @@ -67,10 +67,11 @@ jobs: ${{ env.MAJOR_VERSION }}.${{ env.MINOR_VERSION }} latest ' + tags="$( printf '%s\n' "${tags}" | xargs )" printf %s\\n \ "image=${image_name}" \ - "tags=$( echo ${tags} )" \ - >> $GITHUB_OUTPUT + "tags=${tags}" \ + >> "${GITHUB_OUTPUT}" for tag in ${tags} ; do buildah manifest create "${image_name}:${tag}" @@ -116,82 +117,34 @@ jobs: done - name: Test + shell: bash + env: + LOCAL_IMAGE: ${{ steps.build.outputs.image }} + LOCAL_TAGS: ${{ steps.build.outputs.tags }} run: | - image='${{ steps.build.outputs.image }}' - ids="$( - for tag in ${{ steps.build.outputs.tags }} ; do - buildah manifest inspect "${image}:${tag}" \ - | jq -r '.manifests[]|.digest' \ - | while read id ; do - buildah images --format '{{.ID}}{{.Digest}}' \ - | sed -n "s/${id}//p" - done - done - )" - ids="$( printf %s "${ids}" | sort -u )" - for id in ${ids} ; do - podman history "${id}" - buildah bud \ - --build-arg=base="${id}" \ - --file=Dockerfile.test \ - "images/${image}" - done - buildah rmi --prune || true - - - if: ${{ github.ref == 'refs/heads/master' }} - name: Check Tags - run: | - # Quay.io does not support immutable images. - # => Check for duplicate tags to avoid overwriting existing images. - existing_tags="$( - skopeo list-tags docker://quay.io/bioconda/${{ steps.build.outputs.image }} \ - | jq -r '.Tags[]' - )" \ - || { - echo 'Could not list tags via skopeo.' - exit 1 - } - for tag in ${{ steps.build.outputs.tags }} ; do - case "${tag}" in - latest | '${{ env.MAJOR_VERSION }}' ) ;; - * ) - if printf %s "${existing_tags}" | grep -qxF "${tag}" ; then - printf 'Tag %s already exists!\n' "${tag}" - exit 1 - fi - esac - done + .github/scripts/test-local-container \ + 'images/${{ env.IMAGE_NAME }}/Dockerfile.test' \ + 'images/${{ env.IMAGE_NAME }}' - if: ${{ github.ref == 'refs/heads/master' }} - name: Push - uses: redhat-actions/push-to-registry@v2 + id: publish + name: Publish + uses: ./.github/actions/publish-container with: image: ${{ steps.build.outputs.image }} tags: ${{ steps.build.outputs.tags }} - registry: ${{ secrets.QUAY_BIOCONDA_REPO }} + mutable-tags: >- + latest + ${{ env.MAJOR_VERSION }} + registry: ${{ vars.QUAY_BIOCONDA_REPO }} username: ${{ secrets.QUAY_BIOCONDA_USERNAME }} password: ${{ secrets.QUAY_BIOCONDA_TOKEN }} - if: ${{ github.ref == 'refs/heads/master' }} name: Test Pushed + env: + TEST_IMAGE: ${{ steps.publish.outputs.test-image }} run: | - image='${{ env.IMAGE_NAME }}' - ids="$( - for tag in ${{ steps.build.outputs.tags }} ; do - buildah manifest inspect "${image}:${tag}" \ - | jq -r '.manifests[]|.digest' \ - | while read id ; do - buildah images --format '{{.ID}}{{.Digest}}' \ - | sed -n "s/${id}//p" - done - done - )" - ids="$( printf %s "${ids}" | sort -u )" - for id in ${ids} ; do - podman history "${id}" - buildah bud \ - --build-arg=base="${id}" \ - --file=Dockerfile.test \ - "images/${image}" - done - buildah rmi --prune || true + .github/scripts/test-pushed-container \ + 'images/${{ env.IMAGE_NAME }}/Dockerfile.test' \ + 'images/${{ env.IMAGE_NAME }}' diff --git a/.github/workflows/bioconda-recipes-issue-responder.yaml b/.github/workflows/bioconda-recipes-issue-responder.yaml deleted file mode 100644 index ef4fe4ce73b..00000000000 --- a/.github/workflows/bioconda-recipes-issue-responder.yaml +++ /dev/null @@ -1,104 +0,0 @@ -name: 'Build & Push: bioconda-recipes-issue-responder' -on: - push: - branches: - - master - paths: - - images/bioconda-recipes-issue-responder/* - - .github/workflows/bioconda-recipes-issue-responder.yaml - pull_request: - paths: - - images/bioconda-recipes-issue-responder/* - - .github/workflows/bioconda-recipes-issue-responder.yaml - -jobs: - build: - name: Build & Push - runs-on: ubuntu-22.04 - env: - IMAGE_NAME: bioconda-recipes-issue-responder - IMAGE_VERSION: '1.1.1' - - steps: - - uses: actions/checkout@v6 - - - name: Build - id: buildah-build - uses: redhat-actions/buildah-build@v2 - with: - image: ${{ env.IMAGE_NAME }} - tags: >- - latest - ${{ env.IMAGE_VERSION }} - context: ./images/${{ env.IMAGE_NAME }} - dockerfiles: | - ./images/${{ env.IMAGE_NAME }}/Dockerfile - - - name: Test - run: | - image='${{ steps.buildah-build.outputs.image }}' - ids="$( - for tag in ${{ steps.buildah-build.outputs.tags }} ; do - buildah images --quiet --no-trunc "${image}:${tag}" - done - )" - ids="$( printf %s "${ids}" | sort -u )" - for id in ${ids} ; do - podman history "${id}" - buildah bud \ - --build-arg=base="${id}" \ - --file=Dockerfile.test \ - "images/${image}" - done - buildah rmi --prune || true - - - if: ${{ github.ref == 'refs/heads/master' }} - name: Check Tags - run: | - # Quay.io does not support immutable images. - # => Check for duplicate tags to avoid overwriting existing images. - existing_tags="$( - skopeo list-tags docker://quay.io/bioconda/${{ steps.buildah-build.outputs.image }} \ - | jq -r '.Tags[]' - )" \ - || { - echo 'Could not list tags via skopeo.' - exit 1 - } - for tag in ${{ steps.buildah-build.outputs.tags }} ; do - if [ \! "${tag}" = latest ] ; then - if printf %s "${existing_tags}" | grep -qxF "${tag}" ; then - printf 'Tag %s already exists!\n' "${tag}" - exit 1 - fi - fi - done - - - if: ${{ github.ref == 'refs/heads/master' }} - name: Push - uses: redhat-actions/push-to-registry@v2 - with: - image: ${{ steps.buildah-build.outputs.image }} - tags: ${{ steps.buildah-build.outputs.tags }} - registry: ${{ secrets.QUAY_BIOCONDA_REPO }} - username: ${{ secrets.QUAY_BIOCONDA_USERNAME }} - password: ${{ secrets.QUAY_BIOCONDA_TOKEN }} - - - if: ${{ github.ref == 'refs/heads/master' }} - name: Test Pushed - run: | - image='${{ steps.buildah-build.outputs.image }}' - ids="$( - for tag in ${{ steps.buildah-build.outputs.tags }} ; do - buildah images --quiet --no-trunc "${image}:${tag}" - done - )" - ids="$( printf %s "${ids}" | sort -u )" - for id in ${ids} ; do - podman history "${id}" - buildah bud \ - --build-arg=base="${id}" \ - --file=Dockerfile.test \ - "images/${image}" - done - buildah rmi --prune || true diff --git a/.github/workflows/bot.yaml b/.github/workflows/bot.yaml index 3812f8b33ec..16dfe8da4a7 100644 --- a/.github/workflows/bot.yaml +++ b/.github/workflows/bot.yaml @@ -30,10 +30,15 @@ jobs: options: --privileged env: IMAGE_NAME: bot - IMAGE_VERSION: '1.4.1' + IMAGE_VERSION: '1.4.2' steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 + + - name: Set up QEMU + run: | + podman run --rm --privileged \ + docker.io/tonistiigi/binfmt --install arm64 - name: Install Tools run: | @@ -57,83 +62,46 @@ jobs: - name: Build id: buildah-build - uses: redhat-actions/buildah-build@v2 + uses: redhat-actions/buildah-build@v3 with: image: ${{ env.IMAGE_NAME }} tags: >- ${{ matrix.tag }} ${{ matrix.tag }}-${{ env.IMAGE_VERSION }} + archs: amd64,arm64 context: ./images/${{ env.IMAGE_NAME }} - dockerfiles: | + containerfiles: | ./images/${{ env.IMAGE_NAME }}/Dockerfile build-args: | packages=${{ matrix.packages }} - name: Test + shell: bash + env: + LOCAL_IMAGE: ${{ steps.buildah-build.outputs.image }} + LOCAL_TAGS: ${{ steps.buildah-build.outputs.tags }} run: | - image='${{ steps.buildah-build.outputs.image }}' - ids="$( - for tag in ${{ steps.buildah-build.outputs.tags }} ; do - buildah images --quiet --no-trunc "${image}:${tag}" - done - )" - ids="$( printf %s "${ids}" | sort -u )" - for id in ${ids} ; do - podman history "${id}" - buildah bud \ - --build-arg=base="${id}" \ - --file=Dockerfile.test \ - 'images/${{ env.IMAGE_NAME }}' - done - buildah rmi --prune || true - - - if: ${{ github.ref == 'refs/heads/master' }} - name: Check Tags - run: | - # Quay.io does not support immutable images. - # => Check for duplicate tags to avoid overwriting existing images. - existing_tags="$( - skopeo list-tags docker://quay.io/bioconda/${{ steps.buildah-build.outputs.image }} \ - | jq -r '.Tags[]' - )" \ - || { - echo 'Could not list tags via skopeo.' - exit 1 - } - for tag in ${{ steps.buildah-build.outputs.tags }} ; do - if [ \! "${tag}" = '${{ matrix.tag }}' ] ; then - if printf %s "${existing_tags}" | grep -qxF "${tag}" ; then - printf 'Tag %s already exists!\n' "${tag}" - exit 1 - fi - fi - done + .github/scripts/test-local-container \ + 'images/${{ env.IMAGE_NAME }}/Dockerfile.test' \ + 'images/${{ env.IMAGE_NAME }}' - if: ${{ github.ref == 'refs/heads/master' }} - name: Push - uses: redhat-actions/push-to-registry@v2 + id: publish + name: Publish + uses: ./.github/actions/publish-container with: image: ${{ steps.buildah-build.outputs.image }} tags: ${{ steps.buildah-build.outputs.tags }} - registry: ${{ secrets.QUAY_BIOCONDA_REPO }} + mutable-tags: ${{ matrix.tag }} + registry: ${{ vars.QUAY_BIOCONDA_REPO }} username: ${{ secrets.QUAY_BIOCONDA_USERNAME }} password: ${{ secrets.QUAY_BIOCONDA_TOKEN }} - if: ${{ github.ref == 'refs/heads/master' }} name: Test Pushed + env: + TEST_IMAGE: ${{ steps.publish.outputs.test-image }} run: | - image='${{ steps.buildah-build.outputs.image }}' - ids="$( - for tag in ${{ steps.buildah-build.outputs.tags }} ; do - buildah images --quiet --no-trunc "${image}:${tag}" - done - )" - ids="$( printf %s "${ids}" | sort -u )" - for id in ${ids} ; do - podman history "${id}" - buildah bud \ - --build-arg=base="${id}" \ - --file=Dockerfile.test \ - 'images/${{ env.IMAGE_NAME }}' - done - buildah rmi --prune || true + .github/scripts/test-pushed-container \ + 'images/${{ env.IMAGE_NAME }}/Dockerfile.test' \ + 'images/${{ env.IMAGE_NAME }}' diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index cb021affc03..04fb16ff113 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -14,50 +14,16 @@ jobs: build: name: Build image runs-on: ubuntu-24.04 - strategy: - matrix: - include: - - arch: arm64 - image: bioconda-utils-build-env-cos7-aarch64 - base_image: quay.io/condaforge/linux-anvil-aarch64 - - arch: amd64 - image: bioconda-utils-build-env-cos7-x86_64 - base_image: quay.io/condaforge/linux-anvil-cos7-x86_64 + container: + # travier/podman-action contains newer podman/buildah versions. + image: quay.io/travier/podman-action + options: --privileged steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: fetch-depth: 0 - - id: get-tag - run: | - tag=${{ github.event.release && github.event.release.tag_name || github.sha }} - - # https://github.blog/changelog/2022-10-11-github-actions-deprecating-save-state-and-set-output-commands/ - # printf %s "::set-output name=tag::${tag#v}" - printf %s "tag=${tag#v}" >> $GITHUB_OUTPUT - - - name: Install qemu dependency - run: | - sudo apt-get update - sudo apt-get install -y qemu-user-static - - - name: Build image - id: buildah-build - uses: redhat-actions/buildah-build@v2 + - name: Build and test image + uses: ./.github/actions/build-bioconda-utils-container with: - image: ${{ matrix.image }} - arch: ${{ matrix.arch }} - build-args: | - BASE_IMAGE=${{ matrix.base_image }} - tags: >- - latest - ${{ steps.get-tag.outputs.tag }} - dockerfiles: | - ./Dockerfile - - - name: Test built image - run: | - image='${{ steps.buildah-build.outputs.image }}' - for tag in ${{ steps.buildah-build.outputs.tags }} ; do - podman run --rm "${image}:${tag}" bioconda-utils --version - done + tags: ${{ github.sha }} diff --git a/.github/workflows/changevisibility.yml b/.github/workflows/changevisibility.yml index 5aa2f2b3079..a14599c8cf8 100644 --- a/.github/workflows/changevisibility.yml +++ b/.github/workflows/changevisibility.yml @@ -1,28 +1,28 @@ -name: Change Container Visibility -on: - workflow_dispatch: - schedule: - - cron: '0 2 * * *' # run at 2 AM UTC -jobs: - changevisibility: - runs-on: ubuntu-latest - defaults: - run: - shell: bash -l {0} - steps: - - uses: actions/checkout@v6 - - - name: Check Containers and Set Public - run: | - python -m pip install requests - python .github/quay-namespace-info.py --namespace biocontainers --changevisibility - env: - QUAY_OAUTH_TOKEN: ${{ secrets.QUAY_BIOCONTAINERS_TOKEN }} - - - name: Upload logs - uses: actions/upload-artifact@v7 - with: - name: logs - path: biocontainers-*.txt - retention-days: 7 - +name: Change Container Visibility +on: + workflow_dispatch: + schedule: + - cron: '0 2 * * *' # run at 2 AM UTC +jobs: + changevisibility: + runs-on: ubuntu-latest + defaults: + run: + shell: bash -l {0} + steps: + - uses: actions/checkout@v7 + + - name: Check Containers and Set Public + run: | + python -m pip install requests + python .github/quay-namespace-info.py --namespace biocontainers --changevisibility + env: + QUAY_OAUTH_TOKEN: ${{ secrets.QUAY_BIOCONTAINERS_TOKEN }} + + - name: Upload logs + uses: actions/upload-artifact@v7 + with: + name: logs + path: biocontainers-*.txt + retention-days: 7 + diff --git a/.github/workflows/create-env.yaml b/.github/workflows/create-env.yaml index 938945a5535..0e4a238ee72 100644 --- a/.github/workflows/create-env.yaml +++ b/.github/workflows/create-env.yaml @@ -25,7 +25,7 @@ jobs: IMAGE_NAME: create-env steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Set up QEMU run: | @@ -61,11 +61,11 @@ jobs: | grep BIOCONDA_UTILS_TAG \ | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' )" - printf 'bioconda_utils_version=%s\n' "${version}" >> $GITHUB_OUTPUT + printf 'bioconda_utils_version=%s\n' "${version}" >> "${GITHUB_OUTPUT}" - name: Build id: build - uses: redhat-actions/buildah-build@v2 + uses: redhat-actions/buildah-build@v3 with: image: ${{ env.IMAGE_NAME }} tags: >- @@ -80,84 +80,34 @@ jobs: bioconda_utils_version=${{ steps.prepare.outputs.bioconda_utils_version }} - name: Test + shell: bash + env: + LOCAL_IMAGE: ${{ steps.build.outputs.image }} + LOCAL_TAGS: ${{ steps.build.outputs.tags }} run: | - set -x - image='${{ steps.build.outputs.image }}' - for tag in ${{ steps.build.outputs.tags }} ; do - buildah manifest inspect \ - "${image}:${tag}" \ - | jq '.manifests|{([.[].digest]|sort|join("+")): [.[]|["'"${tag}"'", .platform.architecture, .digest]|join(" ")]}' - done \ - | jq -rs 'add|add[]' \ - | while read tag arch digest ; do - podman images --format='{{.ID}}|{{.Digest}}|{{.RepoDigests}}' \ - | sed -n "/${digest}/{s/|.*//p;q}" \ - | xargs -n1 -- \ - sh -xc \ - 'podman history "${1}" ; podman inspect "${1}"' -- - buildah bud \ - --arch="${arch}" \ - --build-arg=base="${image}:${tag}" \ - --file=Dockerfile.test \ - "images/${image}" - done - buildah rmi --prune || true + .github/scripts/test-local-container \ + 'images/${{ env.IMAGE_NAME }}/Dockerfile.test' \ + 'images/${{ env.IMAGE_NAME }}' - if: ${{ github.ref == 'refs/heads/master' }} - name: Check Tags - run: | - # Quay.io does not support immutable images. - # => Check for duplicate tags to avoid overwriting existing images. - existing_tags="$( - skopeo list-tags docker://quay.io/bioconda/${{ steps.build.outputs.image }} \ - | jq -r '.Tags[]' - )" \ - || { - echo 'Could not list tags via skopeo.' - exit 1 - } - for tag in ${{ steps.build.outputs.tags }} ; do - case "${tag}" in - latest | '${{ env.MAJOR_VERSION }}' ) ;; - * ) - if printf %s "${existing_tags}" | grep -qxF "${tag}" ; then - printf 'Tag %s already exists!\n' "${tag}" - exit 1 - fi - esac - done - - - if: ${{ github.ref == 'refs/heads/master' }} - name: Push - uses: redhat-actions/push-to-registry@v2 + id: publish + name: Publish + uses: ./.github/actions/publish-container with: image: ${{ steps.build.outputs.image }} tags: ${{ steps.build.outputs.tags }} - registry: ${{ secrets.QUAY_BIOCONDA_REPO }} + mutable-tags: >- + latest + ${{ env.MAJOR_VERSION }} + registry: ${{ vars.QUAY_BIOCONDA_REPO }} username: ${{ secrets.QUAY_BIOCONDA_USERNAME }} password: ${{ secrets.QUAY_BIOCONDA_TOKEN }} - if: ${{ github.ref == 'refs/heads/master' }} name: Test Pushed + env: + TEST_IMAGE: ${{ steps.publish.outputs.test-image }} run: | - set -x - image='${{ steps.build.outputs.image }}' - for tag in ${{ steps.build.outputs.tags }} ; do - buildah manifest inspect \ - "${image}:${tag}" \ - | jq '.manifests|{([.[].digest]|sort|join("+")): [.[]|["'"${tag}"'", .platform.architecture, .digest]|join(" ")]}' - done \ - | jq -rs 'add|add[]' \ - | while read tag arch digest ; do - podman images --format='{{.ID}}|{{.Digest}}|{{.RepoDigests}}' \ - | sed -n "/${digest}/{s/|.*//p;q}" \ - | xargs -n1 -- \ - sh -xc \ - 'podman history "${1}" ; podman inspect "${1}"' -- - buildah bud \ - --arch="${arch}" \ - --build-arg=base="${image}:${tag}" \ - --file=Dockerfile.test \ - "images/${image}" - done - buildah rmi --prune || true + .github/scripts/test-pushed-container \ + 'images/${{ env.IMAGE_NAME }}/Dockerfile.test' \ + 'images/${{ env.IMAGE_NAME }}' diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index c15154fd5f2..4cc651c6351 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -20,100 +20,50 @@ jobs: package-name: bioconda-utils publish_containers: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 + container: + # travier/podman-action contains newer podman/buildah versions. + image: quay.io/travier/podman-action + options: --privileged needs: release_please if: needs.release_please.outputs.release_created steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: fetch-depth: 0 - id: get-tag run: | - #tag=${{ github.event.release && github.event.release.tag_name || github.sha }} tag=${{ needs.release_please.outputs.tag_name }} - printf %s "tag=${tag#v}" >> $GITHUB_OUTPUT - - - name: Install qemu dependency - run: | - sudo apt-get update - sudo apt-get install -y qemu-user-static + printf %s "tag=${tag#v}" >> "${GITHUB_OUTPUT}" - - name: Build x86_64 Image - id: buildah-build - uses: redhat-actions/buildah-build@v2 + - name: Build and test image + id: build + uses: ./.github/actions/build-bioconda-utils-container with: - image: bioconda-utils-build-env-cos7-x86_64 - arch: amd64 - build-args: | - BASE_IMAGE=quay.io/condaforge/linux-anvil-cos7-x86_64 tags: >- latest ${{ steps.get-tag.outputs.tag }} - dockerfiles: | - ./Dockerfile - - name: Push To Quay - uses: redhat-actions/push-to-registry@v2 + - name: Publish + id: publish + uses: ./.github/actions/publish-container with: - image: ${{ steps.buildah-build.outputs.image }} - tags: ${{ steps.buildah-build.outputs.tags }} - registry: ${{ secrets.QUAY_BIOCONDA_REPO }} + image: ${{ steps.build.outputs.image }} + tags: ${{ steps.build.outputs.tags }} + mutable-tags: latest + # TODO: Store the public repository path as a GitHub variable instead of a secret. + registry: ${{ vars.QUAY_BIOCONDA_REPO }} username: ${{ secrets.QUAY_BIOCONDA_USERNAME }} password: ${{ secrets.QUAY_BIOCONDA_TOKEN }} - - name: Build ARM Image - id: buildah-build-arm - uses: redhat-actions/buildah-build@v2 - with: - image: bioconda-utils-build-env-cos7-aarch64 - arch: arm64 - build-args: | - BASE_IMAGE=quay.io/condaforge/linux-anvil-aarch64 - tags: >- - latest - ${{ steps.get-tag.outputs.tag }} - dockerfiles: | - ./Dockerfile - - - name: Push To Quay - uses: redhat-actions/push-to-registry@v2 - with: - image: ${{ steps.buildah-build-arm.outputs.image }} - tags: ${{ steps.buildah-build-arm.outputs.tags }} - registry: ${{ secrets.QUAY_BIOCONDA_REPO }} - username: ${{ secrets.QUAY_BIOCONDA_USERNAME }} - password: ${{ secrets.QUAY_BIOCONDA_TOKEN }} - - - name: Push multi-arch manifest to Quay + - name: Test Pushed + shell: bash env: - QUAY_BIOCONDA_REPO: ${{ secrets.QUAY_BIOCONDA_REPO }} - QUAY_BIOCONDA_USERNAME: ${{ secrets.QUAY_BIOCONDA_USERNAME }} - QUAY_BIOCONDA_TOKEN: ${{ secrets.QUAY_BIOCONDA_TOKEN }} + TEST_IMAGE: ${{ steps.publish.outputs.test-image }} run: | - set -euo pipefail - - registry="${QUAY_BIOCONDA_REPO%%/*}" - printf %s "${QUAY_BIOCONDA_TOKEN}" \ - | docker login "${registry}" \ - --username "${QUAY_BIOCONDA_USERNAME}" \ - --password-stdin + set -euxo pipefail + .github/scripts/test-pushed-container Dockerfile.test . - for tag in latest '${{ steps.get-tag.outputs.tag }}' ; do - docker manifest create \ - "${QUAY_BIOCONDA_REPO}/bioconda-utils-build-env-cos7:${tag}" \ - "${QUAY_BIOCONDA_REPO}/bioconda-utils-build-env-cos7-x86_64:${tag}" \ - "${QUAY_BIOCONDA_REPO}/bioconda-utils-build-env-cos7-aarch64:${tag}" - docker manifest annotate \ - "${QUAY_BIOCONDA_REPO}/bioconda-utils-build-env-cos7:${tag}" \ - "${QUAY_BIOCONDA_REPO}/bioconda-utils-build-env-cos7-x86_64:${tag}" \ - --arch amd64 - docker manifest annotate \ - "${QUAY_BIOCONDA_REPO}/bioconda-utils-build-env-cos7:${tag}" \ - "${QUAY_BIOCONDA_REPO}/bioconda-utils-build-env-cos7-aarch64:${tag}" \ - --arch arm64 - docker manifest push --purge \ - "${QUAY_BIOCONDA_REPO}/bioconda-utils-build-env-cos7:${tag}" - docker manifest inspect \ - "${QUAY_BIOCONDA_REPO}/bioconda-utils-build-env-cos7:${tag}" - done + podman run --rm --pull=always --arch=amd64 "${TEST_IMAGE}" bioconda-utils --version + podman run --rm --pull=always --arch=arm64 "${TEST_IMAGE}" bioconda-utils --version diff --git a/Dockerfile.test b/Dockerfile.test index 5cbffa745f9..7a21f39a238 100644 --- a/Dockerfile.test +++ b/Dockerfile.test @@ -1,4 +1,5 @@ -FROM quay.io/bioconda/bioconda-utils-build-env-cos7 +ARG base=quay.io/bioconda/bioconda-utils-build-env-cos7 +FROM "${base}" # Retrieve index and set TTL to make it last over the duration of the test. RUN . /opt/conda/etc/profile.d/conda.sh && \ conda search bioconda-utils > /dev/null && \ diff --git a/images/bioconda-recipes-issue-responder/Dockerfile b/images/bioconda-recipes-issue-responder/Dockerfile deleted file mode 100644 index f57e2593427..00000000000 --- a/images/bioconda-recipes-issue-responder/Dockerfile +++ /dev/null @@ -1,39 +0,0 @@ -ARG base=quay.io/bioconda/base-glibc-busybox-bash:2.0.0 - -FROM quay.io/bioconda/create-env:2.0.0 as build -RUN /opt/create-env/env-execute \ - create-env \ - --strip-files=\* \ - --remove-paths=\*.a \ - --remove-paths=\*.pyc \ - /usr/local \ - aiohttp \ - anaconda-client \ - ca-certificates \ - git \ - openssh \ - python=3.13 \ - pyyaml \ - skopeo \ - && \ - # Workaround for https://github.com/conda/conda/issues/10490 - export CONDA_REPODATA_THREADS=1 && \ - # We don't need Perl (used by Git for some functionalities). - # => Remove perl package to reduce image size. - /opt/create-env/env-execute \ - conda remove --yes \ - --prefix=/usr/local \ - --force-remove \ - perl - -FROM "${base}" -COPY --from=build /usr/local /usr/local -COPY ./issue-responder /usr/local/bin/ - -# Used environment variables: -# - JOB_CONTEXT -# - BOT_TOKEN -# - GITTER_TOKEN -# - ANACONDA_TOKEN -# - QUAY_OAUTH_TOKEN -# - QUAY_LOGIN diff --git a/images/bioconda-recipes-issue-responder/Dockerfile.test b/images/bioconda-recipes-issue-responder/Dockerfile.test deleted file mode 100644 index 665dc72ed0a..00000000000 --- a/images/bioconda-recipes-issue-responder/Dockerfile.test +++ /dev/null @@ -1,7 +0,0 @@ -ARG base - - -FROM "${base}" -RUN JOB_CONTEXT='{"event": {"issue": {}}}' \ - /usr/local/env-execute \ - issue-responder diff --git a/images/bioconda-recipes-issue-responder/issue-responder b/images/bioconda-recipes-issue-responder/issue-responder deleted file mode 100755 index 9d915f2f528..00000000000 --- a/images/bioconda-recipes-issue-responder/issue-responder +++ /dev/null @@ -1,615 +0,0 @@ -#! /usr/bin/env python - -import logging -import os -import re -import sys -from asyncio import gather, run, sleep -from asyncio.subprocess import create_subprocess_exec -from pathlib import Path -from shutil import which -from subprocess import check_call -from typing import Any, Dict, List, Optional, Set, Tuple -from zipfile import ZipFile - -from aiohttp import ClientSession -from yaml import safe_load - -logger = logging.getLogger(__name__) -log = logger.info - - -async def async_exec( - command: str, *arguments: str, env: Optional[Dict[str, str]] = None -) -> None: - process = await create_subprocess_exec(command, *arguments, env=env) - return_code = await process.wait() - if return_code != 0: - raise RuntimeError( - f"Failed to execute {command} {arguments} (return code: {return_code})" - ) - - -# Post a comment on a given issue/PR with text in message -async def send_comment(session: ClientSession, issue_number: int, message: str) -> None: - token = os.environ["BOT_TOKEN"] - url = ( - f"https://api.github.com/repos/bioconda/bioconda-recipes/issues/{issue_number}/comments" - ) - headers = { - "Authorization": f"token {token}", - "User-Agent": "BiocondaCommentResponder", - } - payload = {"body": message} - log("Sending comment: url=%s", url) - log("Sending comment: payload=%s", payload) - async with session.post(url, headers=headers, json=payload) as response: - status_code = response.status - log("the response code was %d", status_code) - if status_code < 200 or status_code > 202: - sys.exit(1) - - -def list_zip_contents(fname: str) -> [str]: - f = ZipFile(fname) - return [e.filename for e in f.infolist() if e.filename.endswith('.tar.gz') or e.filename.endswith('.tar.bz2')] - - -# Download a zip file from url to zipName.zip and return that path -# Timeout is 30 minutes to compensate for any network issues -async def download_file(session: ClientSession, zipName: str, url: str) -> str: - async with session.get(url, timeout=60*30) as response: - if response.status == 200: - ofile = f"{zipName}.zip" - with open(ofile, 'wb') as fd: - while True: - chunk = await response.content.read(1024*1024*1024) - if not chunk: - break - fd.write(chunk) - return ofile - return None - - -# Find artifact zip files, download them and return their URLs and contents -async def fetch_azure_zip_files(session: ClientSession, buildId: str) -> [(str, str)]: - artifacts = [] - - url = f"https://dev.azure.com/bioconda/bioconda-recipes/_apis/build/builds/{buildId}/artifacts?api-version=4.1" - log("contacting azure %s", url) - async with session.get(url) as response: - # Sometimes we get a 301 error, so there are no longer artifacts available - if response.status == 301: - return artifacts - res = await response.text() - - res_object = safe_load(res) - if res_object['count'] == 0: - return artifacts - - for artifact in res_object['value']: - zipName = artifact['name'] # LinuxArtifacts or OSXArtifacts - zipUrl = artifact['resource']['downloadUrl'] - log(f"zip name is {zipName} url {zipUrl}") - fname = await download_file(session, zipName, zipUrl) - if not fname: - continue - pkgsImages = list_zip_contents(fname) - for pkg in pkgsImages: - artifacts.append((zipUrl, pkg)) - - return artifacts - - -def parse_azure_build_id(url: str) -> str: - return re.search("buildId=(\d+)", url).group(1) - - -# Given a PR and commit sha, fetch a list of the artifact zip files URLs and their contents -async def fetch_pr_sha_artifacts(session: ClientSession, pr: int, sha: str) -> List[Tuple[str, str]]: - url = f"https://api.github.com/repos/bioconda/bioconda-recipes/commits/{sha}/check-runs" - - headers = { - "User-Agent": "BiocondaCommentResponder", - "Accept": "application/vnd.github.antiope-preview+json", - } - async with session.get(url, headers=headers) as response: - response.raise_for_status() - res = await response.text() - check_runs = safe_load(res) - log(f"DEBUG url was {url} returned {check_runs}") - - for check_run in check_runs["check_runs"]: - # The names are "bioconda.bioconda-recipes (test_osx test_osx)" or similar - if check_run["name"].startswith("bioconda.bioconda-recipes (test_"): - # The azure build ID is in the details_url as buildId=\d+ - buildID = parse_azure_build_id(check_run["details_url"]) - log(f"DEBUG buildID is {buildID}") - zipFiles = await fetch_azure_zip_files(session, buildID) - log(f"DEBUG zipFiles are {zipFiles}") - return zipFiles # We've already fetched all possible artifacts - - return [] - - -# Given a PR and commit sha, post a comment with any artifacts -async def make_artifact_comment(session: ClientSession, pr: int, sha: str) -> None: - artifacts = await fetch_pr_sha_artifacts(session, pr, sha) - nPackages = len(artifacts) - log(f"DEBUG the artifacts are {artifacts}") - - if nPackages > 0: - comment = "Package(s) built on Azure are ready for inspection:\n\n" - comment += "Arch | Package | Zip File\n-----|---------|---------\n" - install_noarch = "" - install_linux = "" - install_osx = "" - - # Table of packages and repodata.json - for URL, artifact in artifacts: - if not (package_match := re.match(r"^((.+)\/(.+)\/(.+)\/(.+\.tar\.bz2))$", artifact)): - continue - url, archdir, basedir, subdir, packageName = package_match.groups() - urlBase = URL[:-3] # trim off zip from format= - urlBase += "file&subPath=%2F{}".format("%2F".join([basedir, subdir])) - conda_install_url = urlBase - # N.B., the zip file URL is nearly identical to the URL for the individual member files. It's unclear if there's an API for getting the correct URL to the files themselves - #pkgUrl = "%2F".join([urlBase, packageName]) - #repoUrl = "%2F".join([urlBase, "current_repodata.json"]) - #resp = await session.get(repoUrl) - - if subdir == "noarch": - comment += "noarch |" - elif subdir == "linux-64": - comment += "linux-64 |" - else: - comment += "osx-64 |" - comment += f" {packageName} | [{archdir}]({URL})\n" - - # Conda install examples - comment += "***\n\nYou may also use `conda` to install these after downloading and extracting the appropriate zip file. From the LinuxArtifacts or OSXArtifacts directories:\n\n" - comment += "```conda install -c ./packages \n```\n" - - # Table of containers - comment += "***\n\nDocker image(s) built (images are in the LinuxArtifacts zip file above):\n\n" - comment += "Package | Tag | Install with `docker`\n" - comment += "--------|-----|----------------------\n" - - for URL, artifact in artifacts: - if artifact.endswith(".tar.gz"): - image_name = artifact.split("/").pop()[: -len(".tar.gz")] - if ':' in image_name: - package_name, tag = image_name.split(':', 1) - #image_url = URL[:-3] # trim off zip from format= - #image_url += "file&subPath=%2F{}.tar.gz".format("%2F".join(["images", '%3A'.join([package_name, tag])])) - comment += f"[{package_name}] | {tag} | " - comment += f'
show`gzip -dc LinuxArtifacts/images/{image_name}.tar.gz \\| docker load`\n' - comment += "\n\n" - else: - comment = ( - "No artifacts found on the most recent Azure build. " - "Either the build failed, the artifacts have were removed due to age, or the recipe was blacklisted/skipped." - ) - await send_comment(session, pr, comment) - - -# Post a comment on a given PR with its CircleCI artifacts -async def artifact_checker(session: ClientSession, issue_number: int) -> None: - url = f"https://api.github.com/repos/bioconda/bioconda-recipes/pulls/{issue_number}" - headers = { - "User-Agent": "BiocondaCommentResponder", - } - async with session.get(url, headers=headers) as response: - response.raise_for_status() - res = await response.text() - pr_info = safe_load(res) - - await make_artifact_comment(session, issue_number, pr_info["head"]["sha"]) - - -# Return true if a user is a member of bioconda -async def is_bioconda_member(session: ClientSession, user: str) -> bool: - token = os.environ["BOT_TOKEN"] - url = f"https://api.github.com/orgs/bioconda/members/{user}" - headers = { - "Authorization": f"token {token}", - "User-Agent": "BiocondaCommentResponder", - } - rc = 404 - async with session.get(url, headers=headers) as response: - try: - response.raise_for_status() - rc = response.status - except: - # Do nothing, this just prevents things from crashing on 404 - pass - - return rc == 204 - - -# Reposts a quoted message in a given issue/PR if the user isn't a bioconda member -async def comment_reposter(session: ClientSession, user: str, pr: int, message: str) -> None: - if await is_bioconda_member(session, user): - log("Not reposting for %s", user) - return - log("Reposting for %s", user) - await send_comment( - session, - pr, - f"Reposting for @{user} to enable pings (courtesy of the BiocondaBot):\n\n> {message}", - ) - - -# Fetch and return the JSON of a PR -# This can be run to trigger a test merge -async def get_pr_info(session: ClientSession, pr: int) -> Any: - token = os.environ["BOT_TOKEN"] - url = f"https://api.github.com/repos/bioconda/bioconda-recipes/pulls/{pr}" - headers = { - "Authorization": f"token {token}", - "User-Agent": "BiocondaCommentResponder", - } - async with session.get(url, headers=headers) as response: - response.raise_for_status() - res = await response.text() - pr_info = safe_load(res) - return pr_info - - -# Update a branch from upstream master, this should be run in a try/catch -async def update_from_master_runner(session: ClientSession, pr: int) -> None: - async def git(*args: str) -> None: - return await async_exec("git", *args) - - # Setup git, otherwise we can't push - await git("config", "--global", "user.email", "biocondabot@gmail.com") - await git("config", "--global", "user.name", "BiocondaBot") - - pr_info = await get_pr_info(session, pr) - remote_branch = pr_info["head"]["ref"] - remote_repo = pr_info["head"]["repo"]["full_name"] - - max_depth = 2000 - # Clone - await git( - "clone", - f"--depth={max_depth}", - f"--branch={remote_branch}", - f"git@github.com:{remote_repo}.git", - "bioconda-recipes", - ) - - async def git_c(*args: str) -> None: - return await git("-C", "bioconda-recipes", *args) - - # Add/pull upstream - await git_c("remote", "add", "upstream", "https://github.com/bioconda/bioconda-recipes") - await git_c("fetch", f"--depth={max_depth}", "upstream", "master") - - # Merge - await git_c("merge", "upstream/master") - - await git_c("push") - - -# Merge the upstream master branch into a PR branch, leave a message on error -async def update_from_master(session: ClientSession, pr: int) -> None: - try: - await update_from_master_runner(session, pr) - except Exception as e: - await send_comment( - session, - pr, - "I encountered an error updating your PR branch. You can report this to bioconda/core if you'd like.\n-The Bot", - ) - sys.exit(1) - - -# Ensure there's at least one approval by a member -async def approval_review(session: ClientSession, issue_number: int) -> bool: - token = os.environ["BOT_TOKEN"] - url = f"https://api.github.com/repos/bioconda/bioconda-recipes/pulls/{issue_number}/reviews" - headers = { - "Authorization": f"token {token}", - "User-Agent": "BiocondaCommentResponder", - } - async with session.get(url, headers=headers) as response: - response.raise_for_status() - res = await response.text() - reviews = safe_load(res) - - approved_reviews = [review for review in reviews if review["state"] == "APPROVED"] - if not approved_reviews: - return False - - # Ensure the review author is a member - return any( - gather( - *( - is_bioconda_member(session, review["user"]["login"]) - for review in approved_reviews - ) - ) - ) - - -# Check the mergeable state of a PR -async def check_is_mergeable( - session: ClientSession, issue_number: int, second_try: bool = False -) -> bool: - token = os.environ["BOT_TOKEN"] - # Sleep a couple of seconds to allow the background process to finish - if second_try: - await sleep(3) - - # PR info - url = f"https://api.github.com/repos/bioconda/bioconda-recipes/pulls/{issue_number}" - headers = { - "Authorization": f"token {token}", - "User-Agent": "BiocondaCommentResponder", - } - async with session.get(url, headers=headers) as response: - response.raise_for_status() - res = await response.text() - pr_info = safe_load(res) - - # We need mergeable == true and mergeable_state == clean, an approval by a member and - if pr_info.get("mergeable") is None and not second_try: - return await check_is_mergeable(session, issue_number, True) - elif ( - pr_info.get("mergeable") is None - or not pr_info["mergeable"] - or pr_info["mergeable_state"] != "clean" - ): - return False - - return await approval_review(session, issue_number) - - -# Ensure uploaded containers are in repos that have public visibility -async def toggle_visibility(session: ClientSession, container_repo: str) -> None: - url = f"https://quay.io/api/v1/repository/biocontainers/{container_repo}/changevisibility" - QUAY_OAUTH_TOKEN = os.environ["QUAY_OAUTH_TOKEN"] - headers = { - "Authorization": f"Bearer {QUAY_OAUTH_TOKEN}", - "Content-Type": "application/json", - } - body = {"visibility": "public"} - rc = 0 - try: - async with session.post(url, headers=headers, json=body) as response: - rc = response.status - except: - # Do nothing - pass - log("Trying to toggle visibility (%s) returned %d", url, rc) - - -# Download an artifact from CircleCI, rename and upload it -async def download_and_upload(session: ClientSession, x: str) -> None: - basename = x.split("/").pop() - # the tarball needs a regular name without :, the container needs pkg:tag - image_name = basename.replace("%3A", ":").replace("\n", "").replace(".tar.gz", "") - file_name = basename.replace("%3A", "_").replace("\n", "") - - async with session.get(x) as response: - with open(file_name, "wb") as file: - logged = 0 - loaded = 0 - while chunk := await response.content.read(256 * 1024): - file.write(chunk) - loaded += len(chunk) - if loaded - logged >= 50 * 1024 ** 2: - log("Downloaded %.0f MiB: %s", max(1, loaded / 1024 ** 2), x) - logged = loaded - log("Downloaded %.0f MiB: %s", max(1, loaded / 1024 ** 2), x) - - if x.endswith(".gz"): - # Container - log("uploading with skopeo: %s", file_name) - # This can fail, retry with 5 second delays - count = 0 - maxTries = 5 - success = False - QUAY_LOGIN = os.environ["QUAY_LOGIN"] - env = os.environ.copy() - # TODO: Fix skopeo package to find certificates on its own. - skopeo_path = which("skopeo") - if not skopeo_path: - raise RuntimeError("skopeo not found") - env["SSL_CERT_DIR"] = str(Path(skopeo_path).parents[1].joinpath("ssl")) - while count < maxTries: - try: - await async_exec( - "skopeo", - "--command-timeout", - "600s", - "copy", - f"docker-archive:{file_name}", - f"docker://quay.io/biocontainers/{image_name}", - "--dest-creds", - QUAY_LOGIN, - env=env, - ) - success = True - break - except: - count += 1 - if count == maxTries: - raise - await sleep(5) - if success: - await toggle_visibility(session, basename.split("%3A")[0]) - elif x.endswith(".bz2"): - # Package - log("uploading package") - ANACONDA_TOKEN = os.environ["ANACONDA_TOKEN"] - await async_exec("anaconda", "-t", ANACONDA_TOKEN, "upload", file_name, "--force") - - log("cleaning up") - os.remove(file_name) - - -# Upload artifacts to quay.io and anaconda, return the commit sha -# Only call this for mergeable PRs! -async def upload_artifacts(session: ClientSession, pr: int) -> str: - # Get last sha - pr_info = await get_pr_info(session, pr) - sha: str = pr_info["head"]["sha"] - - # Fetch the artifacts - artifacts = await fetch_pr_sha_artifacts(session, pr, sha) - artifacts = [artifact for artifact in artifacts if artifact.endswith((".gz", ".bz2"))] - assert artifacts - - # Download/upload Artifacts - for artifact in artifacts: - await download_and_upload(session, artifact) - - return sha - - -# Assume we have no more than 250 commits in a PR, which is probably reasonable in most cases -async def get_pr_commit_message(session: ClientSession, issue_number: int) -> str: - token = os.environ["BOT_TOKEN"] - url = f"https://api.github.com/repos/bioconda/bioconda-recipes/pulls/{issue_number}/commits" - headers = { - "Authorization": f"token {token}", - "User-Agent": "BiocondaCommentResponder", - } - async with session.get(url, headers=headers) as response: - response.raise_for_status() - res = await response.text() - commits = safe_load(res) - message = "".join(f" * {commit['commit']['message']}\n" for commit in reversed(commits)) - return message - - -# Merge a PR -async def merge_pr(session: ClientSession, pr: int) -> None: - token = os.environ["BOT_TOKEN"] - await send_comment( - session, - pr, - "I will attempt to upload artifacts and merge this PR. This may take some time, please have patience.", - ) - - try: - mergeable = await check_is_mergeable(session, pr) - log("mergeable state of %s is %s", pr, mergeable) - if not mergeable: - await send_comment(session, pr, "Sorry, this PR cannot be merged at this time.") - else: - log("uploading artifacts") - sha = await upload_artifacts(session, pr) - log("artifacts uploaded") - - # Carry over last 250 commit messages - msg = await get_pr_commit_message(session, pr) - - # Hit merge - url = f"https://api.github.com/repos/bioconda/bioconda-recipes/pulls/{pr}/merge" - headers = { - "Authorization": f"token {token}", - "User-Agent": "BiocondaCommentResponder", - } - payload = { - "sha": sha, - "commit_title": f"[ci skip] Merge PR {pr}", - "commit_message": f"Merge PR #{pr}, commits were: \n{msg}", - "merge_method": "squash", - } - log("Putting merge commit") - async with session.put(url, headers=headers, json=payload) as response: - rc = response.status - log("body %s", payload) - log("merge_pr the response code was %s", rc) - except: - await send_comment( - session, - pr, - "I received an error uploading the build artifacts or merging the PR!", - ) - logger.exception("Upload failed", exc_info=True) - - -# Add the "Please review and merge" label to a PR -async def add_pr_label(session: ClientSession, pr: int) -> None: - token = os.environ["BOT_TOKEN"] - url = f"https://api.github.com/repos/bioconda/bioconda-recipes/issues/{pr}/labels" - headers = { - "Authorization": f"token {token}", - "User-Agent": "BiocondaCommentResponder", - } - payload = {"labels": ["please review & merge"]} - async with session.post(url, headers=headers, json=payload) as response: - response.raise_for_status() - - -async def gitter_message(session: ClientSession, msg: str) -> None: - token = os.environ["GITTER_TOKEN"] - room_id = "57f3b80cd73408ce4f2bba26" - url = f"https://api.gitter.im/v1/rooms/{room_id}/chatMessages" - headers = { - "Authorization": f"Bearer {token}", - "Content-Type": "application/json", - "Accept": "application/json", - "User-Agent": "BiocondaCommentResponder", - } - payload = {"text": msg} - log("Sending request to %s", url) - async with session.post(url, headers=headers, json=payload) as response: - response.raise_for_status() - - -async def notify_ready(session: ClientSession, pr: int) -> None: - try: - await gitter_message( - session, - f"PR ready for review: https://github.com/bioconda/bioconda-recipes/pull/{pr}", - ) - except Exception: - logger.exception("Posting to Gitter failed", exc_info=True) - # Do not die if we can't post to gitter! - - -# This requires that a JOB_CONTEXT environment variable, which is made with `toJson(github)` -async def main() -> None: - job_context = safe_load(os.environ["JOB_CONTEXT"]) - log("%s", job_context) - if job_context["event"]["issue"].get("pull_request") is None: - return - issue_number = job_context["event"]["issue"]["number"] - - original_comment = job_context["event"]["comment"]["body"] - log("the comment is: %s", original_comment) - - comment = original_comment.lower() - async with ClientSession() as session: - if comment.startswith(("@bioconda-bot", "@biocondabot")): - if "please update" in comment: - await update_from_master(session, issue_number) - elif " hello" in comment: - await send_comment(session, issue_number, "Yes?") - elif " please fetch artifacts" in comment or " please fetch artefacts" in comment: - await artifact_checker(session, issue_number) - elif " please merge" in comment: - await send_comment(session, issue_number, "Sorry, I'm currently disabled") - #await merge_pr(session, issue_number) - elif " please add label" in comment: - await add_pr_label(session, issue_number) - await notify_ready(session, issue_number) - # else: - # # Methods in development can go below, flanked by checking who is running them - # if job_context["actor"] != "dpryan79": - # console.log("skipping") - # sys.exit(0) - elif "@bioconda/" in comment: - await comment_reposter( - session, job_context["actor"], issue_number, original_comment - ) - - -if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) - run(main())