From 04a3a05f0ca8d6cf63e5f5e3345a5fa7aa8bae42 Mon Sep 17 00:00:00 2001 From: Rickard Date: Wed, 7 Feb 2024 19:03:29 +0100 Subject: [PATCH 1/5] Define CUDA version in dockerfile to allow Dependabot to pick it up --- .github/workflows/python-package.yml | 15 ++++++++++---- docker/cuda.Dockerfile | 7 +++++++ scripts/get_docker_info.sh | 31 ++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 4 deletions(-) create mode 100644 docker/cuda.Dockerfile create mode 100755 scripts/get_docker_info.sh diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 265128637..662b3ef46 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -88,7 +88,7 @@ jobs: matrix: os: [ubuntu-latest, windows-latest] arch: [x86_64, aarch64] - cuda_version: ['12.1.0'] + cuda_version: ['12'] exclude: - os: windows-latest # This probably requires arm64 Windows agents arch: aarch64 @@ -97,6 +97,13 @@ jobs: # Check out code - uses: actions/checkout@v4 # Linux: We use Docker to build cross platform Cuda (aarch64 is built in emulation) + - name: Set CUDA version + shell: bash + run: | + cuda_version=$(scripts/get_docker_info.sh docker/cuda.Dockerfile cuda${{ matrix.cuda_version }}) + echo "CUDA version to use is $cuda_version" + echo "CUDA_VERSION=$cuda_version" >> $GITHUB_ENV + # On Linux we use CMake within Docker - name: Set up Docker multiarch if: startsWith(matrix.os, 'ubuntu') uses: docker/setup-qemu-action@v2 @@ -111,7 +118,7 @@ jobs: if: startsWith(matrix.os, 'windows') id: cuda-toolkit with: - cuda: ${{ matrix.cuda_version }} + cuda: ${{ vars.CUDA_VERSION }} method: 'local' # sub-packages: '["nvcc","cudart","nvrtc_dev","cublas_dev","cusparse_dev","visual_studio_integration"]' - name: Add msbuild to PATH @@ -133,7 +140,7 @@ jobs: build_arch=${{ matrix.arch }} for NO_CUBLASLT in ON OFF; do if [ ${build_os:0:6} == ubuntu ]; then - image=nvidia/cuda:${{ matrix.cuda_version }}-devel-ubuntu22.04 + image=nvidia/cuda:${{ vars.CUDA_VERSION }}-devel-ubuntu22.04 echo "Using image $image" docker run --platform linux/$build_arch -i -w /src -v $PWD:/src $image sh -c \ "apt-get update \ @@ -150,7 +157,7 @@ jobs: - name: Upload build artifact uses: actions/upload-artifact@v4 with: - name: shared_library_cuda_${{ matrix.os }}_${{ matrix.arch }}_${{ matrix.cuda_version }} + name: shared_library_cuda_${{ matrix.os }}_${{ matrix.arch }}_${{ vars.CUDA_VERSION }} path: output/* retention-days: 7 build-wheels: diff --git a/docker/cuda.Dockerfile b/docker/cuda.Dockerfile new file mode 100644 index 000000000..86f8a8116 --- /dev/null +++ b/docker/cuda.Dockerfile @@ -0,0 +1,7 @@ +# Note: This Dockerfile is currently not used for building an image, but +# for extracting the current version of CUDA to use. +# By using a Dockerfile, it is possible to automatically upgrade CUDA +# patch versions through Dependabot. + +FROM nvidia/cuda:11.8.0-devel-ubuntu22.04 AS cuda11 +FROM nvidia/cuda:12.1.0-devel-ubuntu22.04 AS cuda12 diff --git a/scripts/get_docker_info.sh b/scripts/get_docker_info.sh new file mode 100755 index 000000000..41413029e --- /dev/null +++ b/scripts/get_docker_info.sh @@ -0,0 +1,31 @@ +#!/bin/sh + +version_only= +if [ "$1" = "-v" ]; then + version_only=1 + shift 1 +fi + +dockerfile=$1 +shift 1 + +if [ ! -f $dockerfile ]; then + echo "Dockerfile not found" >> /dev/stderr + exit 1 +fi + +if [ -z "$1" ]; then + echo "No target specified" >> /dev/stderr + exit 1 +fi + +tag=$(grep "AS $1\$" $dockerfile | sed -e 's/FROM *//' -e 's/ *AS .*//') +if [ -z "$tag" ]; then + echo "Target $1 not defined" >> /dev/stderr + exit 1 +fi +if [ "$version_only" = "1" ]; then + echo $tag | sed -e 's/.*://' +else + echo $tag +fi From 1acc07ec5c7607278d389feea10194ebe98bc432 Mon Sep 17 00:00:00 2001 From: Rickard Date: Wed, 7 Feb 2024 19:27:18 +0100 Subject: [PATCH 2/5] Added dependabot section --- .github/dependabot.yml | 6 ++++++ .github/workflows/python-package.yml | 10 ++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 8a36c3689..542168889 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -9,3 +9,9 @@ updates: update-types: [major] minor-patch: update-types: [minor, patch] + - package-ecosystem: docker + directory: "/docker" + schedule: + interval: "weekly" + ignore: + - update-types: [ "version-update:semver-major" ] diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 662b3ef46..cfb071e86 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -33,6 +33,8 @@ jobs: - os: windows-latest # This probably requires arm64 Windows agents arch: aarch64 runs-on: ${{ matrix.os }} # One day, we could run them on native agents. Azure supports this now but it's planned only for Q3 2023 for hosted agents + env: + CUDA_VERSION: notset steps: # Check out code - uses: actions/checkout@v4 @@ -100,7 +102,7 @@ jobs: - name: Set CUDA version shell: bash run: | - cuda_version=$(scripts/get_docker_info.sh docker/cuda.Dockerfile cuda${{ matrix.cuda_version }}) + cuda_version=$(scripts/get_docker_info.sh -v docker/cuda.Dockerfile cuda${{ matrix.cuda_version }} | sed -e 's/-.*//') echo "CUDA version to use is $cuda_version" echo "CUDA_VERSION=$cuda_version" >> $GITHUB_ENV # On Linux we use CMake within Docker @@ -118,7 +120,7 @@ jobs: if: startsWith(matrix.os, 'windows') id: cuda-toolkit with: - cuda: ${{ vars.CUDA_VERSION }} + cuda: ${{ env.CUDA_VERSION }} method: 'local' # sub-packages: '["nvcc","cudart","nvrtc_dev","cublas_dev","cusparse_dev","visual_studio_integration"]' - name: Add msbuild to PATH @@ -140,7 +142,7 @@ jobs: build_arch=${{ matrix.arch }} for NO_CUBLASLT in ON OFF; do if [ ${build_os:0:6} == ubuntu ]; then - image=nvidia/cuda:${{ vars.CUDA_VERSION }}-devel-ubuntu22.04 + image=nvidia/cuda:${{ env.CUDA_VERSION }}-devel-ubuntu22.04 echo "Using image $image" docker run --platform linux/$build_arch -i -w /src -v $PWD:/src $image sh -c \ "apt-get update \ @@ -157,7 +159,7 @@ jobs: - name: Upload build artifact uses: actions/upload-artifact@v4 with: - name: shared_library_cuda_${{ matrix.os }}_${{ matrix.arch }}_${{ vars.CUDA_VERSION }} + name: shared_library_cuda_${{ matrix.os }}_${{ matrix.arch }}_${{ env.CUDA_VERSION }} path: output/* retention-days: 7 build-wheels: From efcd78fd019ad0290587c0de11435b502fd44be6 Mon Sep 17 00:00:00 2001 From: Rickard Date: Tue, 27 Feb 2024 22:51:29 +0100 Subject: [PATCH 3/5] Use Python for parsing Dockerfile --- .github/workflows/python-package.yml | 2 +- scripts/get_docker_info.py | 29 ++++++++++++++++++++++++++ scripts/get_docker_info.sh | 31 ---------------------------- 3 files changed, 30 insertions(+), 32 deletions(-) create mode 100755 scripts/get_docker_info.py delete mode 100755 scripts/get_docker_info.sh diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index cfb071e86..42544696c 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -102,7 +102,7 @@ jobs: - name: Set CUDA version shell: bash run: | - cuda_version=$(scripts/get_docker_info.sh -v docker/cuda.Dockerfile cuda${{ matrix.cuda_version }} | sed -e 's/-.*//') + cuda_version=$(scripts/get_docker_info.py -v docker/cuda.Dockerfile cuda${{ matrix.cuda_version }} | sed -e 's/-.*//') echo "CUDA version to use is $cuda_version" echo "CUDA_VERSION=$cuda_version" >> $GITHUB_ENV # On Linux we use CMake within Docker diff --git a/scripts/get_docker_info.py b/scripts/get_docker_info.py new file mode 100755 index 000000000..63ee7f6cb --- /dev/null +++ b/scripts/get_docker_info.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python + +import argparse +import re + +from_line_re = re.compile(r"FROM\s+(?P\S+)\s+AS\s+(?P\S+)") + + +def find_image_in_dockerfile(dockerfile, target): + with open(dockerfile) as f: + for line in f.readlines(): + if (m := from_line_re.match(line)) and m.group("target") == target: + return m.group("image") + raise ValueError(f"Target {target} not defined in {dockerfile}") + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("dockerfile") + ap.add_argument("target") + ap.add_argument("-v", "--version-only", action="store_true") + args = ap.parse_args() + image = find_image_in_dockerfile(args.dockerfile, args.target) + if args.version_only: + image = image.rpartition(":")[-1] + print(image) + +if __name__ == '__main__': + main() diff --git a/scripts/get_docker_info.sh b/scripts/get_docker_info.sh deleted file mode 100755 index 41413029e..000000000 --- a/scripts/get_docker_info.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/sh - -version_only= -if [ "$1" = "-v" ]; then - version_only=1 - shift 1 -fi - -dockerfile=$1 -shift 1 - -if [ ! -f $dockerfile ]; then - echo "Dockerfile not found" >> /dev/stderr - exit 1 -fi - -if [ -z "$1" ]; then - echo "No target specified" >> /dev/stderr - exit 1 -fi - -tag=$(grep "AS $1\$" $dockerfile | sed -e 's/FROM *//' -e 's/ *AS .*//') -if [ -z "$tag" ]; then - echo "Target $1 not defined" >> /dev/stderr - exit 1 -fi -if [ "$version_only" = "1" ]; then - echo $tag | sed -e 's/.*://' -else - echo $tag -fi From 288939317ab1d130200697b533d09b3907bc5a6b Mon Sep 17 00:00:00 2001 From: Rickard Date: Fri, 29 Mar 2024 11:07:08 +0100 Subject: [PATCH 4/5] Update get_docker_info.py --- scripts/get_docker_info.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/scripts/get_docker_info.py b/scripts/get_docker_info.py index 63ee7f6cb..ce21d2a84 100755 --- a/scripts/get_docker_info.py +++ b/scripts/get_docker_info.py @@ -18,12 +18,15 @@ def main(): ap = argparse.ArgumentParser() ap.add_argument("dockerfile") ap.add_argument("target") + ap.add_argument("-t", "--tag-only", action="store_true") ap.add_argument("-v", "--version-only", action="store_true") args = ap.parse_args() - image = find_image_in_dockerfile(args.dockerfile, args.target) - if args.version_only: - image = image.rpartition(":")[-1] - print(image) + result = find_image_in_dockerfile(args.dockerfile, args.target) + if args.tag_only or args.version_only: + result = result.rpartition(":")[-1] + if args.version_only: + result = result.split("-")[0] + print(result) if __name__ == '__main__': main() From 1243718088e7a841e8bd2e1f205f0bed827d2c42 Mon Sep 17 00:00:00 2001 From: Rickard Date: Fri, 29 Mar 2024 11:07:43 +0100 Subject: [PATCH 5/5] Update python-package.yml --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index c8ee39c6d..8376332ed 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -95,7 +95,7 @@ jobs: - name: Set CUDA version shell: bash run: | - cuda_version=$(scripts/get_docker_info.py -v docker/cuda.Dockerfile cuda${{ matrix.cuda_version }} | sed -e 's/-.*//') + cuda_version=$(scripts/get_docker_info.py -v docker/cuda.Dockerfile cuda${{ matrix.cuda_version }}) echo "CUDA version to use is $cuda_version" echo "CUDA_VERSION=$cuda_version" >> $GITHUB_ENV # On Linux we use CMake within Docker