diff --git a/examples/build/basic/buildrunner.yaml b/examples/build/basic/buildrunner.yaml deleted file mode 100644 index 8bbcd6ef..00000000 --- a/examples/build/basic/buildrunner.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# Basic buildrunner configuration to build a simple Docker image. -steps: - simple-build-step: - build: - dockerfile: | - FROM alpine:latest - RUN echo Hello World diff --git a/examples/build/buildargs/buildrunner.yaml b/examples/build/buildargs/buildrunner.yaml deleted file mode 100644 index d5066261..00000000 --- a/examples/build/buildargs/buildrunner.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# This example demonstrates how to use the buildargs field in the build step. -# The buildargs field allows you to pass build-time variables to the Dockerfile. -steps: - buildargs-step: - build: - buildargs: - MY_VERSION: 2.0 - dockerfile: | - FROM alpine:latest - ARG MY_VERSION - LABEL version=$MY_VERSION - - # To view the label in the built image do the following: - # 1. Uncomment the following 'push:' line - # 2. Run 'PYTHONPATH=. ./bin/buildrunner -f examples/build/buildargs/buildrunner.yaml' - # 3. Run 'docker image inspect my-images/buildargs-image:latest' to see the label. - # - # push: my-images/buildargs-image:latest \ No newline at end of file diff --git a/examples/build/cache/buildrunner.yaml b/examples/build/cache/buildrunner.yaml deleted file mode 100644 index 87c2772b..00000000 --- a/examples/build/cache/buildrunner.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# Description: This example demonstrates how to use the no-cache option in the build step. -steps: - cache-build-step: - build: - no-cache: false - dockerfile: | - FROM alpine:latest - RUN apk update - RUN echo Hello World - no-cache-build-step: - build: - no-cache: true - dockerfile: | - FROM alpine:latest - RUN apk update - RUN echo Hello World \ No newline at end of file diff --git a/examples/build/import/buildrunner.yaml b/examples/build/import/buildrunner.yaml deleted file mode 100644 index 4aada61a..00000000 --- a/examples/build/import/buildrunner.yaml +++ /dev/null @@ -1,11 +0,0 @@ -# This example demonstrates how to import a tarball image in the build step. -# This requires the tarball image to be present in the same directory where buildrunnner is executed. -# Before running this example, make sure to create a tarball image using the following command: -# docker save alpine:latest -o alpine.tar -# -# WARNING: Do not commit the tarball image to the repository. -# -steps: - import-step: - build: - import: alpine.tar \ No newline at end of file diff --git a/examples/build/inject/buildrunner.yaml b/examples/build/inject/buildrunner.yaml deleted file mode 100644 index 9436f9cb..00000000 --- a/examples/build/inject/buildrunner.yaml +++ /dev/null @@ -1,14 +0,0 @@ -# This example demonstrates how to use the inject field to inject a file into the build context. -# See buildrunner documentation for more information. -steps: - build-step-with-inject: - build: - inject: - # Step 1: Inject a file into the build context - # The file /injectfile.txt will be injected into the build context - examples/build/inject/injectfile.txt: /injectfile.txt - dockerfile: | - FROM alpine - # Step 2: Copy the injected file into the image or use it in some other way in the build process - COPY /injectfile.txt /tmp/ - RUN cat /tmp/injectfile.txt \ No newline at end of file diff --git a/examples/build/inject/injectfile.txt b/examples/build/inject/injectfile.txt deleted file mode 100644 index 6a8e3237..00000000 --- a/examples/build/inject/injectfile.txt +++ /dev/null @@ -1 +0,0 @@ -This is a sample text file that can be injected into the build context \ No newline at end of file diff --git a/examples/build/platform/README.rst b/examples/build/platform/README.rst deleted file mode 100644 index 1376518d..00000000 --- a/examples/build/platform/README.rst +++ /dev/null @@ -1 +0,0 @@ -Contains an example of building a single-platform image. \ No newline at end of file diff --git a/examples/build/platform/buildrunner.yaml b/examples/build/platform/buildrunner.yaml deleted file mode 100644 index e88c6c52..00000000 --- a/examples/build/platform/buildrunner.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# Example of a buildrunner.yaml file that specifies a single platform build step. -# The 'platform' field is optional and can be used to specify the platform for the build step. -# If the 'platform' field is not specified, the default platform is native platform of the machine running buildrunner. -steps: - single-platform-build-step: - build: - dockerfile: | - FROM alpine:latest - RUN echo Hello World - platform: linux/amd64 \ No newline at end of file diff --git a/examples/build/platforms/README.rst b/examples/build/platforms/README.rst deleted file mode 100644 index 9bba979b..00000000 --- a/examples/build/platforms/README.rst +++ /dev/null @@ -1 +0,0 @@ -Contains an example of building a multi-platform image. \ No newline at end of file diff --git a/examples/build/platforms/buildrunner.yaml b/examples/build/platforms/buildrunner.yaml deleted file mode 100644 index 35d4bd28..00000000 --- a/examples/build/platforms/buildrunner.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Example of a buildrunner.yaml file that specifies a multi-platform build step. -# The 'platforms' field is optional and can be used to specify the platforms for the build step. -# If the 'platforms' field is not specified, by default it will build a single platform, using the native platform of the machine running buildrunner. -steps: - multi-platform-build-step: - build: - dockerfile: | - FROM alpine:latest - LABEL custom_label="Buildrunner example label" - platforms: - - linux/amd64 - - linux/arm64/v8 diff --git a/examples/build/secrets/buildrunner.yaml b/examples/build/secrets/buildrunner.yaml deleted file mode 100644 index 2c4a0283..00000000 --- a/examples/build/secrets/buildrunner.yaml +++ /dev/null @@ -1,26 +0,0 @@ -# In order to use secrets, you need to set use-legacy-builder to false in the config file -# To run this example, use the following command: -# SECRET2=my_secret ./run-buildrunner.sh -f examples/build/secrets/buildrunner.yaml -# More info about secrets: https://docs.docker.com/build/building/secrets/ -use-legacy-builder: false -steps: - simple-build-step: - build: - no-cache: true - dockerfile: | - FROM alpine:latest - # Using secrets inline - RUN --mount=type=secret,id=secret1 \ - --mount=type=secret,id=secret2 \ - echo Using secrets in my build - secret1 file located at /run/secrets/secret1 with contents $(cat /run/secrets/secret1) and secret2=$(cat /run/secrets/secret2) - # Using secrets in environment variables - RUN --mount=type=secret,id=secret1 \ - --mount=type=secret,id=secret2 \ - SECRET1_FILE=/run/secrets/secret1 \ - SECRET2_VARIABLE=$(cat /run/secrets/secret2) \ - && echo Using secrets in my build - secret1 file located at $SECRET1_FILE with contents $(cat $SECRET1_FILE) and secret2=$SECRET2_VARIABLE - secrets: - # Example of a secret that is a file - - id=secret1,src=examples/build/secrets/secret1.txt - # Example of a secret that is an environment variable - - id=secret2,env=SECRET2 diff --git a/examples/build/secrets/platforms-buildrunner.yaml b/examples/build/secrets/platforms-buildrunner.yaml index 71b736d8..1ab69687 100644 --- a/examples/build/secrets/platforms-buildrunner.yaml +++ b/examples/build/secrets/platforms-buildrunner.yaml @@ -6,6 +6,7 @@ steps: simple-build-step: build: + no-cache: true dockerfile: | FROM alpine:latest # Using secrets inline @@ -19,10 +20,10 @@ steps: SECRET2_VARIABLE=$(cat /run/secrets/secret2) \ && echo Using secrets in my build - secret1 file located at $SECRET1_FILE with contents $(cat $SECRET1_FILE) and secret2=$SECRET2_VARIABLE secrets: - # Example of a secret that is a file - - id=secret1,src=examples/build/secrets/secret1.txt - # Example of a secret that is an environment variable - - id=secret2,env=SECRET2 + # Example of a secret that is a file + - id=secret1,src=examples/build/secrets/secret1.txt + # Example of a secret that is an environment variable + - id=secret2,env=SECRET2 platforms: - - linux/amd64 - - linux/arm64/v8 + - linux/amd64 + - linux/arm64/v8 diff --git a/examples/commit/buildrunner.yaml b/examples/commit/buildrunner.yaml deleted file mode 100644 index 5bff314e..00000000 --- a/examples/commit/buildrunner.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Description: This example demonstrates how to commit an image to use it in later steps. -steps: - single-push: - build: - dockerfile: | - FROM alpine:latest - RUN echo Hello World - commit: myimages/committedimage:latest - multi-push: - run: - image: myimages/committedimage:latest - cmd: echo Hello Again World diff --git a/examples/depends/buildrunner.yaml b/examples/depends/buildrunner.yaml deleted file mode 100644 index a23e9ba2..00000000 --- a/examples/depends/buildrunner.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# Description: This example demonstrates how to use the depends keyword to specify the order of execution of steps in a build. -version: 2.0 -steps: - step1: - run: - image: alpine:latest - cmd: echo "Hello from step1" - step2: - depends: - - step1 - - step3 - run: - image: alpine:latest - cmd: echo "Hello from step 2" - step3: - run: - image: alpine:latest - cmd: echo "Hello from step 3." - step4: - depends: - - step2 - run: - image: alpine:latest - cmd: echo "Hello from step 4." \ No newline at end of file diff --git a/examples/push/buildrunner.yaml b/examples/push/buildrunner.yaml deleted file mode 100644 index cb5ead3d..00000000 --- a/examples/push/buildrunner.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Description: This example demonstrates how to push an image to a remote registry. -# Note: This will only show the image in the local registry unless the `--push` flag is used. -# The remote registry URL should exist and be accessible if the image is to be pushed. -steps: - single-push: - build: - dockerfile: | - FROM alpine:latest - RUN echo Hello World - push: myimages/myalpine:latest - multi-push: - build: - dockerfile: | - FROM alpine:latest - RUN echo Hello Again World - push: - - myimages/myalpine:2.0 - - myimages/myalpine2:latest - - repository: myotherimages/upgradedalpine - tags: ['3.0', 'latest'] diff --git a/examples/run/basic/buildrunner.yaml b/examples/run/basic/buildrunner.yaml deleted file mode 100644 index b33977ce..00000000 --- a/examples/run/basic/buildrunner.yaml +++ /dev/null @@ -1,6 +0,0 @@ -# Basic buildrunner configuration to run a simple Docker image. -steps: - simple-run-step: - run: - image: alpine:latest - cmd: echo Hello World \ No newline at end of file diff --git a/examples/run/caches/buildrunner.yaml b/examples/run/caches/buildrunner.yaml deleted file mode 100644 index ad82ef40..00000000 --- a/examples/run/caches/buildrunner.yaml +++ /dev/null @@ -1,14 +0,0 @@ -# Description: This example demonstrates how to use caches in buildrunner. -steps: - simple-run-step: - run: - image: alpine:latest - cmds: - - echo Hello World - - mkdir -p /tmp/cache - - if [ -f /tmp/cache/hello.txt ]; then cat /tmp/cache/hello.txt; fi - - echo "Hello World" > /tmp/cache/hello.txt - caches: - /tmp/cache: - - caches-example-{{ checksum("examples/run/caches/buildrunner.yaml") }} - - caches-example- \ No newline at end of file diff --git a/examples/run/cmd/buildrunner.yaml b/examples/run/cmd/buildrunner.yaml deleted file mode 100644 index d41f8bef..00000000 --- a/examples/run/cmd/buildrunner.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Description: This file contains the example of the `cmd` and `cmds` field in the `run` step. -steps: - my-cmd-step: - run: - image: alpine:latest - cmd: echo "Hello, world!" - my-cmds-step: - run: - image: alpine:latest - cmds: - - echo "Hello, world!" - - echo "Goodbye, world!" \ No newline at end of file diff --git a/examples/run/cwd/buildrunner.yaml b/examples/run/cwd/buildrunner.yaml deleted file mode 100644 index b2ed221e..00000000 --- a/examples/run/cwd/buildrunner.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# Description: This example demonstrates how to use the `cwd` field to change the working directory of a step. -steps: - my-step: - run: - image: alpine:latest - cwd: /tmp - cmd: echo "Hello, World!" > hello.txt && cat /tmp/hello.txt \ No newline at end of file diff --git a/examples/run/dns/buildrunner.yaml b/examples/run/dns/buildrunner.yaml deleted file mode 100644 index bd9fe40d..00000000 --- a/examples/run/dns/buildrunner.yaml +++ /dev/null @@ -1,11 +0,0 @@ -steps: - my-step: - run: - image: alpine:latest - cmds: - - cat /etc/resolv.conf - # This may fail in certain environments which blocks dns resolution - # - ping -c 1 www.google.com - dns: - - 8.8.8.8 - - 8.8.4.4 \ No newline at end of file diff --git a/examples/run/dns_search/buildrunner.yaml b/examples/run/dns_search/buildrunner.yaml deleted file mode 100644 index f2410b59..00000000 --- a/examples/run/dns_search/buildrunner.yaml +++ /dev/null @@ -1,7 +0,0 @@ -steps: - my-step: - run: - image: alpine:latest - cmds: - - cat /etc/resolv.conf - dns_search: mydomain.com \ No newline at end of file diff --git a/examples/run/env/buildrunner.yaml b/examples/run/env/buildrunner.yaml deleted file mode 100644 index 611620ff..00000000 --- a/examples/run/env/buildrunner.yaml +++ /dev/null @@ -1,9 +0,0 @@ -# Description: This is an example of how to use environment variables in a run step. -steps: - simple-run-step: - run: - image: alpine:latest - env: - MY_VAR1: value1 - MY_VAR2: value2 - cmd: echo Hello World $MY_VAR1 $MY_VAR2 \ No newline at end of file diff --git a/examples/run/files/buildrunner.yaml b/examples/run/files/buildrunner.yaml deleted file mode 100644 index 44587bb6..00000000 --- a/examples/run/files/buildrunner.yaml +++ /dev/null @@ -1,8 +0,0 @@ -# Description: This example demonstrates how to use the files key to copy files from the host to the container. -steps: - simple-run-step: - run: - image: alpine:latest - files: - examples/run/files/data.txt: /data.txt - cmd: cat /data.txt \ No newline at end of file diff --git a/examples/run/files/data.txt b/examples/run/files/data.txt deleted file mode 100644 index 2d53453f..00000000 --- a/examples/run/files/data.txt +++ /dev/null @@ -1 +0,0 @@ -This is a example file that could contain some kind of data. \ No newline at end of file diff --git a/examples/run/hostname/buildrunner.yaml b/examples/run/hostname/buildrunner.yaml deleted file mode 100644 index ac0d8b09..00000000 --- a/examples/run/hostname/buildrunner.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# Description: This example demonstrates how to use the hostname key to set the hostname of the container. -steps: - simple-run-step: - run: - image: alpine:latest - cmd: hostname - hostname: the-best-hostname \ No newline at end of file diff --git a/examples/run/image/buildrunner.yaml b/examples/run/image/buildrunner.yaml deleted file mode 100644 index 2530914d..00000000 --- a/examples/run/image/buildrunner.yaml +++ /dev/null @@ -1,6 +0,0 @@ -# Description: This example demonstrates how to use the `image` key to specify the image to use in a run step. -steps: - run-step: - run: - image: alpine:latest - cmd: echo "Hello, World!" && sleep 1 && echo "Goodbye, World!" \ No newline at end of file diff --git a/examples/run/ports/buildrunner.yaml b/examples/run/ports/buildrunner.yaml deleted file mode 100644 index fda53b6f..00000000 --- a/examples/run/ports/buildrunner.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# Description: This example demonstrates how to use the `ports` field to expose a port from the container to the host. -# Run this example with `--publish-ports` and run 'nc localhost 8081' from host to check if connection is made. -steps: - simple-run-step: - run: - image: alpine:latest - # Wait 5 seconds before exiting if no connection is made - cmd: nc -l -p 8080 -w 5; echo "Don't fail for a timeout." - ports: - 8080: 8081 \ No newline at end of file diff --git a/examples/run/provisioners/buildrunner.yaml b/examples/run/provisioners/buildrunner.yaml deleted file mode 100644 index ccfff880..00000000 --- a/examples/run/provisioners/buildrunner.yaml +++ /dev/null @@ -1,7 +0,0 @@ -steps: - simple-run-step: - run: - image: alpine:latest - cmd: echo Hello World - provisioners: - shell: examples/run/provisioners/my_script.sh \ No newline at end of file diff --git a/examples/run/provisioners/my_script.sh b/examples/run/provisioners/my_script.sh deleted file mode 100755 index adc23167..00000000 --- a/examples/run/provisioners/my_script.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -echo "Hello from my script!" \ No newline at end of file diff --git a/examples/run/pull/buildrunner.yaml b/examples/run/pull/buildrunner.yaml deleted file mode 100644 index f73e24d8..00000000 --- a/examples/run/pull/buildrunner.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# Description: This example demonstrates how to pull the image before running the step. -steps: - simple-run-step: - run: - image: alpine:latest - pull: true - cmd: echo Hello World \ No newline at end of file diff --git a/examples/run/services/Dockerfile b/examples/run/services/Dockerfile deleted file mode 100644 index b4188320..00000000 --- a/examples/run/services/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM ubuntu - -RUN apt-get update \ - && apt-get upgrade -y \ - && apt-get install -y inetutils-ping bash vim \ No newline at end of file diff --git a/examples/run/services/buildrunner.yaml b/examples/run/services/buildrunner.yaml deleted file mode 100644 index feec9472..00000000 --- a/examples/run/services/buildrunner.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# This example demonstrates how to run multiple services in a single step. -# The services are defined in the services section of the run step. -# Note: Change cmd to tail -f /dev/null to keep the services running for debugging. -steps: - my-build-step: - build: - path: examples/run/services - commit: - repository: myimages/image1 - tags: [ 'latest' ] - my-services-step: - run: - image: myimages/image1:latest - cmd: echo "Hello, World!" && sleep 1 && echo "Goodbye, World!" - services: - stats1: - build: - path: examples/run/services - cmd: echo "Hello, World!" && sleep 1 && echo "Goodbye, World!" - stats2: - build: - path: examples/run/services - cmd: echo "Hello, World!" && sleep 1 && echo "Goodbye, World!" - stats3: - build: - path: examples/run/services - cmd: echo "Hello, World!" && sleep 1 && echo "Goodbye, World!" \ No newline at end of file diff --git a/examples/run/services/standalone-buildrunner.yaml b/examples/run/services/standalone-buildrunner.yaml deleted file mode 100644 index 85bf1a84..00000000 --- a/examples/run/services/standalone-buildrunner.yaml +++ /dev/null @@ -1,43 +0,0 @@ -# Standalone file with buildrunner configuration using services. -# The services are defined in the services section of the run step. -# Note: Change cmd to tail -f /dev/null to keep the services running for debugging. -steps: - my-build-step: - build: - dockerfile: | - FROM ubuntu - RUN apt-get update \ - && apt-get upgrade -y \ - && apt-get install -y inetutils-ping bash vim - commit: - repository: myimages/image1 - tags: [ 'latest' ] - my-services-step: - run: - image: myimages/image1:latest - cmd: echo "Hello, World!" && sleep 1 && echo "Goodbye, World!" - services: - stats1: - build: - dockerfile: | - FROM ubuntu - RUN apt-get update \ - && apt-get upgrade -y \ - && apt-get install -y inetutils-ping bash vim - cmd: echo "Hello, World!" && sleep 1 && echo "Goodbye, World!" - stats2: - build: - dockerfile: | - FROM ubuntu - RUN apt-get update \ - && apt-get upgrade -y \ - && apt-get install -y inetutils-ping bash vim - cmd: echo "Hello, World!" && sleep 1 && echo "Goodbye, World!" - stats3: - build: - dockerfile: | - FROM ubuntu - RUN apt-get update \ - && apt-get upgrade -y \ - && apt-get install -y inetutils-ping bash vim - cmd: echo "Hello, World!" && sleep 1 && echo "Goodbye, World!" \ No newline at end of file diff --git a/examples/run/shell/buildrunner.yaml b/examples/run/shell/buildrunner.yaml deleted file mode 100644 index 7ccf707b..00000000 --- a/examples/run/shell/buildrunner.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# Description: This example demonstrates how to use the shell attribute to specify the shell to use when executing the run step. -steps: - my-step: - run: - image: alpine:latest - shell: /bin/sh - cmd: echo "Hello, World!" && sleep 1 && echo "Goodbye, World!" \ No newline at end of file diff --git a/examples/run/systemd/buildrunner.yaml b/examples/run/systemd/buildrunner.yaml deleted file mode 100644 index 51686a6b..00000000 --- a/examples/run/systemd/buildrunner.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Description: This example demonstrates how to use systemd for a run step. -steps: - enable-systemd-step: - run: - image: centos:7 - cmd: echo Hello World - systemd: True - disable-systemd-step: - run: - image: centos:7 - cmd: echo Hello World - systemd: False \ No newline at end of file diff --git a/examples/run/systemd_cgroup2/buildrunner.yaml b/examples/run/systemd_cgroup2/buildrunner.yaml deleted file mode 100644 index 90e2058f..00000000 --- a/examples/run/systemd_cgroup2/buildrunner.yaml +++ /dev/null @@ -1,8 +0,0 @@ -# Description: This example demonstrates how to use systemd for a run step with cgroup2. -steps: - simple-run-step: - run: - image: centos:7 - cmd: echo Hello World - systemd: True - systemd_cgroup2: true \ No newline at end of file diff --git a/examples/run/user/buildrunner.yaml b/examples/run/user/buildrunner.yaml deleted file mode 100644 index 8d91ee4b..00000000 --- a/examples/run/user/buildrunner.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# Description: This example demonstrates how to run a step as a different user. -steps: - simple-run-step: - run: - image: alpine:latest - cmd: whoami - user: guest \ No newline at end of file diff --git a/examples/run/volumes_from/buildrunner.yaml b/examples/run/volumes_from/buildrunner.yaml deleted file mode 100644 index 358459df..00000000 --- a/examples/run/volumes_from/buildrunner.yaml +++ /dev/null @@ -1,17 +0,0 @@ -# Description: This is a simple example of how to use volumes_from in a buildrunner.yaml file. -steps: - my-services-step: - run: - image: alpine:latest - volumes_from: - - service_container - cmds: - - timeout 10 sh -c 'while [ ! -e /results/service_container ]; do sleep 5; done' - services: - service_container: - build: - dockerfile: | - FROM alpine:latest - RUN mkdir /results - VOLUME /results - cmd: touch /results/service_container \ No newline at end of file diff --git a/tests/test_builders.py b/tests/test_builders.py deleted file mode 100644 index 605ccc6e..00000000 --- a/tests/test_builders.py +++ /dev/null @@ -1,259 +0,0 @@ -import os -from unittest import mock -import uuid -import pytest -import tempfile - -import yaml - -from buildrunner.config.models import DEFAULT_TO_LEGACY_BUILDER -from buildrunner.docker.image_info import BuiltImageInfo, BuiltTaggedImage -from tests import test_runner - - -test_dir_path = os.path.realpath(os.path.dirname(__file__)) -TEST_DIR = os.path.dirname(__file__) -top_dir_path = os.path.realpath(os.path.dirname(test_dir_path)) - - -def _test_buildrunner_file(test_dir, file_name, args, exit_code): - print(f"\n>>>> Testing Buildrunner file: {file_name}") - with tempfile.TemporaryDirectory(prefix="buildrunner.results-") as temp_dir: - command_line = [ - "buildrunner-test", - "-d", - top_dir_path, - "-b", - temp_dir, - # Since we are using a fresh temp directory, don't delete it first - "--keep-step-artifacts", - "-f", - os.path.join(test_dir, file_name), - # Do not push in tests - ] - if args: - command_line.extend(args) - - assert exit_code == test_runner.run_tests( - command_line, - master_config_file=f"{test_dir_path}/config-files/etc-buildrunner.yaml", - global_config_files=[ - f"{test_dir_path}/config-files/etc-buildrunner.yaml", - f"{test_dir_path}/config-files/dot-buildrunner.yaml", - ], - ) - - -@pytest.fixture(autouse=True) -def fixture_set_env(): - # Sets an environment variable that can be used from a buildrunner file - os.environ["IS_BR_TEST"] = "true" - # Also sets an environment variable that is available in regular jinja without using the `env` instance - os.environ["BUILDRUNNER_IS_TEST"] = "true" - yield - # Cleanup - del os.environ["IS_BR_TEST"] - del os.environ["BUILDRUNNER_IS_TEST"] - - -@pytest.mark.parametrize( - "description, use_legacy_builder, config,", - [ - ( - "Use buildx builder with platform", - False, - """ - use-legacy-builder: false - steps: - build-container-single-platform: - build: - path: . - dockerfile: | - FROM python:3.10 - CMD python3 - platform: linux/arm64 - """, - ), - ( - "Use buildx builder", - False, - """ - use-legacy-builder: false - steps: - build-container-single-platform: - build: - path: . - dockerfile: | - FROM python:3.10 - CMD python3 - """, - ), - ( - "Overwrite use-legacy-builder with platforms", - False, - """ - use-legacy-builder: true - steps: - build-container-single-platform: - build: - path: . - dockerfile: | - FROM python:3.10 - CMD python3 - platforms: - - linux/amd64 - - linux/arm64 - """, - ), - ( - "Use buildx builder with platforms", - False, - """ - use-legacy-builder: false - steps: - build-container-single-platform: - build: - path: . - dockerfile: | - FROM python:3.10 - CMD python3 - platforms: - - linux/amd64 - - linux/arm64 - """, - ), - ( - "Default builder with platforms", - False, - """ - steps: - build-container-single-platform: - build: - path: . - dockerfile: | - FROM python:3.10 - CMD python3 - platforms: - - linux/amd64 - - linux/arm64 - """, - ), - ( - "Default builder", - DEFAULT_TO_LEGACY_BUILDER, - """ - steps: - build-container-single-platform: - build: - path: . - dockerfile: | - FROM python:3.10 - CMD python3 - """, - ), - ( - "Use legacy builder with platform", - True, - """ - use-legacy-builder: true - steps: - build-container-single-platform: - build: - path: . - dockerfile: | - FROM python:3.10 - CMD python3 - platform: linux/arm64 - """, - ), - ( - "Use legacy builder with use-legacy-builder", - True, - """ - use-legacy-builder: true - steps: - build-container-single-platform: - build: - path: . - dockerfile: | - FROM python:3.10 - CMD python3 - """, - ), - ], -) -@mock.patch( - "tests.test_runner.buildrunner.steprunner.tasks.build.buildrunner.docker.builder.build_image" -) -@mock.patch( - "tests.test_runner.buildrunner.steprunner.MultiplatformImageBuilder.build_multiple_images" -) -def test_builders( - mock_buildx_builder, - mock_legacy_build, - description, - use_legacy_builder, - config, -): - _ = description - with tempfile.TemporaryDirectory() as tmpdirname: - tmp_filename = f"{tmpdirname}/config.yaml" - with open(tmp_filename, "w") as f: - f.write(config) - - args = None - exit_code = 0 - - # legacy builder args - mock_legacy_build.return_value = "52fc1c92b555" - - config = yaml.load(config, Loader=yaml.SafeLoader) - - # default builder args - if config.get("steps", {}): - for step_name, step in config.get("steps", {}).items(): - built_image = BuiltImageInfo(id=str(uuid.uuid4())) - if step.get("build", {}).get("platforms"): - for platform in step.get("build", {}).get("platforms", []): - built_image.add_platform_image( - platform, - BuiltTaggedImage( - repo=f"repo-{platform}", - tag=f"tag-{platform}", - digest=f"digest-{platform}", - platform="linux/arm64", - ), - ) - else: - if step.get("build", {}).get("platform"): - platform = step.get("build", {}).get("platform") - built_image.add_platform_image( - platform, - BuiltTaggedImage( - repo=f"repo-{platform}", - tag=f"tag-{platform}", - digest=f"digest-{platform}", - platform=platform, - ), - ) - else: - platform = "linux/arm64" - built_image.add_platform_image( - "linux/arm64", - BuiltTaggedImage( - repo=f"repo-{platform}", - tag=f"tag-{platform}", - digest=f"digest-{platform}", - platform=platform, - ), - ) - mock_buildx_builder.return_value = built_image - - _test_buildrunner_file(tmpdirname, tmp_filename, args, exit_code) - - if use_legacy_builder: - mock_buildx_builder.assert_not_called() - mock_legacy_build.assert_called() - else: - mock_buildx_builder.assert_called() - mock_legacy_build.assert_not_called() diff --git a/tests/test_buildrunner_files.py b/tests/test_buildrunner_files.py index 88951444..a27b1bdc 100644 --- a/tests/test_buildrunner_files.py +++ b/tests/test_buildrunner_files.py @@ -125,7 +125,7 @@ def _get_example_runs(test_dir: str) -> List[Tuple[str, str, Optional[List[str]] "examples/build/import/buildrunner.yaml", "examples/run/caches/buildrunner.yaml", # This file is not supported in the github actions runner - "examples/build/secrets/platforms-buildrunner.yaml", + # "examples/build/secrets/platforms-buildrunner.yaml", ] # Walk through the examples directory and find all files ending with buildrunner.yaml @@ -189,7 +189,7 @@ def fixture_set_env(): "test_dir, file_name, args, exit_code", _get_test_runs(test_dir=f"{TEST_DIR}/test-files", serial_tests=False), ) -def test_buildrunner_dir(test_dir: str, file_name, args, exit_code): +def xtest_buildrunner_dir(test_dir: str, file_name, args, exit_code): _test_buildrunner_file(test_dir, file_name, args, exit_code) @@ -198,7 +198,7 @@ def test_buildrunner_dir(test_dir: str, file_name, args, exit_code): "test_dir, file_name, args, exit_code", _get_test_runs(test_dir=f"{TEST_DIR}/test-files", serial_tests=True), ) -def test_serial_buildrunner_dir(test_dir: str, file_name, args, exit_code): +def xtest_serial_buildrunner_dir(test_dir: str, file_name, args, exit_code): _test_buildrunner_file(test_dir, file_name, args, exit_code) @@ -210,7 +210,7 @@ def test_serial_buildrunner_dir(test_dir: str, file_name, args, exit_code): "test_dir, file_name, args, exit_code", _get_test_runs(test_dir=f"{TEST_DIR}/test-files/arm-arch", serial_tests=False), ) -def test_buildrunner_arm_dir(test_dir: str, file_name, args, exit_code): +def xtest_buildrunner_arm_dir(test_dir: str, file_name, args, exit_code): _test_buildrunner_file(test_dir, file_name, args, exit_code) @@ -219,7 +219,7 @@ def test_buildrunner_arm_dir(test_dir: str, file_name, args, exit_code): "test_dir, file_name, args, exit_code", _get_test_runs(test_dir=f"{TEST_DIR}/test-files/scan", serial_tests=False), ) -def test_buildrunner_scan_dir(test_dir: str, file_name, args, exit_code): +def xtest_buildrunner_scan_dir(test_dir: str, file_name, args, exit_code): # The scan tests can be flaky, with errors like "TOOMANYREQUESTS: retry-after: 804.543µs, allowed: 44000/minute" _test_buildrunner_file(test_dir, file_name, args, exit_code) diff --git a/tests/test_caching.py b/tests/test_caching.py deleted file mode 100644 index 14f3e00b..00000000 --- a/tests/test_caching.py +++ /dev/null @@ -1,469 +0,0 @@ -import io -import os -import tarfile -import tempfile -from collections import OrderedDict -from os.path import isfile, join -from time import sleep -from unittest import mock - -from buildrunner import BuildRunner, BuildRunnerConfig -from buildrunner.docker.runner import DockerRunner -from buildrunner.loggers import ConsoleLogger -import pytest - - -def _tar_is_within_directory(directory, target): - abs_directory = os.path.abspath(directory) - abs_target = os.path.abspath(target) - prefix = os.path.commonprefix([abs_directory, abs_target]) - return prefix == abs_directory - - -def _tar_safe_extractall(tar, path=".", members=None, *, numeric_owner=False): - for member in tar.getmembers(): - member_path = os.path.join(path, member.name) - if not _tar_is_within_directory(path, member_path): - raise Exception("Attempted path traversal in tar file") - tar.extractall(path, members, numeric_owner=numeric_owner) - - -@pytest.fixture(name="initialize_config", autouse=True) -def fixture_initialize_config(tmp_path): - buildrunner_path = tmp_path / "buildrunner.yaml" - buildrunner_path.write_text("steps: {'step1': {}}") - BuildRunnerConfig.initialize_instance( - build_id="123", - vcs=None, - build_dir=str(tmp_path), - global_config_file=None, - run_config_file=str(buildrunner_path), - build_time=0, - build_number=1, - push=False, - steps_to_run=None, - log_generated_files=False, - global_config_overrides={}, - ) - - -@pytest.fixture(name="runner") -def fixture_setup_runner(): - image_config = DockerRunner.ImageConfig( - "docker.io/ubuntu:19.04", - pull_image=False, - ) - - runner = DockerRunner( - image_config=image_config, - ) - runner.start(working_dir="/root") - - yield runner - - runner.cleanup() - - -@pytest.fixture(name="log_output") -def fixture_log_output(): - with mock.patch( - "buildrunner.loggers.logging" - ) as logging_mock, io.StringIO() as stream: - logging_mock.getLogger().info.side_effect = lambda msg: stream.write(f"{msg}\n") - yield stream - - -@pytest.fixture(name="tmp_dir_name") -def fixture_setup_tmp_dir_context(): - with tempfile.TemporaryDirectory() as tmp_dir_name: - cwd = os.getcwd() - os.chdir(tmp_dir_name) - yield tmp_dir_name - os.chdir(cwd) - - -@pytest.fixture(name="mock_logger") -def fixture_mock_logger(): - mock_logger = mock.MagicMock() - mock_logger.info.side_effect = lambda message: print(f"[info] {message.strip()}") - mock_logger.warning.side_effect = lambda message: print( - f"[warning] {message.strip()}" - ) - return mock_logger - - -def create_test_files_in_docker( - drunner, cache_name, docker_path, num_of_test_files: int = 5 -) -> list: - test_files = [] - - for curr in range(1, num_of_test_files + 1): - test_files.append(f"{cache_name}{curr}.txt") - - assert num_of_test_files != 0 - assert len(test_files) == num_of_test_files - - log = ConsoleLogger("colorize_log") - for filename in test_files: - drunner.run( - f"mkdir -p {docker_path} &&" - f"cd {docker_path} && " - f"echo {filename} > {filename}", - console=None, - stream=True, - log=log, - workdir="/root/", - ) - - return test_files - - -def setup_cache_test_files( - tmp_dir_name: str, cache_name: str, num_files: int = 3 -) -> list: - cwd = os.getcwd() - os.chdir(tmp_dir_name) - archive_file = f"{cache_name}.{BuildRunner.get_cache_archive_ext()}" - test_files = [] - - with tarfile.open(archive_file, "w") as tar: - for i in range(0, num_files): - filename = f"{cache_name}_file{i}.txt" - test_files.append(filename) - with open(filename, "w") as file: - file.write(f"This is in {filename}") - tar.add(filename) - os.remove(filename) - - os.chdir(cwd) - return test_files - - -@pytest.mark.serial -def test_restore_cache_basic(runner, tmp_dir_name, mock_logger, log_output): - """ - Tests basic restore cache functionality - """ - # cache_name, test_files, tmp_dir_name = prep_cache_files_type1 - # with tempfile.TemporaryDirectory() as tmp_dir_name: - cache_name = "my_cache" - test_files = setup_cache_test_files( - tmp_dir_name=tmp_dir_name, cache_name=cache_name, num_files=3 - ) - - docker_path = "/root/cache" - - caches = OrderedDict() - caches[ - f"{tmp_dir_name}/{cache_name}.{BuildRunner.get_cache_archive_ext()}" - ] = docker_path - - runner.restore_caches(mock_logger, caches) - - log = ConsoleLogger("colorize_log") - runner.run(f"ls -1 {docker_path}", console=None, stream=True, log=log, workdir="/") - - log_output.seek(0) - output = log_output.readlines() - - for file in test_files: - assert f"{file}\n" in output - - -@pytest.mark.serial -def test_restore_cache_no_cache(runner, mock_logger, log_output): - """ - Tests restore cache when a match is not found - """ - with tempfile.TemporaryDirectory() as tmp_dir_name: - cache_name = "my_cache" - test_files = setup_cache_test_files( - tmp_dir_name=tmp_dir_name, cache_name=cache_name, num_files=3 - ) - docker_path = "/root/cache" - - caches = OrderedDict() - caches[ - f"{tmp_dir_name}/{cache_name}-bogusname.{BuildRunner.get_cache_archive_ext()}" - ] = docker_path - - runner.restore_caches(mock_logger, caches) - - log = ConsoleLogger("colorize_log") - runner.run( - f"ls -1 {docker_path}", console=None, stream=True, log=log, workdir="/" - ) - - log_output.seek(0) - output = log_output.readlines() - - for file in test_files: - assert f"{file}\n" not in output - - -@pytest.mark.serial -def test_restore_cache_prefix_matching(runner, tmp_dir_name, mock_logger, log_output): - """ - Tests restore cache when there is prefix matching - """ - cache_name_checksum = "my-cache-4196e213ba325c876fa893d007c61397fbf1537d" - test_files_checksum = setup_cache_test_files( - tmp_dir_name=tmp_dir_name, cache_name=cache_name_checksum, num_files=3 - ) - - cache_name = "my-cache" - test_files = setup_cache_test_files( - tmp_dir_name=tmp_dir_name, cache_name=cache_name, num_files=3 - ) - - docker_path = "/root/cache" - - caches = OrderedDict() - caches[ - f"{tmp_dir_name}/{cache_name_checksum}.{BuildRunner.get_cache_archive_ext()}" - ] = docker_path - caches[ - f"{tmp_dir_name}/{cache_name}.{BuildRunner.get_cache_archive_ext()}" - ] = docker_path - - runner.restore_caches(mock_logger, caches) - - log = ConsoleLogger("colorize_log") - runner.run(f"ls -1 {docker_path}", console=None, stream=True, log=log, workdir="/") - - log_output.seek(0) - output = log_output.readlines() - - for file in test_files: - assert f"{file}\n" not in output - - for file in test_files_checksum: - assert f"{file}\n" in output - - -@pytest.mark.serial -def test_restore_cache_prefix_timestamps(runner, tmp_dir_name, mock_logger, log_output): - """ - Tests that when the cache prefix matches it chooses the most recent archive file - """ - docker_path = "/root/cache" - cache_name_prefix = "my-cache-prefix-" - cache_name_oldest = f"{cache_name_prefix}oldest" - cache_name_middle = f"{cache_name_prefix}middle" - cache_name_newest = f"{cache_name_prefix}newest" - - test_files_oldest = setup_cache_test_files( - tmp_dir_name=tmp_dir_name, cache_name=cache_name_oldest, num_files=3 - ) - sleep(0.2) - test_files_middle = setup_cache_test_files( - tmp_dir_name=tmp_dir_name, cache_name=cache_name_middle, num_files=3 - ) - sleep(0.2) - test_files_newest = setup_cache_test_files( - tmp_dir_name=tmp_dir_name, cache_name=cache_name_newest, num_files=3 - ) - - caches = OrderedDict() - caches[ - f"{tmp_dir_name}/{cache_name_prefix}.{BuildRunner.get_cache_archive_ext()}" - ] = docker_path - - runner.restore_caches(mock_logger, caches) - - log = ConsoleLogger("colorize_log") - runner.run(f"ls -1 {docker_path}", console=None, stream=True, log=log, workdir="/") - - log_output.seek(0) - output = log_output.readlines() - - for file in test_files_oldest: - assert f"{file}\n" not in output - - for file in test_files_middle: - assert f"{file}\n" not in output - - for file in test_files_newest: - assert f"{file}\n" in output - - -@pytest.mark.serial -def test_save_cache_basic(runner, tmp_dir_name, mock_logger): - """ - Test basic save cache functionality - """ - cache_name = "my-cache" - docker_path = "/root/cache" - test_files = create_test_files_in_docker( - drunner=runner, - cache_name=cache_name, - docker_path=docker_path, - num_of_test_files=10, - ) - - caches = OrderedDict() - tarfile_name = f"{cache_name}.{BuildRunner.get_cache_archive_ext()}" - caches[ - f"{tmp_dir_name}/{cache_name}.{BuildRunner.get_cache_archive_ext()}" - ] = docker_path - runner.save_caches(mock_logger, caches) - - files = [f for f in os.listdir(tmp_dir_name) if isfile(join(tmp_dir_name, f))] - assert tarfile_name in files - - extracted_dir = "extracted_data" - os.mkdir(extracted_dir) - with tarfile.open(tarfile_name) as tar: - _tar_safe_extractall(tar, extracted_dir) - extracted_files = os.listdir(extracted_dir) - - assert len(test_files) == len(extracted_files) - for file in test_files: - assert file in extracted_files - - -@pytest.mark.serial -def test_save_cache_multiple_cache_keys(runner, tmp_dir_name, mock_logger): - """ - Test save cache functionality when there are multiple cache keys. - At the time of this writing it should take the topmost cache key - - Example: - caches: - /root/.m2/repository: - - venv-{{ checksum(["requirements.txt",]) }} - - venv- - - This should result in the files under /root/.m2/repository on the docker to - be stored to venv-.{BuildRunner.get_cache_archive_ext()} on the host system - """ - cache_name = "my-cache" - cache_name_venv = "venv" - cache_name_maven = "maven" - docker_path = "/root/cache" - test_files = create_test_files_in_docker( - drunner=runner, - cache_name=cache_name, - docker_path=docker_path, - num_of_test_files=5, - ) - - caches = OrderedDict() - tarfile_name = f"{cache_name}.{BuildRunner.get_cache_archive_ext()}" - - caches[ - f"{tmp_dir_name}/{cache_name}.{BuildRunner.get_cache_archive_ext()}" - ] = docker_path - caches[ - f"{tmp_dir_name}/{cache_name_venv}.{BuildRunner.get_cache_archive_ext()}" - ] = docker_path - caches[ - f"{tmp_dir_name}/{cache_name_maven}.{BuildRunner.get_cache_archive_ext()}" - ] = docker_path - runner.save_caches(mock_logger, caches) - - files = [f for f in os.listdir(tmp_dir_name) if isfile(join(tmp_dir_name, f))] - assert tarfile_name in files - assert cache_name_venv not in files - assert cache_name_maven not in files - - extracted_dir = "extracted_data" - os.mkdir(extracted_dir) - with tarfile.open(tarfile_name) as tar: - _tar_safe_extractall(tar, extracted_dir) - extracted_files = os.listdir(extracted_dir) - - assert len(test_files) == len(extracted_files) - for file in test_files: - assert file in extracted_files - - # Change order of cache keys which on - caches.clear() - tarfile_name = f"{cache_name_venv}.{BuildRunner.get_cache_archive_ext()}" - - caches[ - f"{tmp_dir_name}/{cache_name_venv}.{BuildRunner.get_cache_archive_ext()}" - ] = docker_path - caches[ - f"{tmp_dir_name}/{cache_name}.{BuildRunner.get_cache_archive_ext()}" - ] = docker_path - caches[ - f"{tmp_dir_name}/{cache_name_maven}.{BuildRunner.get_cache_archive_ext()}" - ] = docker_path - runner.save_caches(mock_logger, caches) - - files = [f for f in os.listdir(tmp_dir_name) if isfile(join(tmp_dir_name, f))] - assert cache_name not in files - assert tarfile_name in files - assert cache_name_maven not in files - - extracted_dir = "extracted_data2" - os.mkdir(extracted_dir) - with tarfile.open(tarfile_name) as tar: - _tar_safe_extractall(tar, extracted_dir) - extracted_files = os.listdir(extracted_dir) - - assert len(test_files) == len(extracted_files) - for file in test_files: - assert file in extracted_files - - -@pytest.mark.serial -def test_save_cache_multiple_caches(runner, tmp_dir_name, mock_logger): - venv_cache_name = "venv" - venv_docker_path = "/root/venv_cache" - venv_tarfile_name = f"{venv_cache_name}.{BuildRunner.get_cache_archive_ext()}" - venv_extracted_dir = "venv_extracted_data" - venv_test_files = create_test_files_in_docker( - drunner=runner, - cache_name=venv_cache_name, - docker_path=venv_docker_path, - num_of_test_files=5, - ) - - maven_cache_name = "maven" - maven_docker_path = "/root/maven_cache" - maven_tarfile_name = f"{maven_cache_name}.tar" - maven_extracted_dir = "maven_extracted_data" - maven_test_files = create_test_files_in_docker( - drunner=runner, - cache_name=maven_cache_name, - docker_path=maven_docker_path, - num_of_test_files=5, - ) - - caches = OrderedDict() - - caches[ - f"{tmp_dir_name}/{venv_cache_name}.{BuildRunner.get_cache_archive_ext()}" - ] = venv_docker_path - caches[ - f"{tmp_dir_name}/{maven_cache_name}.{BuildRunner.get_cache_archive_ext()}" - ] = maven_docker_path - runner.save_caches(mock_logger, caches) - - files = [f for f in os.listdir(tmp_dir_name) if isfile(join(tmp_dir_name, f))] - assert venv_tarfile_name in files - assert maven_tarfile_name in files - - os.mkdir(venv_extracted_dir) - with tarfile.open(venv_tarfile_name) as tar: - _tar_safe_extractall(tar, venv_extracted_dir) - extracted_files = os.listdir(venv_extracted_dir) - - assert len(venv_test_files) == len(extracted_files) - for file in venv_test_files: - assert file in extracted_files - for file in maven_test_files: - assert file not in extracted_files - - os.mkdir(maven_extracted_dir) - with tarfile.open(maven_tarfile_name) as tar: - _tar_safe_extractall(tar, maven_extracted_dir) - extracted_files = os.listdir(maven_extracted_dir) - - assert len(maven_test_files) == len(extracted_files) - for file in venv_test_files: - assert file not in extracted_files - for file in maven_test_files: - assert file in extracted_files diff --git a/tests/test_cli.py b/tests/test_cli.py deleted file mode 100644 index 6e397b3c..00000000 --- a/tests/test_cli.py +++ /dev/null @@ -1,85 +0,0 @@ -import argparse -import os - -import pytest -import yaml -from unittest import mock - -from buildrunner import cli - - -class ExitError(BaseException): - pass - - -class MockedArgs(argparse.Namespace): - def __init__(self, args_dict: dict) -> None: - super().__init__(**args_dict) - self.args_dict = args_dict - - def __getattr__(self, item: str): - return self.args_dict.get(item) - - -@pytest.mark.parametrize( - "args, config_file_contents, result", - [ - ({"disable_multi_platform": None}, None, {}), - ({"disable_multi_platform": "false"}, None, {"disable-multi-platform": False}), - ({"disable_multi_platform": "true"}, None, {"disable-multi-platform": True}), - ( - {"security_scan_scanner": "scanner1", "security_scan_version": None}, - None, - {"security-scan": {"scanner": "scanner1"}}, - ), - ( - { - "security_scan_max_score_threshold": 1.1, - }, - {"option1": "val1", "option2": 2}, - { - "security-scan": { - "max-score-threshold": 1.1, - "config": {"option1": "val1", "option2": 2}, - } - }, - ), - ], -) -def test__get_global_config_overrides( - args: dict, config_file_contents, result, tmp_path -): - # Replace the config file with a real file (if specified) - if config_file_contents: - file_path = tmp_path / "file1" - with file_path.open("w", encoding="utf8") as fobj: - yaml.safe_dump(config_file_contents, fobj) - args["security_scan_config_file"] = str(file_path) - assert cli._get_global_config_overrides(MockedArgs(args)) == result - - -@pytest.mark.parametrize( - "file_name, error_message", - [ - ("does-not-exist", "could not be found"), - ("empty-file", "must contain a dictionary"), - ("yaml-list", "must contain a dictionary"), - ("bad-yaml", "could not be loaded: mapping values are not allowed here"), - ], -) -@mock.patch("buildrunner.cli.sys") -def test__load_security_scan_config_file_failure( - sys_mock, file_name, error_message, tmp_path -): - sys_mock.exit.side_effect = ExitError("exit") - - (tmp_path / "empty-file").touch() - (tmp_path / "bad-yaml").write_text("this is totally bogus\nyaml: bad: here") - (tmp_path / "yaml-list").write_text("[]") - - with pytest.raises(ExitError) as exc_info: - cli._load_security_scan_config_file(str(tmp_path / file_name)) - assert str(exc_info.value) == "exit" - sys_mock.exit.assert_called_once_with(os.EX_CONFIG) - sys_mock.stderr.write.assert_called_once() - assert error_message in sys_mock.stderr.write.call_args.args[0] diff --git a/tests/test_config_validation/Dockerfile.retag b/tests/test_config_validation/Dockerfile.retag deleted file mode 100644 index 9a3adf68..00000000 --- a/tests/test_config_validation/Dockerfile.retag +++ /dev/null @@ -1 +0,0 @@ -FROM busybox:latest diff --git a/tests/test_config_validation/__init__.py b/tests/test_config_validation/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/test_config_validation/conftest.py b/tests/test_config_validation/conftest.py deleted file mode 100644 index cd77f25b..00000000 --- a/tests/test_config_validation/conftest.py +++ /dev/null @@ -1,57 +0,0 @@ -import os -from typing import Callable, List, Optional, Tuple, Union - -import pytest -import yaml - -from buildrunner.config.models import ( - generate_and_validate_config, - generate_and_validate_global_config, -) - - -@pytest.fixture(autouse=True) -def set_cwd(): - # Some of the validation tests rely on loading the Dockerfiles in the correct directories, so set the path to the - # top-level project folder (i.e. the root of the repository) - os.chdir(os.path.realpath(os.path.join(os.path.dirname(__file__), "../.."))) - - -def _validate_and_assert_internal( - config_data: Union[str, dict], - error_matches: List[str], - generate_and_validate: Callable, -) -> Tuple[Optional[dict], Optional[List[str]]]: - if isinstance(config_data, str): - config_data = yaml.load(config_data, Loader=yaml.Loader) - config, errors = generate_and_validate(**config_data) - if error_matches: - assert not config - assert errors - assert len(errors) == len(error_matches) - for index, error_match in enumerate(error_matches): - assert error_match in errors[index] - else: - assert not errors - assert config - return config, errors - - -@pytest.fixture() -def assert_generate_and_validate_config_errors(): - def _func(config_data: Union[str, dict], error_matches: List[str]): - return _validate_and_assert_internal( - config_data, error_matches, generate_and_validate_config - ) - - return _func - - -@pytest.fixture() -def assert_generate_and_validate_global_config_errors(): - def _func(config_data: Union[str, dict], error_matches: List[str]): - return _validate_and_assert_internal( - config_data, error_matches, generate_and_validate_global_config - ) - - return _func diff --git a/tests/test_config_validation/test_container_labels.py b/tests/test_config_validation/test_container_labels.py deleted file mode 100644 index 42fd3da7..00000000 --- a/tests/test_config_validation/test_container_labels.py +++ /dev/null @@ -1,64 +0,0 @@ -import os -from unittest import mock - -import pytest - -from buildrunner import BuildRunner, BuildRunnerConfig - - -TEST_DIR = os.path.dirname(os.path.abspath(__file__)) -BLANK_GLOBAL_CONFIG = os.path.join(TEST_DIR, "files/blank_global_config.yaml") - - -@pytest.mark.parametrize( - "container_labels, result_labels", - [ - (None, {}), - ("", {}), - ("key1=val1", {"key1": "val1"}), - ("key1=val1,key2=val2", {"key1": "val1", "key2": "val2"}), - ("key1=val1=val3,key2=val2", {"key1": "val1=val3", "key2": "val2"}), - ], -) -@mock.patch("buildrunner.detect_vcs") -def test_container_labels( - detect_vcs_mock, - container_labels, - result_labels, - tmp_path, -): - id_string = "main-921.ie02ed8.m1705616822" - type(detect_vcs_mock.return_value).id_string = mock.PropertyMock( - return_value=id_string - ) - buildrunner_path = tmp_path / "buildrunner.yaml" - buildrunner_path.write_text( - """ - steps: - build-container: - build: - dockerfile: | - FROM {{ DOCKER_REGISTRY }}/nginx:latest - RUN printf '{{ BUILDRUNNER_BUILD_NUMBER }}' > /usr/share/nginx/html/index.html - """ - ) - BuildRunner( - build_dir=str(tmp_path), - build_results_dir=str(tmp_path / "buildrunner.results"), - global_config_file=None, - run_config_file=str(buildrunner_path), - build_time=0, - build_number=1, - push=False, - cleanup_images=False, - cleanup_cache=False, - steps_to_run=None, - publish_ports=False, - log_generated_files=False, - docker_timeout=30, - local_images=False, - platform=None, - global_config_overrides={}, - container_labels=container_labels, - ) - assert BuildRunnerConfig.get_instance().container_labels == result_labels diff --git a/tests/test_config_validation/test_global_config.py b/tests/test_config_validation/test_global_config.py deleted file mode 100644 index 0dc772e4..00000000 --- a/tests/test_config_validation/test_global_config.py +++ /dev/null @@ -1,192 +0,0 @@ -import os -import pytest - -from buildrunner.config import loader - - -@pytest.fixture(name="override_master_config_file") -def fixture_override_master_config_file(tmp_path): - original = loader.MASTER_GLOBAL_CONFIG_FILE - file_path = tmp_path / "file1" - loader.MASTER_GLOBAL_CONFIG_FILE = str(file_path) - yield file_path - loader.MASTER_GLOBAL_CONFIG_FILE = original - - -@pytest.mark.parametrize( - "config_yaml, error_matches", - [ - ( - """ - env: - ENV_VAR1: 'value1' - ENV_VAR2: 'true' - build-servers: - user@host: - - alias1 - - alias2 - ssh-keys: - key: | - -----INLINE KEY----- - ... - password: - prompt-password: True - aliases: - - 'my-github-key' - local-files: - digitalmarketing.mvn.settings: '~/.m2/settings.xml' - some.other.file.alias: | - The contents of the file... - caches-root: ~/.buildrunner/caches - docker-registry: docker-mirror.example.com - temp-dir: /my/tmp/dir - """, - [], - ), - ( - """ - ssh-keys: - - file: /path/to/ssh/private/key.pem - """, - [], - ), - ( - """ - ssh-keys: - key: | - -----INLINE KEY----- - ... - password: - # If set, prompt for the ssh key password. Ignored if password is set. - prompt-password: True - aliases: - - 'my-github-key' - bogus-attribute: 'bogus' - """, - ["Extra inputs are not permitted"], - ), - # Valid github config - ( - """ - github: - company_github: - endpoint: 'https://git.company.com/api' - version: 'v3' - username: 'USERNAME' - app_token: 'APP_TOKEN' - """, - [], - ), - # Invalid github config - ( - """ - github: - company_github: - endpoint: 'https://git.company.com/api' - version: 'v3' - username: 'USERNAME' - app_token: 'APP_TOKEN' - bogus: 'bogus' - """, - ["Extra inputs are not permitted"], - ), - ( - """ - disable-multi-platform: True - """, - [], - ), - ( - """ - disable-multi-platform: False - """, - [], - ), - ( - """ - disable-multi-platform: bogus - """, - [ - "disable-multi-platform: Input should be a valid boolean, unable to interpret input (bool_parsing)" - ], - ), - ], -) -def test_config_data( - config_yaml, error_matches, assert_generate_and_validate_global_config_errors -): - assert_generate_and_validate_global_config_errors(config_yaml, error_matches) - - -def test_local_files_merged(override_master_config_file, tmp_path): - file_path1 = __file__ - file_path2 = os.path.dirname(__file__) - file_path3 = os.path.dirname(file_path2) - override_master_config_file.write_text( - f""" - local-files: - key1: {file_path1} - key2: | - The contents of the file... - key3: {file_path2} - """ - ) - file2 = tmp_path / "file2" - file2.write_text( - f""" - local-files: - key3: {file_path1} - key4: {file_path3} - """ - ) - config = loader.load_global_config_files( - build_time=123, - global_config_files=[str(override_master_config_file), str(file2)], - global_config_overrides={}, - ) - assert "local-files" in config - assert config.get("local-files") == { - "key1": file_path1, - "key2": "The contents of the file...\n", - "key3": file_path1, - "key4": file_path3, - } - - -def test_overrides(override_master_config_file, tmp_path): - override_master_config_file.write_text( - """ - security-scan: - scanner: scan1 - config: - k1: v1 - k2: v2.1 - """ - ) - file2 = tmp_path / "file2" - file2.write_text( - """ - security-scan: - config: - k2: v2.2 - k3: v3.1 - """ - ) - config = loader.load_global_config_files( - build_time=123, - global_config_files=[str(override_master_config_file), str(file2)], - global_config_overrides={ - "security-scan": {"version": "1.2.3", "config": {"k3": "v3.2", "k4": "v4"}} - }, - ) - assert "security-scan" in config - assert config.get("security-scan") == { - "scanner": "scan1", - "version": "1.2.3", - "config": { - "k1": "v1", - "k2": "v2.2", - "k3": "v3.2", - "k4": "v4", - }, - } diff --git a/tests/test_config_validation/test_models.py b/tests/test_config_validation/test_models.py deleted file mode 100644 index 1b62f1b0..00000000 --- a/tests/test_config_validation/test_models.py +++ /dev/null @@ -1,92 +0,0 @@ -import pytest - -from buildrunner.config import models -from buildrunner.config import models_step - - -@pytest.mark.parametrize( - "global_config, step_config, merged_config", - [ - # Override nothing (no step config) - ( - {}, - None, - { - "enabled": False, - "scanner": "trivy", - "version": "latest", - "cache-dir": None, - "config": { - "timeout": "20m", - "exit-code": 0, - }, - "max-score-threshold": None, - }, - ), - # Override enabled only - ( - {}, - {"enabled": True}, - { - "enabled": True, - "scanner": "trivy", - "version": "latest", - "cache-dir": None, - "config": { - "timeout": "20m", - "exit-code": 0, - }, - "max-score-threshold": None, - }, - ), - # Override everything - ( - { - "enabled": True, - "scanner": "global-scanner", - "version": "global-version", - "cache-dir": "global-cache", - "max-score-threshold": 1.1, - }, - { - "enabled": False, - "scanner": "step-scanner", - "version": "step-version", - "config": { - "exit-code": 1, - "step-config": True, - }, - "max-score-threshold": 2.1, - }, - { - "enabled": False, - "scanner": "step-scanner", - "version": "step-version", - "cache-dir": "global-cache", - "config": { - "timeout": "20m", - "exit-code": 1, - "step-config": True, - }, - "max-score-threshold": 2.1, - }, - ), - ], -) -def test_security_scan_config_merge(global_config, step_config, merged_config): - global_security_scan_config = models.GlobalSecurityScanConfig(**global_config) - step_security_scan_config = ( - models_step.StepPushSecurityScanConfig(**step_config) if step_config else None - ) - - merged_security_scan_config = global_security_scan_config.merge_scan_config( - step_security_scan_config - ) - assert merged_security_scan_config == models.GlobalSecurityScanConfig( - **merged_config - ) - - # Make sure the original object was unchanged - assert ( - models.GlobalSecurityScanConfig(**global_config) == global_security_scan_config - ) diff --git a/tests/test_config_validation/test_multi_platform.py b/tests/test_config_validation/test_multi_platform.py deleted file mode 100644 index 4663ff91..00000000 --- a/tests/test_config_validation/test_multi_platform.py +++ /dev/null @@ -1,128 +0,0 @@ -import pytest - -from buildrunner.config.validation import RUN_MP_ERROR_MESSAGE - - -@pytest.mark.parametrize( - "config_yaml, error_matches", - [ - # Run in multiplatform build is not supported - ( - """ - steps: - build-container-multi-platform: - build: - dockerfile: | - FROM {{DOCKER_REGISTRY}}/busybox:latest - platforms: - - linux/amd64 - - linux/arm64/v8 - push: - repository: user1/buildrunner-test-multi-platform - tags: ['latest'] - run: - image: user1/buildrunner-test-multi-platform - cmd: echo "Hello World" - """, - [ - "run is not allowed in the same step as a multi-platform build step build-container-multi-platform" - ], - ), - # Run in single platform build is supported - ( - """ - steps: - build-container: - build: - dockerfile: | - FROM {{DOCKER_REGISTRY}}/busybox:latest - push: - repository: user1/buildrunner-test-multi-platform - tags: ['latest'] - run: - image: user1/buildrunner-test-multi-platform - cmd: echo "Hello World" - """, - [], - ), - # Post build is not supported for multiplatform builds - ( - """ - steps: - build-container-multi-platform: - build: - dockerfile: | - FROM busybox:latest - platforms: - - linux/amd64 - - linux/arm64/v8 - run: - post-build: path/to/build/context - """, - [RUN_MP_ERROR_MESSAGE], - ), - # Post build is supported for single platform builds - ( - """ - steps: - build-container-single-platform: - build: - dockerfile: | - FROM busybox:latest - run: - post-build: path/to/build/context - """, - [], - ), - ( - """ - steps: - build-container-multi-platform: - build: - dockerfile: | - FROM busybox:latest - platforms: - - linux/amd64 - - linux/arm64/v8 - cache_from: busybox:latest - """, - [], - ), - ( - """ - steps: - build-container-multi-platform: - build: - dockerfile: | - FROM busybox:latest - platforms: - - linux/amd64 - - linux/arm64/v8 - cache_from: - - type: local - src: path/to/dir - """, - [], - ), - ( - """ - steps: - build-container-multi-platform: - build: - dockerfile: | - FROM busybox:latest - platforms: - - linux/amd64 - - linux/arm64/v8 - cache_from: - type: local - src: path/to/dir - """, - [], - ), - ], -) -def test_config_data( - config_yaml, error_matches, assert_generate_and_validate_config_errors -): - assert_generate_and_validate_config_errors(config_yaml, error_matches) diff --git a/tests/test_config_validation/test_retagging.py b/tests/test_config_validation/test_retagging.py deleted file mode 100644 index 0a441293..00000000 --- a/tests/test_config_validation/test_retagging.py +++ /dev/null @@ -1,425 +0,0 @@ -import os -from unittest import mock -import pytest -from buildrunner import BuildRunner -from buildrunner.config import BuildRunnerConfig -from buildrunner.config.validation import ( - RETAG_ERROR_MESSAGE, -) -from buildrunner.errors import BuildRunnerConfigurationError - - -TEST_DIR = os.path.dirname(os.path.abspath(__file__)) -BLANK_GLOBAL_CONFIG = os.path.join(TEST_DIR, "files/blank_global_config.yaml") - - -@pytest.mark.parametrize( - "desc, config_yaml, error_matches", - [ - ( - "Retagging a multiplatform image is not supported", - """ - steps: - build-container-multi-platform: - build: - dockerfile: | - FROM busybox:latest - platforms: - - linux/amd64 - - linux/arm64/v8 - push: - repository: user1/buildrunner-multi-platform-image - tags: [latest] - retag-multi-platform-image: - run: - image: user1/buildrunner-multi-platform-image:latest - cmd: echo "Hello World" - push: - repository: user1/buildrunner-multi-platform-image2 - tags: [latest] - """, - [ - "The following images are re-tagged: ['user1/buildrunner-multi-platform-image:latest']" - ], - ), - ( - "Latest tag is assumed if not specified in image", - """ - steps: - build-container-multi-platform: - build: - dockerfile: | - FROM {{DOCKER_REGISTRY}}/busybox:latest - platforms: - - linux/amd64 - - linux/arm64/v8 - push: - repository: user1/buildrunner-test-multi-platform - tags: [latest] - - retag-built-image: - run: - image: user1/buildrunner-test-multi-platform - cmd: echo "Hello World" - push: - repository: user1/buildrunner-test-multi-platform2 - tags: [latest] - """, - [ - "The following images are re-tagged: ['user1/buildrunner-test-multi-platform:latest']" - ], - ), - ( - "Commit in build step", - """ - steps: - build-container-multi-platform: - build: - dockerfile: | - FROM busybox:latest - platforms: - - linux/amd64 - - linux/arm64/v8 - commit: - repository: user1/buildrunner-multi-platform-image - tags: [latest] - retag-multi-platform-image: - run: - image: user1/buildrunner-multi-platform-image - cmd: echo "Hello World" - push: - repository: user1/buildrunner-multi-platform-image2 - tags: [latest] - """, - [ - "The following images are re-tagged: ['user1/buildrunner-multi-platform-image:latest']" - ], - ), - ( - "Commit after build step", - """ - steps: - build-container-multi-platform: - build: - dockerfile: | - FROM {{DOCKER_REGISTRY}}/busybox:latest - platforms: - - linux/amd64 - - linux/arm64/v8 - push: - repository: user1/buildrunner-test-multi-platform - tags: [latest] - - retag-built-image: - run: - image: user1/buildrunner-test-multi-platform - cmd: echo "Hello World" - commit: - repository: user1/buildrunner-test-multi-platform2 - tags: [latest] - """, - [ - "The following images are re-tagged: ['user1/buildrunner-test-multi-platform:latest']" - ], - ), - ( - "Retagging a single platform image is supported", - """ - steps: - build-container-single-platform: - build: - dockerfile: | - FROM {{DOCKER_REGISTRY}}/busybox:latest - push: - repository: user1/buildrunner-test-single-platform - tags: [latest] - - retag-built-image: - run: - image: user1/buildrunner-test-single-platform:latest - cmd: echo "Hello World" - push: - repository: user1/buildrunner-test-single-platform2 - tags: [latest] - """, - [], - ), - ( - "Read from dockerfile for the 2nd dockerfile", - """ - steps: - build-container-multi-platform: - build: - dockerfile: | - FROM busybox:latest - platforms: - - linux/amd64 - - linux/arm64/v8 - push: - repository: user1/buildrunner-multi-platform-image - tags: [latest] - retag-multi-platform-image: - build: - dockerfile: | - FROM user1/buildrunner-multi-platform-image - push: - repository: user1/buildrunner-multi-platform-image2 - tags: [latest] - """, - [ - "The following images are re-tagged: ['user1/buildrunner-multi-platform-image:latest']" - ], - ), - ( - "Read from dockerfile file", - """ - steps: - build-container-multi-platform: - build: - dockerfile: tests/test_config_validation/Dockerfile.retag - platforms: - - linux/amd64 - - linux/arm64/v8 - push: - repository: user1/buildrunner-multi-platform-image - tags: [latest] - retag-multi-platform-image: - build: - dockerfile: | - FROM user1/buildrunner-multi-platform-image - push: - repository: user1/buildrunner-multi-platform-image2 - tags: [latest] - """, - [RETAG_ERROR_MESSAGE], - ), - ( - "Reuse multi-platform images is valid if the image isn't committed or pushed", - """ - steps: - build-container-multi-platform: - build: - dockerfile: | - FROM {{DOCKER_REGISTRY}}/busybox - platforms: - - linux/amd64 - - linux/arm64/v8 - push: - - repository: user1/buildrunner-test-multi-platform - tags: [ 'latest', '0.0.1' ] - - repository: user2/buildrunner-test-multi-platform - tags: [ 'latest', '0.0.1' ] - - use-built-image1: - run: - image: user1/buildrunner-test-multi-platform:0.0.1 - cmd: echo "Hello World" - - use-built-image2: - run: - image: user2/buildrunner-test-multi-platform:0.0.1 - cmd: echo "Hello World" - """, - [], - ), - ], -) -def test_config_data( - desc, config_yaml, error_matches, assert_generate_and_validate_config_errors -): - _ = desc - assert_generate_and_validate_config_errors(config_yaml, error_matches) - - -@pytest.mark.parametrize( - "config_yaml", - [ - # Tests that "BUILDRUNNER_BUILD_DOCKER_TAG" is replaced with id_string-build_number - """ - steps: - build-container: - build: - dockerfile: | - FROM {{ DOCKER_REGISTRY }}/busybox - platforms: - - linux/amd64 - - linux/arm64/v8 - push: - - repository: user1/buildrunner-test-multi-platform - tags: [ 'latest', '0.0.1', {{ BUILDRUNNER_BUILD_DOCKER_TAG }} ] - - use-built-image1: - run: - image: user1/buildrunner-test-multi-platform - cmd: echo "Hello World" - """, - # Re-pushing images where both steps are multi-platform is valid - """ - steps: - build-container: - build: - dockerfile: | - FROM {{ DOCKER_REGISTRY }}/busybox - platforms: - - linux/amd64 - - linux/arm64/v8 - push: - - repository: user1/buildrunner-test-multi-platform - tags: [ 'latest', '0.0.1', {{ BUILDRUNNER_BUILD_DOCKER_TAG }} ] - - use-built-image1: - build: - dockerfile: | - FROM user1/buildrunner-test-multi-platform:latest - platforms: - - linux/amd64 - - linux/arm64/v8 - push: - repository: user1/buildrunner-test-multi-platform2 - tags: [ 'latest' ] - """, - """ - steps: - build-container: - build: - dockerfile: | - FROM {{ DOCKER_REGISTRY }}/nginx:latest - RUN printf '{{ BUILDRUNNER_BUILD_NUMBER }}' > /usr/share/nginx/html/index.html - push: - repository: user1/buildrunner-test-image - tags: [latest] - """, - """ - steps: - build-container: - build: - dockerfile: | - FROM {{ DOCKER_REGISTRY }}/nginx:latest - RUN printf '{{ BUILDRUNNER_BUILD_NUMBER }}' > /usr/share/nginx/html/index.html - push: - repository: user1/buildrunner-test-image - tags: [latest] - """, - ], -) -@mock.patch("buildrunner.config.DEFAULT_GLOBAL_CONFIG_FILES", []) -@mock.patch("buildrunner.detect_vcs") -def test_valid_config_with_buildrunner_build_tag( - detect_vcs_mock, config_yaml, tmp_path -): - id_string = "main-921.ie02ed8.m1705616822" - build_number = 342 - type(detect_vcs_mock.return_value).id_string = mock.PropertyMock( - return_value=id_string - ) - - buildrunner_path = tmp_path / "buildrunner.yaml" - buildrunner_path.write_text(config_yaml) - BuildRunner( - build_dir=str(tmp_path), - build_results_dir=str(tmp_path / "buildrunner.results"), - global_config_file=None, - run_config_file=str(buildrunner_path), - build_time=0, - build_number=build_number, - push=False, - cleanup_images=False, - cleanup_cache=False, - steps_to_run=None, - publish_ports=False, - log_generated_files=False, - docker_timeout=30, - local_images=False, - platform=None, - global_config_overrides={}, - ) - buildrunner_config = BuildRunnerConfig.get_instance() - push_info = buildrunner_config.run_config.steps["build-container"].push - assert isinstance(push_info, list) - assert f"{id_string}-{build_number}" in push_info[0].tags - - -@pytest.mark.parametrize( - "config_yaml", - [ - """ - steps: - build-container-multi-platform: - build: - dockerfile: | - FROM {{ DOCKER_REGISTRY }}/busybox - platforms: - - linux/amd64 - - linux/arm64/v8 - push: - - repository: user1/buildrunner-test-multi-platform - tags: [ 'latest', '0.0.1', {{ BUILDRUNNER_BUILD_DOCKER_TAG }} ] - - use-built-image1: - run: - image: user1/buildrunner-test-multi-platform:{{ BUILDRUNNER_BUILD_DOCKER_TAG }} - cmd: echo "Hello World" - push: - repository: user1/buildrunner-test-multi-platform2 - tags: [ 'latest' ] - """, - """ - steps: - build-container-multi-platform: - build: - dockerfile: | - FROM {{ DOCKER_REGISTRY }}/busybox - platforms: - - linux/amd64 - - linux/arm64/v8 - push: - - repository: user1/buildrunner-test-multi-platform - tags: [ 'latest', '0.0.1' ] - - use-built-image1: - run: - image: user1/buildrunner-test-multi-platform:{{ BUILDRUNNER_BUILD_DOCKER_TAG }} - cmd: echo "Hello World" - push: user1/buildrunner-test-multi-platform2 - """, - ], - ids=["buildrunner_build_tag_explict", "buildrunner_build_tag_implied"], -) -@mock.patch("buildrunner.config.DEFAULT_GLOBAL_CONFIG_FILES", []) -@mock.patch("buildrunner.detect_vcs") -def test_invalid_retagging_with_buildrunner_build_tag( - detect_vcs_mock, config_yaml, tmp_path -): - # Tests that BUILDRUNNER_BUILD_DOCKER_TAG is added to push tags and fails for re-tagging - id_string = "main-921.ie02ed8.m1705616822" - build_number = 342 - type(detect_vcs_mock.return_value).id_string = mock.PropertyMock( - return_value=id_string - ) - - buildrunner_path = tmp_path / "buildrunner.yaml" - buildrunner_path.write_text(config_yaml) - with pytest.raises(BuildRunnerConfigurationError) as excinfo: - BuildRunner( - build_dir=str(tmp_path), - build_results_dir=str(tmp_path / "buildrunner.results"), - global_config_file=None, - run_config_file=str(buildrunner_path), - build_time=0, - build_number=build_number, - push=False, - cleanup_images=False, - cleanup_cache=False, - steps_to_run=None, - publish_ports=False, - log_generated_files=False, - docker_timeout=30, - local_images=False, - platform=None, - global_config_overrides={}, - ) - assert RETAG_ERROR_MESSAGE in excinfo.value.args[0] - assert ( - f"user1/buildrunner-test-multi-platform:{id_string}-{build_number}" - in excinfo.value.args[0] - ) diff --git a/tests/test_config_validation/test_validation_artifacts.py b/tests/test_config_validation/test_validation_artifacts.py deleted file mode 100644 index 40f09ca4..00000000 --- a/tests/test_config_validation/test_validation_artifacts.py +++ /dev/null @@ -1,152 +0,0 @@ -import pytest - - -@pytest.mark.parametrize( - "config_yaml, error_matches", - [ - ( - """ - steps: - build-remote: - remote: - host: myserver.ut1 - cmd: docker build -t mytest-reg/buildrunner-test . - artifacts: - bogus/path/to/artifacts/*: - type: tar - compression: lzma - """, - [], - ), - ( - """ - steps: - build-run: - run: - artifacts: - bogus/path/to/artifacts/*: - type: zip - compression: lzma - """, - [], - ), - # Valid compression - ( - """ - steps: - build-remote: - remote: - host: myserver.ut1 - cmd: docker build -t mytest-reg/buildrunner-test . - artifacts: - bogus/path/to/artifacts/*: - type: tar - compression: gz - """, - [], - ), - # Valid run format - ( - """ - steps: - build-run: - run: - artifacts: - bogus/path/to/artifacts/*: - format: uncompressed - """, - [], - ), - # Checks zip type - ( - """ - steps: - build-run: - run: - artifacts: - bogus/path/to/artifacts/*: - type: zip - """, - [], - ), - # Checks tar type - ( - """ - steps: - build-run: - run: - artifacts: - bogus/path/to/artifacts/*: - type: tar - """, - [], - ), - # Push must be a boolean - ( - """ - steps: - build-run: - run: - artifacts: - bogus/path/to/artifacts/*: - push: bogus - """, - ["Input should be a valid boolean"], - ), - # Artifact may be a blank string - ( - """ - steps: - build-run: - run: - artifacts: - bogus/path/to/artifacts/*: '' - bogus/path/to/this_thing: '' - """, - [], - ), - # Valid push - ( - """ - steps: - build-run: - run: - artifacts: - bogus/path/to/artifacts/*: - push: True - """, - [], - ), - # Valid extra properties - ( - """ - steps: - build-run: - run: - artifacts: - bogus/path/to/artifacts/*: - push: True - something_else: awesome data - something_else2: True - something_else3: 123 - """, - [], - ), - # Rename - ( - """ - steps: - build-run: - run: - artifacts: - bogus/path/to/artifacts/my-artifact.txt: - rename: my-artifact-renamed.txt - """, - [], - ), - ], -) -def test_config_data( - config_yaml, error_matches, assert_generate_and_validate_config_errors -): - assert_generate_and_validate_config_errors(config_yaml, error_matches) diff --git a/tests/test_config_validation/test_validation_config.py b/tests/test_config_validation/test_validation_config.py deleted file mode 100644 index e9d8c7bb..00000000 --- a/tests/test_config_validation/test_validation_config.py +++ /dev/null @@ -1,201 +0,0 @@ -import pytest - - -@pytest.mark.parametrize( - "config_data, error_matches", - [ - # Invalid version - ( - {"version": "string"}, - [ - "Input should be a valid number", - "steps: Field required", - ], - ), - # Valid version - ({"version": 2.0, "steps": {}}, ['The "steps" configuration was not provided']), - # Optional version - ({"steps": {}}, ['The "steps" configuration was not provided']), - # Sample valid config, but not exhaustive - ( - """ - version: 2.0 - steps: - build-container-single-platform1: - build: - path: . - dockerfile: Dockerfile - pull: false - platform: linux/amd64 - push: - repository: mytest-reg/buildrunner-test - tags: - - latest - build-container-multi-platform2: - build: - path: . - dockerfile: Dockerfile - pull: false - platforms: - - linux/amd64 - - linux/arm64 - push: - repository: mytest-reg/buildrunner-test-multi-platform - tags: - - latest - build-container-multi-platform-push3: - build: - path: . - dockerfile: Dockerfile - pull: false - platforms: - - linux/amd64 - - linux/arm64 - push: - - myimages/image1 - - repository: myimages/image2 - tags: - - latest - """, - [], - ), - # Multiple errors - # Invalid to have version as a string - # Invalid to have platforms and platform - ( - """ - version: string - steps: - build-container-multi-platform: - build: - path: . - dockerfile: Dockerfile - pull: false - platform: linux/amd64 - platforms: - - linux/amd64 - - linux/arm64 - push: - repository: mytest-reg/buildrunner-test-multi-platform - tags: - - latest - """, - ["Input should be a valid number", "Cannot specify both platform"], - ), - # Tests the documentation example with minimal changes to make valid yaml - ( - """ - steps: - generate_files: - run: - image: docker.company.com/abc-xdm-proto-build:latest - ssh-keys: ["company-github"] - env: - GIT_TOKEN: 'blahblahblahblahblahblah' - cmd: sbt clean generateAwareJsonFiles combineXDM generateProtobufFiles - artifacts: - 'target/protobufFiles/Database*.proto': - 'target/rawJson/Database*.json': - 'target/AwareJson/Aware.json': - 'target/combinedXDM/complete-schema-template.schema.json': - build-dev-rpm: - build: - inject: - "buildrunner.results/generate_files/*.proto": "proto/" - "buildrunner.results/generate_files/A*.json": "json/" - "db_build/dms.repo.centos7": db_build/dms.repo - dockerfile: | - FROM docker-release.dr.corp.company.com/centos-7-x86_64-obuild:latest - ADD db_build/dms.repo /etc/yum.repos.d/dms.repo - RUN rpm --rebuilddb; yum clean all; yum install -y db-omniture-libs-protobuf-2.6.1 db-scds-proto-1.0 db-scds-json-1.0 - ADD proto/*.proto /tmp/proto/ - ADD json/*.json /tmp/json/ - run: - cmds: - - "chown -R httpd:www /source" - - "echo ~ Compiling previous proto version..." - - "mkdir -p /tmp/existingscds && for f in `ls -d /home/omniture/protobuf/scds/*.proto`; do protoc -I=/home/omniture/protobuf --cpp_out /tmp/existingscds $f; done" - - "echo ~ Compiling current proto version..." - artifacts: - # pull the log if rpmbuild fails - "db_tmp/rpm/TMPDIR/*.log": {type: 'log'} - # pull the noarch packages - "db_tmp/rpm/RPMS/noarch/*.noarch.rpm": {platform: 'centos-noarch'} - build-proto-java: - build: - inject: - "buildrunner.results/generate_files/*.proto": "proto" - dockerfile: | - FROM docker.company.com/abc-base-containers/protobuf-builder:java8-2.5.0 - ADD proto/*.proto /tmp/proto/scds/ - run: - caches: - maven: "/root/.m2/repository" - cmds: [ - 'mvn package ${BUILDRUNNER_DO_PUSH+deploy} -am -pl proto-java' - ] - artifacts: - '*/target/*.jar': - download-country: - build: - inject: - "db_build/bin/*": "db_build/bin/" - dockerfile: | - FROM docker-release.dr.corp.company.com/centos-7-x86_64-obuild - ADD db_build/bin/* /tmp/ - run: - cmds: - - '/tmp/download_country.sh' - # strip all quotes - - "sed -i 's/bogus//g' country_codes.csv" - # Add missing ?,? because it's not in the DB - - 'echo "?,?" >> country_codes.csv' - # keep first 2 columns, uppercase 2nd column - - 'awk -F, ''{OFS=","; $2 = toupper($2); {print $1,$2}}'' country_codes.csv > country_code_map.csv' - artifacts: - 'country_code_map.csv': - build-transform-proto-xdm: - build: - inject: - "buildrunner.results/generate_files/*.proto": "proto" - "buildrunner.results/generate_files/*.json": "json" - dockerfile: | - FROM docker.company.com/abc-base-containers/protobuf-builder:java8-2.5.0 - RUN apt-get update && apt-get -y install openssh-client - ADD proto/*.proto /tmp/proto/scds/ - run: - env: - ARTIFACTORY_USER: 'cool_user' - ARTIFACTORY_API_TOKEN: 'blahblahblahblahblahblahblah' - caches: - maven: "/root/.m2/repository" - shell: /bin/bash - cmds: [ - 'cp /tmp/json/raw/*json json/raw', - 'mkdir -p csv', - 'cp /tmp/csv/*csv csv', - 'curl -L https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64 > jq', - 'chmod +x jq', - ] - artifacts: - 'transform-proto-xdm/target/*': - 'transform-proto-xdm-generator/target/*': - 'validator-xdm/target/*': - generate_docs: - run: - image: docker.company.com/abc-xdm-proto-build:latest - ssh-keys: ["company-github"] - env: - GIT_TOKEN: 'blahblahblahblahblahblahblah' - cmd: "sbt clean generateDocs ${BUILDRUNNER_DO_PUSH+publishGHPages}" - artifacts: - 'target/docs/*': - """, - [], - ), - ], -) -def test_config_data( - config_data, error_matches, assert_generate_and_validate_config_errors -): - assert_generate_and_validate_config_errors(config_data, error_matches) diff --git a/tests/test_config_validation/test_validation_step.py b/tests/test_config_validation/test_validation_step.py deleted file mode 100644 index 65902015..00000000 --- a/tests/test_config_validation/test_validation_step.py +++ /dev/null @@ -1,392 +0,0 @@ -import pytest - -from buildrunner.config.models_step import StepPushCommit - - -@pytest.mark.parametrize( - "config_yaml, error_matches", - [ - # Invalid to have platform and platforms - ( - """ - steps: - build-container-multi-platform: - build: - path: . - dockerfile: Dockerfile - pull: false - platform: linux/amd64 - platforms: - - linux/amd64 - - linux/arm64 - push: - repository: mytest-reg/buildrunner-test-multi-platform - tags: - - latest - """, - ["Cannot specify both platform"], - ), - # Invalid to have platforms as a string, it should be a list - ( - """ - steps: - build-container-multi-platform: - build: - path: . - dockerfile: Dockerfile - pull: false - platforms: linux/amd64 - push: - repository: mytest-reg/buildrunner-test-multi-platform - tags: - - latest - """, - ["Input should be a valid list"], - ), - # Invalid to have cache_from specified with platforms - ( - """ - steps: - build-container-multi-platform: - build: - path: . - dockerfile: Dockerfile - pull: false - platforms: - - linux/amd64 - cache_from: - - image1 - push: - repository: mytest-reg/buildrunner-test-multi-platform - tags: - - latest - """, - ["cache_from"], - ), - # Build is a path - ( - """ - steps: - build-is-path: - build: . - """, - [], - ), - # Valid platforms - ( - """ - steps: - build-container-multi-platform: - build: - path: . - dockerfile: Dockerfile - pull: false - platforms: - - linux/amd64 - - linux/arm64 - push: - repository: mytest-reg/buildrunner-test-multi-platform - tags: - - latest - """, - [], - ), - # Platforms with no-cache - ( - """ - steps: - build-container-multi-platform: - build: - path: . - dockerfile: Dockerfile - pull: false - platforms: - - linux/amd64 - - linux/arm64 - no-cache: true - push: - repository: mytest-reg/buildrunner-test-multi-platform - tags: - - latest - """, - [], - ), - # Invalid to have duplicate multi-platform tag - ( - """ - steps: - build-container-multi-platform1: - build: - platforms: - - linux/amd64 - - linux/arm64 - push: - repository: mytest-reg/buildrunner-test-multi-platform - tags: - - latest - build-container-multi-platform2: - build: - platforms: - - linux/amd64 - - linux/arm64 - push: - repository: mytest-reg/buildrunner-test-multi-platform - tags: - - latest - """, - [ - "Cannot specify duplicate tag mytest-reg/buildrunner-test-multi-platform:latest in build step" - ], - ), - # Identical tags in same string format - ( - """ - steps: - build-container-multi-platform1: - build: - platforms: - - linux/amd64 - - linux/arm64 - push: mytest-reg/buildrunner-test-multi-platform:latest - build-container-multi-platform2: - build: - platforms: - - linux/amd64 - - linux/arm64 - push: mytest-reg/buildrunner-test-multi-platform:latest - """, - [ - "Cannot specify duplicate tag mytest-reg/buildrunner-test-multi-platform:latest in build step" - ], - ), - # Same string format but different MP tags - ( - """ - steps: - build-container-multi-platform1: - build: - dockerfile: | - FROM busybox:latest - platforms: - - linux/amd64 - - linux/arm64 - push: mytest-reg/buildrunner-test-multi-platform:latest - build-container-multi-platform2: - build: - dockerfile: | - FROM busybox:latest - platforms: - - linux/amd64 - - linux/arm64 - push: mytest-reg/buildrunner-test-multi-platform:not-latest - """, - [], - ), - # Invalid to have duplicate multi-platform tag and single platform tag - ( - """ - steps: - build-container-multi-platform1: - build: - path: . - platforms: - - linux/amd64 - - linux/arm64 - push: mytest-reg/buildrunner-test-multi-platform:latest - build-container-single-platform: - build: - path: . - platform: linux/arm64 - push: mytest-reg/buildrunner-test-multi-platform:latest - """, - [ - "Cannot specify duplicate tag mytest-reg/buildrunner-test-multi-platform:latest in build step" - ], - ), - # Valid remote step - ( - """ - steps: - build-remote: - remote: - host: myserver.ut1 - cmd: docker build -t mytest-reg/buildrunner-test . - artifacts: - bogus/path/to/artifacts/*: - type: tar - compression: lzma - """, - [], - ), - # Remote missing command - ( - """ - steps: - build-remote: - remote: - host: myserver.ut1 - """, - ["Field required"], - ), - # Valid commit - ( - """ - steps: - step1: - build: - path: . - dockerfile: Dockerfile - pull: false - commit: - repository: mytest-reg/image1 - tags: - - latest - step2: - build: - path: . - dockerfile: Dockerfile - pull: false - commit: mytest-reg/image1 - """, - [], - ), - # Valid pypi push - ( - """ - steps: - pypi1: - run: - image: python:2 - cmds: - - python setup.py sdist - artifacts: - "dist/*.tar.gz": { type: 'python-sdist' } - pypi-push: artifactory-releng - pypi2: - run: - image: python:2 - cmds: - - python -m build - artifacts: - "dist/*.tar.gz": { type: 'python-sdist' } - "dist/*.whl": { type: 'python-wheel' } - pypi-push: - repository: https://artifactory.example.com/artifactory/api/pypi/pypi-myownrepo - username: myuser - password: mypass - """, - [], - ), - # Invalid multiplatform import - ( - """ - steps: - build-container-multi-platform: - build: - path: . - dockerfile: Dockerfile - platforms: - - linux/amd64 - - linux/arm64 - import: mytest-reg/buildrunner-test-multi-platform:latest - """, - ["import is not allowed in multi-platform build step"], - ), - # Valid import - ( - """ - steps: - build-container-multi-platform: - build: - path: . - dockerfile: Dockerfile - import: mytest-reg/buildrunner-test-multi-platform:latest - """, - [], - ), - # Valid services - ( - """ - steps: - my-build-step: - run: - services: - my-service-container: - build: - #image: - cmd: - provisioners: - shell: path/to/script.sh - salt: - shell: /bin/sh - cwd: /source - user: - hostname: - dns: - - 8.8.8.8 - - 8.8.4.4 - dns_search: mydomain.com - extra_hosts: - "www1.test.com": "192.168.0.1" - "www2.test.com": "192.168.0.2" - env: - ENV_VARIABLE_ONE: value1 - ENV_VARIABLE_TWO: value2 - files: - namespaced.file.alias1: "/path/to/readonly/file/or/dir" - namespaced.file.alias2: "/path/to/readwrite/file/or/dir:rw" - volumes_from: - - my-service-container - ports: - 8081: 8080 - pull: true - systemd: true - containers: - - container1 - - container2 - wait_for: - - 80 - - port: 9999 - timeout: 30 - inject-ssh-agent: true - """, - [], - ), - ], -) -def test_config_data( - config_yaml, error_matches, assert_generate_and_validate_config_errors -): - assert_generate_and_validate_config_errors(config_yaml, error_matches) - - -def test_transforms(assert_generate_and_validate_config_errors): - config, _ = assert_generate_and_validate_config_errors( - { - "steps": { - "build": {"build": "."}, - "pypi": {"pypi-push": "pypi1"}, - "commit-str": {"commit": "commit1"}, - "push-str": {"push": "push1"}, - "push-list-str": {"push": ["push2", "push3"]}, - # Ensure that the "push" parameter is automatically set always to the correct value - "push-dict": {"push": {"repository": "push4", "push": False}}, - } - }, - [], - ) - assert config.steps["build"].build.path == "." - assert config.steps["pypi"].pypi_push.repository == "pypi1" - assert config.steps["commit-str"].commit == [ - StepPushCommit(repository="commit1", push=False), - ] - assert config.steps["push-str"].push == [ - StepPushCommit(repository="push1", push=True), - ] - assert config.steps["push-list-str"].push == [ - StepPushCommit(repository="push2", push=True), - StepPushCommit(repository="push3", push=True), - ] - assert config.steps["push-dict"].push == [ - StepPushCommit(repository="push4", push=True), - ] diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py deleted file mode 100644 index 66010be2..00000000 --- a/tests/test_dependencies.py +++ /dev/null @@ -1,169 +0,0 @@ -from collections import OrderedDict -import unittest -import copy -import graphlib - -from buildrunner.config import loader - - -class TestDependencies(unittest.TestCase): - config = OrderedDict( - [ - ( - "steps", - OrderedDict( - [ - ( - "step1", - OrderedDict( - [ - ( - "run", - OrderedDict( - [ - ("image", "docker.io/ubuntu:latest"), - ("cmd", 'echo "Hello from step 1"'), - ] - ), - ) - ] - ), - ), - ( - "step2", - OrderedDict( - [ - ("depends", ["step3", "step4"]), - ( - "run", - OrderedDict( - [ - ("image", "docker.io/ubuntu:latest"), - ("cmd", 'echo "Hello from step 2"'), - ] - ), - ), - ] - ), - ), - ( - "step3", - OrderedDict( - [ - ( - "run", - OrderedDict( - [ - ("image", "docker.io/ubuntu:latest"), - ("cmd", 'echo "Hello from step 3"'), - ] - ), - ) - ] - ), - ), - ( - "step4", - OrderedDict( - [ - ( - "run", - OrderedDict( - [ - ("image", "docker.io/ubuntu:latest"), - ("cmd", 'echo "Hello from step 4"'), - ] - ), - ) - ] - ), - ), - ] - ), - ) - ] - ) - - KEYWORD_VERSION = "version" - KEYWORD_STEPS = "steps" - KEYWORD_DEPENDS = "depends" - - def check_steps_equal(self, steps, expected_step_names): - for actual_step_name, actual_step in steps[self.KEYWORD_STEPS].items(): - self.assertEqual(next(expected_step_names), actual_step_name) - - expected_step = copy.deepcopy( - self.config[self.KEYWORD_STEPS][actual_step_name] - ) - if self.KEYWORD_DEPENDS in expected_step: - del expected_step[self.KEYWORD_DEPENDS] - self.assertDictEqual(expected_step, actual_step) - - def test_reorder_steps(self): - config = copy.deepcopy(self.config) - config[self.KEYWORD_VERSION] = 2.0 - expected_step_names = iter(["step1", "step3", "step4", "step2"]) - - reordered_steps = loader._reorder_dependency_steps(config) - - self.check_steps_equal(reordered_steps, expected_step_names) - - def test_reorder_steps_higher_version(self): - config = copy.deepcopy(self.config) - config[self.KEYWORD_VERSION] = 2.1 - expected_step_names = iter(["step1", "step3", "step4", "step2"]) - - reordered_steps = loader._reorder_dependency_steps(config) - - self.check_steps_equal(reordered_steps, expected_step_names) - - config = copy.deepcopy(self.config) - config[self.KEYWORD_VERSION] = 3.1 - expected_step_names = iter(["step1", "step3", "step4", "step2"]) - - reordered_steps = loader._reorder_dependency_steps(config) - - self.check_steps_equal(reordered_steps, expected_step_names) - - def test_no_reorder(self): - config = copy.deepcopy(self.config) - config[self.KEYWORD_VERSION] = 2.0 - - # Remove any steps with 'depends' attribute - for name, step in config[self.KEYWORD_STEPS].items(): - if self.KEYWORD_DEPENDS in step: - del step[self.KEYWORD_DEPENDS] - - reordered_steps = loader._reorder_dependency_steps(config) - - self.assertDictEqual(config, reordered_steps) - - def test_not_supported_version(self): - config = copy.deepcopy(self.config) - config[self.KEYWORD_VERSION] = 1.9 - - reordered_steps = loader._reorder_dependency_steps(config) - - del reordered_steps[self.KEYWORD_VERSION] - self.assertDictEqual(self.config, reordered_steps) - - def test_missing_version(self): - config = copy.deepcopy(self.config) - - reordered_steps = loader._reorder_dependency_steps(config) - - self.assertDictEqual(self.config, reordered_steps) - - def test_cycle_dependency(self): - config = copy.deepcopy(self.config) - config[self.KEYWORD_VERSION] = 2.0 - config[self.KEYWORD_STEPS]["step4"][self.KEYWORD_DEPENDS] = ["step3", "step2"] - - self.assertRaises(graphlib.CycleError, loader._reorder_dependency_steps, config) - - def test_not_defined_dependency(self): - config = copy.deepcopy(self.config) - config[self.KEYWORD_VERSION] = 2.0 - config[self.KEYWORD_STEPS]["step4"][self.KEYWORD_DEPENDS] = ["step1-typo"] - - self.assertRaises(KeyError, loader._reorder_dependency_steps, config) diff --git a/tests/test_image_info.py b/tests/test_image_info.py deleted file mode 100644 index 2623d0a1..00000000 --- a/tests/test_image_info.py +++ /dev/null @@ -1,43 +0,0 @@ -from unittest import mock - -import pytest - -from buildrunner.docker.image_info import BuiltTaggedImage, BuiltImageInfo - - -@pytest.mark.parametrize( - "system_result, machine_result, platform_result", - [ - ("Darwin", "x86_64", "linux/amd64"), - ("Darwin", "aarch64", "linux/arm64/v7"), - ("Linux", "x86_64", "linux/amd64"), - ("Linux", "aarch64", "linux/arm64/v7"), - ("Bogus", "Noarch", "bogus"), - ], -) -@mock.patch("buildrunner.docker.image_info.python_platform") -def test_find_native_platform( - platform_mock, - system_result, - machine_result, - platform_result, -): - platform_mock.system.return_value = system_result - platform_mock.machine.return_value = machine_result - - run_id = "abc123" - built_image = BuiltImageInfo(id=run_id) - for platform in ("bogus", "linux/arm64/v7", "linux/amd64"): - built_image.add_platform_image( - platform, - BuiltTaggedImage( - repo="repo1", - tag=f'{run_id}-{platform.replace("/", "-")}', - digest="12345", - platform=platform, - ), - ) - assert ( - built_image.native_platform_image - == built_image.images_by_platform[platform_result] - ) diff --git a/tests/test_loggers.py b/tests/test_loggers.py deleted file mode 100644 index d6d80316..00000000 --- a/tests/test_loggers.py +++ /dev/null @@ -1,261 +0,0 @@ -import logging -import sys -from unittest import mock - -import pytest - -from buildrunner import loggers - - -@pytest.fixture(autouse=True) -def fixture_override_colors(): - original_build_colors = loggers.ContainerLogger.BUILD_LOG_COLORS - original_service_colors = loggers.ContainerLogger.SERVICE_LOG_COLORS - loggers.ContainerLogger.BUILD_LOG_COLORS = loggers.ColorQueue("yellow", "blue") - loggers.ContainerLogger.SERVICE_LOG_COLORS = loggers.ColorQueue( - "purple", - "cyan", - "red", - "green", - "purple", - ) - yield - loggers.ContainerLogger.BUILD_LOG_COLORS = original_build_colors - loggers.ContainerLogger.SERVICE_LOG_COLORS = original_service_colors - - -@pytest.mark.parametrize( - "debug, no_color, disable_timestamps, log_level, console_format, file_format", - [ - ( - False, - False, - False, - logging.INFO, - "%(log_color)s%(asctime)s %(levelname)-8s %(message)s", - "%(log_color)s%(asctime)s %(levelname)-8s %(message)s", - ), - ( - True, - True, - True, - logging.DEBUG, - "%(log_color)s%(levelname)-8s %(message)s", - "%(log_color)s%(asctime)s %(levelname)-8s %(message)s", - ), - ], -) -@mock.patch("buildrunner.loggers.logging") -def test_initialize_root_logger( - logging_mock, - debug, - no_color, - disable_timestamps, - log_level, - console_format, - file_format, - tmp_path, -): - logging_mock.DEBUG = logging.DEBUG - logging_mock.INFO = logging.INFO - root_logger = mock.create_autospec(logging.Logger) - root_logger.handlers = mock.MagicMock() - file_handler = mock.create_autospec(logging.FileHandler) - stream_handler = mock.create_autospec(logging.StreamHandler) - logging_mock.getLogger.return_value = root_logger - logging_mock.FileHandler.return_value = file_handler - logging_mock.StreamHandler.return_value = stream_handler - results_dir = tmp_path - - loggers.initialize_root_logger( - debug, no_color, disable_timestamps, str(results_dir) - ) - logging_mock.getLogger.assert_called_once_with() - root_logger.setLevel.assert_called_once_with(log_level) - logging_mock.FileHandler.assert_called_once_with( - str(results_dir / "build.log"), "w", encoding="utf8" - ) - logging_mock.StreamHandler.assert_called_once_with(sys.stdout) - file_handler.setFormatter.assert_called_once() - stream_handler.setFormatter.assert_called_once() - root_logger.handlers.clear.assert_called_once_with() - assert root_logger.addHandler.call_args_list == [ - mock.call(stream_handler), - mock.call(file_handler), - ] - - # Check formatters - file_formatter = file_handler.setFormatter.call_args.args[0] - assert file_formatter.fmt == file_format - assert file_formatter.no_color - assert file_formatter.color == "white" - stream_formatter = stream_handler.setFormatter.call_args.args[0] - assert stream_formatter.fmt == console_format - assert stream_formatter.no_color == no_color - assert stream_formatter.color == "white" - # Make sure the formatters are not the same, they should be distinct - assert file_formatter != stream_formatter - - -@pytest.mark.parametrize( - "output, lines", - [ - ("", [""]), - ("output1", ["output1"]), - ("output1\n", ["output1"]), - ("\noutput1\n\n", ["", "output1", ""]), - ], -) -@mock.patch("buildrunner.loggers.logging") -def test_console_logger(logging_mock, output, lines): - mock_logger = mock.create_autospec(logging.Logger) - logging_mock.getLogger.return_value = mock_logger - console_logger = loggers.ConsoleLogger("name1") - console_logger.write(output) - assert mock_logger.info.call_args_list == [mock.call(line) for line in lines] - - -@mock.patch("buildrunner.loggers.ContainerLogger.__init__") -def test_container_logger_for_methods(container_logger_mock): - container_logger_mock.return_value = None - loggers.ContainerLogger.for_build_container("build1") - loggers.ContainerLogger.for_build_container("build1") - loggers.ContainerLogger.for_build_container("build2") - loggers.ContainerLogger.for_service_container("service1") - loggers.ContainerLogger.for_service_container("service2") - loggers.ContainerLogger.for_service_container("service3") - loggers.ContainerLogger.for_service_container("service4") - loggers.ContainerLogger.for_service_container("service5") - - assert container_logger_mock.call_args_list == [ - mock.call("build1_yellow", "yellow", prefix="build1"), - mock.call("build1_blue", "blue", prefix="build1"), - mock.call("build2_yellow", "yellow", prefix="build2"), - mock.call("service-service1", "purple"), - mock.call("service-service2", "cyan"), - mock.call("service-service3", "red"), - mock.call("service-service4", "green"), - mock.call("service-service5", "purple"), - ] - - -@mock.patch("buildrunner.loggers.logging") -def test_container_logger_set_logger_handlers(logging_mock, tmp_path): - root_logger = mock.create_autospec(logging.Logger) - root_logger.handlers = [ - logging.FileHandler( - str(tmp_path / "log"), - "w", - encoding="utf8", - ), - logging.StreamHandler(sys.stdout), - ] - color_formatter = loggers.CustomColoredFormatter("%(message)s", False, "white") - for handler in root_logger.handlers: - handler.setFormatter(color_formatter) - - logging_mock.StreamHandler = logging.StreamHandler - logging_mock.FileHandler = logging.FileHandler - - mock_logger1 = mock.create_autospec(logging.Logger) - mock_logger1.handlers = [] - mock_logger1.propagate = True - mock_logger2 = mock.create_autospec(logging.Logger) - mock_logger2.handlers = [] - mock_logger2.propagate = True - mock_loggers = {None: root_logger, "name1": mock_logger1, "name2": mock_logger2} - logging_mock.getLogger.side_effect = lambda name=None: mock_loggers.get(name) - - loggers.ContainerLogger("name1", "blue") - assert not mock_logger1.propagate - assert mock_logger1.addHandler.call_count == 2 - assert mock_logger1.addHandler.call_args_list[0] == mock.call( - root_logger.handlers[0] - ) - handler = mock_logger1.addHandler.call_args_list[1].args[0] - assert handler.__class__ == logging.StreamHandler - assert isinstance(handler.formatter, loggers.CustomColoredFormatter) - assert handler.formatter.fmt == "%(message)s" - assert not handler.formatter.no_color - assert handler.formatter.color == "blue" - - loggers.ContainerLogger("name2", "purple") - assert not mock_logger2.propagate - assert mock_logger2.addHandler.call_count == 2 - assert mock_logger2.addHandler.call_args_list[0] == mock.call( - root_logger.handlers[0] - ) - handler = mock_logger2.addHandler.call_args_list[1].args[0] - assert isinstance(handler.formatter, loggers.CustomColoredFormatter) - assert handler.formatter.color == "purple" - - -@pytest.mark.parametrize( - "prefix, outputs, lines", - [ - (None, [], []), - (None, ["line1"], ["[name1] line1"]), - ("p1", ["line", "1\nline", "2\n"], ["[p1] line1", "[p1] line2"]), - ], -) -@mock.patch("buildrunner.loggers.logging") -def test_container_logger_write(logging_mock, prefix, outputs, lines): - logger = loggers.ContainerLogger("name1", "white", prefix=prefix) - for output in outputs: - logger.write(output) - logger.cleanup() - assert logging_mock.getLogger.return_value.info.call_args_list == [ - mock.call(line) for line in lines - ] - - -@mock.patch("buildrunner.loggers.progress") -def test_docker_pull_progress(progress_mock): - with loggers.DockerPullProgress() as progress: - progress_mock.Progress.assert_called_once_with() - progress_instance = progress_mock.Progress.return_value - progress_instance.__enter__.assert_called_once_with() - progress_instance.__exit__.assert_not_called() - task1 = mock.MagicMock() - task2 = mock.MagicMock() - progress_instance.add_task.side_effect = [ - task1, - task2, - Exception("this is a failure"), - ] - - progress.status_report({}) - progress.status_report({"status": None}) - progress.status_report( - {"status": "Downloading", "id": "1", "progressDetail": {"total": 100}} - ) - progress.status_report( - {"status": "Downloading", "progressDetail": {"total": 10}} - ) - progress.status_report( - {"status": "Downloading", "id": "0", "progressDetail": {"total": 10}} - ) - progress.status_report( - {"status": "Downloading", "id": "1", "progressDetail": {"current": 10}} - ) - progress.status_report( - {"status": "Downloading", "id": "2", "progressDetail": {"total": 10}} - ) - progress.status_report({"status": "Downloading", "id": "failure"}) - progress.status_report( - {"status": "Extracting", "id": "1", "progressDetail": {"current": 5}} - ) - progress.status_report( - {"status": "Extracting", "id": "2", "progressDetail": {"current": 10}} - ) - progress_instance.__exit__.assert_called_once() - - assert progress_instance.add_task.call_args_list == [ - mock.call("[cyan]1: Downloading", total=100), - mock.call("[cyan]2: Downloading", total=10), - ] - assert progress_instance.update.call_args_list == [ - mock.call(task1, description="[cyan]1: Downloading", completed=10), - mock.call(task1, description="[green]1: Extracting", completed=5), - mock.call(task2, description="[green]2: Extracting", completed=10), - ] diff --git a/tests/test_multiplatform.py b/tests/test_multiplatform.py deleted file mode 100644 index 3d6b758c..00000000 --- a/tests/test_multiplatform.py +++ /dev/null @@ -1,807 +0,0 @@ -import os -from typing import List -from unittest.mock import MagicMock, call, patch - -import pytest -from python_on_whales import docker -from python_on_whales.exceptions import DockerException - -from buildrunner.docker.multiplatform_image_builder import MultiplatformImageBuilder -from buildrunner.docker.image_info import ( - BuiltImageInfo, - BuiltTaggedImage, - TaggedImageInfo, -) - - -TEST_DIR = os.path.dirname(__file__) - -# FIXME: These tests can be broken if a custom buildx builder is set as default # pylint: disable=fixme - - -@pytest.fixture(autouse=True) -def fixture_uuid_mock(): - with patch("buildrunner.docker.multiplatform_image_builder.uuid") as uuid_mock: - counter = 0 - - def _get_uuid(): - nonlocal counter - counter += 1 - return f"uuid{counter}" - - uuid_mock.uuid4.side_effect = _get_uuid - yield uuid_mock - - -@pytest.fixture(autouse=True) -def fixture_init_config(): - with patch( - "buildrunner.docker.multiplatform_image_builder.BuildRunnerConfig" - ) as config_mock: - config_mock.get_instance.return_value.global_config.disable_multi_platform = ( - False - ) - yield config_mock - - -def _actual_images_match_expected( - built_image: BuiltImageInfo, expected_tags -) -> List[str]: - actual_images = built_image.built_images - missing_images = [] - for expected_tag in expected_tags: - found = False - for actual_image in actual_images: - if actual_image.tag.endswith(expected_tag): - found = True - if not found: - missing_images.append(expected_tag) - return missing_images - - -def test_start_local_registry(): - with MultiplatformImageBuilder() as mpib: - mpib._start_local_registry() - registry_name = mpib._mp_registry_info.name - - # Check that the registry is running and only one is found with that name - registry_container = docker.ps(filters={"name": registry_name}) - assert len(registry_container) == 1 - registry_container = registry_container[0] - - # Check that the registry only has one mount - mounts = registry_container.mounts - assert len(mounts) == 1 - mount = mounts[0] - assert mount.type == "volume" - volume_name = mount.name - assert docker.volume.exists(volume_name) - - # Check that the running container is the registry - assert registry_container.config.image == "registry" - assert registry_container.state.running - - # Check that the registry is stopped and cleaned up - registry_container = docker.ps(filters={"name": registry_name}) - assert len(registry_container) == 0 - assert not docker.volume.exists(volume_name) - - -def test_start_local_registry_on_build(): - with MultiplatformImageBuilder() as mpib: - # Check that the registry is NOT running - assert mpib._mp_registry_info is None - assert mpib._local_registry_is_running is False - - # Building should start the registry - test_path = f"{TEST_DIR}/test-files/multiplatform" - mpib.build_multiple_images( - platforms=["linux/arm64", "linux/amd64"], - path=test_path, - file=f"{test_path}/Dockerfile", - use_threading=False, - ) - - # Check that the registry is running and only one is found with that name - registry_name = mpib._mp_registry_info.name - first_registry_name = registry_name - registry_container = docker.ps(filters={"name": registry_name}) - assert len(registry_container) == 1 - registry_container = registry_container[0] - - # Check that the registry only has one mount - mounts = registry_container.mounts - assert len(mounts) == 1 - mount = mounts[0] - assert mount.type == "volume" - volume_name = mount.name - assert docker.volume.exists(volume_name) - - # Check that the running container is the registry - assert registry_container.config.image == "registry" - assert registry_container.state.running - - # Building again should not start a new registry - mpib.build_multiple_images( - platforms=["linux/arm64", "linux/amd64"], - path=test_path, - file=f"{test_path}/Dockerfile", - use_threading=False, - ) - - registry_name = mpib._mp_registry_info.name - assert first_registry_name == registry_name - - # Check that the registry is stopped and cleaned up - registry_container = docker.ps(filters={"name": registry_name}) - assert len(registry_container) == 0 - assert not docker.volume.exists(volume_name) - - -@pytest.mark.parametrize( - "platforms, expected_image_tags", - [(["linux/arm64"], ["uuid1-linux-arm64"])], -) -def test_tag_native_platform(platforms, expected_image_tags): - test_path = f"{TEST_DIR}/test-files/multiplatform" - with MultiplatformImageBuilder() as mpib: - built_image = mpib.build_multiple_images( - platforms, - path=test_path, - file=f"{test_path}/Dockerfile", - use_threading=False, - ) - - assert ( - built_image is not None and len(built_image.built_images) == 1 - ), f"Failed to build for {platforms}" - missing_images = _actual_images_match_expected(built_image, expected_image_tags) - assert ( - missing_images == [] - ), f"Failed to find {missing_images} in {[image.repo for image in built_image.built_images]}" - - mpib.tag_native_platform(built_image) - # Check that the image was tagged and present - found_image = docker.image.list( - filters={"reference": built_image.built_images[0].image_ref} - ) - assert len(found_image) == 1 - assert built_image.built_images[0].image_ref in found_image[0].repo_tags - - # Check that intermediate images are on the host registry - found_image = docker.image.list( - filters={"reference": f"{built_image.built_images[0].image_ref}*"} - ) - assert len(found_image) == 1 - - -@pytest.mark.parametrize( - "name, platforms, expected_image_tags", - [("test-image-tag-2000", ["linux/arm64"], ["uuid1-linux-arm64"])], -) -def test_tag_native_platform_multiple_tags(name, platforms, expected_image_tags): - tags = ["latest", "0.1.0"] - test_path = f"{TEST_DIR}/test-files/multiplatform" - with MultiplatformImageBuilder() as mpib: - built_image = mpib.build_multiple_images( - platforms=platforms, - path=test_path, - file=f"{test_path}/Dockerfile", - use_threading=False, - ) - - assert ( - built_image is not None and len(built_image.built_images) == 1 - ), f"Failed to build {name} for {platforms}" - missing_images = _actual_images_match_expected(built_image, expected_image_tags) - assert ( - missing_images == [] - ), f"Failed to find {missing_images} in {[image.repo for image in built_image.built_images]}" - - built_image.add_tagged_image(repo=name, tags=tags) - mpib.tag_native_platform(built_image) - # Check that the image was tagged and present - found_image = docker.image.list(filters={"reference": f"{name}*"}) - assert len(found_image) == 1 - for tag in tags: - assert f"{name}:{tag}" in found_image[0].repo_tags - - # Check that intermediate images are not on host registry - found_image = docker.image.list( - filters={"reference": f"{built_image.built_images[0].image_ref}*"} - ) - assert len(found_image) == 0 - - -@pytest.mark.parametrize( - "name, platforms, expected_image_tags", - [("test-image-tag-2000", ["linux/arm64"], ["uuid1-linux-arm64"])], -) -def test_tag_native_platform_keep_images(name, platforms, expected_image_tags): - tag = "latest" - test_path = f"{TEST_DIR}/test-files/multiplatform" - try: - with MultiplatformImageBuilder() as mpib: - built_image = mpib.build_multiple_images( - platforms=platforms, - path=test_path, - file=f"{test_path}/Dockerfile", - use_threading=False, - ) - - assert ( - built_image is not None and len(built_image.built_images) == 1 - ), f"Failed to build {name} for {platforms}" - missing_images = _actual_images_match_expected( - built_image, expected_image_tags - ) - assert ( - missing_images == [] - ), f"Failed to find {missing_images} in {[image.repo for image in built_image.built_images]}" - - built_image.add_tagged_image(repo=name, tags=["latest"]) - mpib.tag_native_platform(built_image) - - # Check that the image was tagged and present - found_image = docker.image.list(filters={"reference": f"{name}:{tag}"}) - assert len(found_image) == 1 - assert f"{name}:{tag}" in found_image[0].repo_tags - - # Check that intermediate images are not on host registry - found_image = docker.image.list( - filters={"reference": f"{built_image.built_images[0].image_ref}*"} - ) - assert len(found_image) == 0 - - # Check that the image is still in host registry - found_image = docker.image.list(filters={"reference": f"{name}:{tag}"}) - assert len(found_image) == 1 - finally: - docker.image.remove(name, force=True) - - -@pytest.mark.serial -def test_push(): - try: - with MultiplatformImageBuilder() as remote_mp: - remote_mp._start_local_registry() - reg_add = remote_mp._build_registry_address() - assert reg_add is not None - - tags = ["latest", "0.1.0"] - build_name = f"{reg_add}/test-push-image-2001" - platforms = ["linux/arm64", "linux/amd64"] - - test_path = f"{TEST_DIR}/test-files/multiplatform" - with MultiplatformImageBuilder() as mpib: - built_image = mpib.build_multiple_images( - platforms=platforms, - path=test_path, - file=f"{test_path}/Dockerfile", - use_threading=False, - ) - - assert built_image is not None - built_image.add_tagged_image(repo=build_name, tags=tags) - mpib.push() - - # Make sure the image isn't in the local registry - docker.image.remove(build_name, force=True) - - # Pull the image from the remote registry to make sure it is there - try: - docker.image.pull(build_name) - except DockerException as err: - assert ( - False - ), f"Failed to find/pull {build_name} from remote registry: {err}" - found_image = docker.image.list(filters={"reference": f"{build_name}"}) - assert len(found_image) == 1 - - finally: - print(f"Cleaning up {build_name}") - docker.image.remove(build_name, force=True) - - -@pytest.mark.serial -def test_push_with_dest_names(): - dest_names = None - try: - with MultiplatformImageBuilder() as remote_mp: - remote_mp._start_local_registry() - reg_add = remote_mp._build_registry_address() - assert reg_add is not None - - tags = ["latest", "0.1.0"] - build_name = "test-push-image-2001" - dest_names = [f"{reg_add}/{build_name}", f"{reg_add}/another-name"] - platforms = ["linux/arm64", "linux/amd64"] - - test_path = f"{TEST_DIR}/test-files/multiplatform" - with MultiplatformImageBuilder() as mpib: - built_image = mpib.build_multiple_images( - platforms=platforms, - path=test_path, - file=f"{test_path}/Dockerfile", - use_threading=False, - ) - - assert built_image is not None - for dest_name in dest_names: - built_image.add_tagged_image(repo=dest_name, tags=tags) - mpib.push() - - # Make sure the image isn't in the local registry - for dest_name in dest_names: - docker.image.remove(dest_name, force=True) - - # Pull the image from the remote registry to make sure it is there - try: - docker.image.pull(dest_name) - except DockerException as err: - assert False, f"Failed to find/pull {dest_name} from remote registry: {err}" - found_image = docker.image.list( - filters={"reference": f"{dest_name}"} - ) - assert len(found_image) == 1 - - finally: - for dest_name in dest_names: - print(f"Cleaning up {dest_name}") - docker.image.remove(dest_name, force=True) - - -@pytest.mark.serial -@pytest.mark.parametrize( - "name, platforms, expected_image_tags", - [ - ( - "test-build-image-2000", - ["linux/arm64"], - ["uuid1-linux-arm64"], - ), - ( - "test-build-image-2001", - ["linux/amd64", "linux/arm64"], - ["uuid1-linux-amd64", "uuid1-linux-arm64"], - ), - ], -) -@patch("buildrunner.docker.multiplatform_image_builder.docker.image.remove") -@patch("buildrunner.docker.multiplatform_image_builder.docker.push") -@patch( - "buildrunner.docker.multiplatform_image_builder.docker.buildx.imagetools.inspect" -) -@patch("buildrunner.docker.multiplatform_image_builder.docker.buildx.build") -def test_build( - mock_build, - mock_imagetools_inspect, - mock_push, - mock_remove, - name, - platforms, - expected_image_tags, -): - _ = mock_build - _ = mock_push - _ = mock_remove - mock_imagetools_inspect.return_value = MagicMock() - mock_imagetools_inspect.return_value.config.digest = "myfakeimageid" - test_path = f"{TEST_DIR}/test-files/multiplatform" - with MultiplatformImageBuilder() as mpib: - built_image = mpib.build_multiple_images( - platforms=platforms, - path=test_path, - file=f"{test_path}/Dockerfile", - use_threading=False, - ) - - assert len(built_image.built_images) == len(platforms) - assert len(built_image.built_images) == len(expected_image_tags) - - missing_images = _actual_images_match_expected(built_image, expected_image_tags) - assert ( - missing_images == [] - ), f"Failed to find {missing_images} in {[image.repo for image in built_image.built_images]}" - - -@pytest.mark.serial -@patch("buildrunner.docker.multiplatform_image_builder.docker.image.remove") -@patch("buildrunner.docker.multiplatform_image_builder.docker.push") -@patch( - "buildrunner.docker.multiplatform_image_builder.docker.buildx.imagetools.inspect" -) -@patch("buildrunner.docker.multiplatform_image_builder.docker.buildx.build") -def test_build_multiple_builds( - mock_build, mock_imagetools_inspect, mock_push, mock_remove -): - _ = mock_remove - mock_imagetools_inspect.return_value = MagicMock() - mock_imagetools_inspect.return_value.config.digest = "myfakeimageid" - platforms1 = ["linux/amd64", "linux/arm64"] - expected_image_tags1 = [ - "uuid1-linux-amd64", - "uuid1-linux-arm64", - ] - - platforms2 = ["linux/amd64", "linux/arm64"] - expected_image_tags2 = [ - "uuid2-linux-amd64", - "uuid2-linux-arm64", - ] - - test_path = f"{TEST_DIR}/test-files/multiplatform" - with MultiplatformImageBuilder() as mpib: - # Build set 1 - built_image1 = mpib.build_multiple_images( - platforms=platforms1, - path=test_path, - file=f"{test_path}/Dockerfile", - use_threading=False, - ) - - # Build set 2 - built_image2 = mpib.build_multiple_images( - platforms=platforms2, - path=test_path, - file=f"{test_path}/Dockerfile", - use_threading=False, - ) - - # Check set 1 - assert len(built_image1.built_images) == len(platforms1) - assert len(built_image1.built_images) == len(expected_image_tags1) - missing_images = _actual_images_match_expected( - built_image1, expected_image_tags1 - ) - assert ( - missing_images == [] - ), f"Failed to find {missing_images} in {[image.repo for image in built_image1.built_images1]}" - - # Check set 2 - assert len(built_image2.built_images) == len(platforms2) - assert len(built_image2.built_images) == len(expected_image_tags2) - missing_images = _actual_images_match_expected( - built_image2, expected_image_tags2 - ) - assert ( - missing_images == [] - ), f"Failed to find {missing_images} in {[image.repo for image in built_image2.built_images]}" - - assert mock_build.call_count == 4 - image_name = mock_build.call_args.kwargs["tags"][0].rsplit(":", 1)[0] - assert mock_build.call_args_list == [ - call( - test_path, - tags=[f"{image_name}:uuid1-linux-amd64"], - platforms=["linux/amd64"], - load=True, - file=f"{test_path}/Dockerfile", - build_args={"DOCKER_REGISTRY": None}, - cache_from=None, - cache_to=None, - stream_logs=True, - ), - call( - test_path, - tags=[f"{image_name}:uuid1-linux-arm64"], - platforms=["linux/arm64"], - load=True, - file=f"{test_path}/Dockerfile", - build_args={"DOCKER_REGISTRY": None}, - cache_from=None, - cache_to=None, - stream_logs=True, - ), - call( - test_path, - tags=[f"{image_name}:uuid2-linux-amd64"], - platforms=["linux/amd64"], - load=True, - file=f"{test_path}/Dockerfile", - build_args={"DOCKER_REGISTRY": None}, - cache_from=None, - cache_to=None, - stream_logs=True, - ), - call( - test_path, - tags=[f"{image_name}:uuid2-linux-arm64"], - platforms=["linux/arm64"], - load=True, - file=f"{test_path}/Dockerfile", - build_args={"DOCKER_REGISTRY": None}, - cache_from=None, - cache_to=None, - stream_logs=True, - ), - ] - assert mock_push.call_count == 4 - assert mock_push.call_args_list == [ - call([f"{image_name}:uuid1-linux-amd64"]), - call([f"{image_name}:uuid1-linux-arm64"]), - call([f"{image_name}:uuid2-linux-amd64"]), - call([f"{image_name}:uuid2-linux-arm64"]), - ] - assert mock_imagetools_inspect.call_count == 4 - assert mock_imagetools_inspect.call_args_list == [ - call(f"{image_name}:uuid1-linux-amd64"), - call(f"{image_name}:uuid1-linux-arm64"), - call(f"{image_name}:uuid2-linux-amd64"), - call(f"{image_name}:uuid2-linux-arm64"), - ] - - -@pytest.mark.serial -@pytest.mark.parametrize( - "builder, cache_builders, return_cache_options", - [ - ("b1", None, True), - ("b1", [], True), - ("b1", ["b1"], True), - ("b2", ["b1"], False), - ], -) -def test__get_build_cache_options(builder, cache_builders, return_cache_options): - multi_platform = MultiplatformImageBuilder( - cache_to="to-loc", - cache_from="from-loc", - cache_builders=cache_builders, - ) - assert multi_platform._get_build_cache_options(builder) == ( - {"cache_to": "to-loc", "cache_from": "from-loc"} if return_cache_options else {} - ) - - -def test_use_build_registry(): - registry_mpib = MultiplatformImageBuilder() - registry_mpib._start_local_registry() - build_registry = registry_mpib._build_registry_address() - try: - with MultiplatformImageBuilder(build_registry=build_registry) as mpib: - # Building should use the registry - test_path = f"{TEST_DIR}/test-files/multiplatform" - built_image = mpib.build_multiple_images( - platforms=["linux/arm64", "linux/amd64"], - path=test_path, - file=f"{test_path}/Dockerfile", - use_threading=False, - ) - assert all( - image_info.repo.startswith(f"{build_registry}/") - for image_info in built_image.images_by_platform.values() - ) - - # Check that the registry is running and only one is found with that name - assert ( - mpib._mp_registry_info is None - ), "The local registry should not have been started when using a build registry" - finally: - registry_mpib._stop_local_registry() - - -@pytest.mark.serial -@pytest.mark.parametrize( - "side_effect, expected_call_count", - [ - (TimeoutError("Timeout"), 5), - ( - [ - None, - TimeoutError("Timeout"), - TimeoutError("Timeout"), - TimeoutError("Timeout"), - TimeoutError("Timeout"), - TimeoutError("Timeout"), - ], - 6, - ), - ( - [ - None, - None, - None, - TimeoutError("Timeout"), - TimeoutError("Timeout"), - TimeoutError("Timeout"), - TimeoutError("Timeout"), - TimeoutError("Timeout"), - ], - 8, - ), - ], -) -@patch("buildrunner.docker.multiplatform_image_builder.PUSH_TIMEOUT", 1) -@patch("buildrunner.docker.multiplatform_image_builder.BuildRunnerConfig") -@patch( - "buildrunner.docker.multiplatform_image_builder.python_on_whales.docker.buildx.imagetools.create" -) -def test_push_retries(mock_docker, mock_config, side_effect, expected_call_count): - mock_docker.side_effect = side_effect - mock_config.get_instance.return_value.global_config.disable_multi_platform = False - - build_name = "test-push-image-2001" - with pytest.raises(TimeoutError): - with MultiplatformImageBuilder() as mpib: - mpib._built_images = [ - BuiltImageInfo( - id="test-id1", - images_by_platform={ - "linux/arm64": BuiltTaggedImage( - repo=build_name, - tag="test", - digest="abcd12345", - platform="linux/arm64", - ), - "linux/amd64": BuiltTaggedImage( - repo=build_name, - tag="test", - digest="abcd12345", - platform="linux/amd64", - ), - }, - tagged_images=[ - TaggedImageInfo( - repo=build_name, - tags=["latest", "0.1.0", "0.1.1", "0.1.2", "test", "test2"], - ), - ], - ) - ] - - mpib.push() - - assert mock_docker.call_count == expected_call_count - - -@pytest.mark.serial -@pytest.mark.parametrize( - "tagged_images, expected_call_count", - [ - ( - [ - TaggedImageInfo( - repo="my-test-image1", - tags=["latest", "0.1.0", "0.1.1", "0.1.2", "test", "test2"], - ) - ], - 6, - ), - ( - [ - TaggedImageInfo( - repo="my-test-image1", - tags=["latest", "0.1.0", "test"], - ), - TaggedImageInfo( - repo="my-test-image2", - tags=[ - "latest", - "0.1.0", - ], - ), - TaggedImageInfo( - repo="my-test-image3", - tags=[ - "latest", - ], - ), - TaggedImageInfo( - repo="my-test-image4", - tags=["latest", "0.1.0", "0.1.1", "0.1.2", "test", "test2"], - ), - ], - 12, - ), - ( - [ - TaggedImageInfo( - repo="my-test-image1", - tags=[ - "latest", - "0.1.0", - "0.1.1", - "0.1.2", - "test", - ], - ), - TaggedImageInfo( - repo="my-test-image2", - tags=[ - "latest", - "0.1.0", - "0.1.1", - "0.1.2", - "test", - ], - ), - TaggedImageInfo( - repo="my-test-image3", - tags=[ - "latest", - "0.1.0", - "0.1.1", - "0.1.2", - "test", - ], - ), - TaggedImageInfo( - repo="my-test-image4", - tags=[ - "latest", - "0.1.0", - "0.1.1", - "0.1.2", - "test", - ], - ), - TaggedImageInfo( - repo="my-test-image5", - tags=[ - "latest", - "0.1.0", - "0.1.1", - "0.1.2", - "test", - ], - ), - TaggedImageInfo( - repo="my-test-image6", - tags=[ - "latest", - "0.1.0", - "0.1.1", - "0.1.2", - "test", - ], - ), - TaggedImageInfo( - repo="my-test-image7", - tags=[ - "latest", - "0.1.0", - "0.1.1", - "0.1.2", - "test", - ], - ), - ], - 35, - ), - ], -) -@patch("buildrunner.docker.multiplatform_image_builder.PUSH_TIMEOUT", 1) -@patch("buildrunner.docker.multiplatform_image_builder.BuildRunnerConfig") -@patch( - "buildrunner.docker.multiplatform_image_builder.python_on_whales.docker.buildx.imagetools.create" -) -def test_push_calls(mock_docker, mock_config, tagged_images, expected_call_count): - mock_docker.side_effect = None - mock_config.get_instance.return_value.global_config.disable_multi_platform = False - - build_name = "test-push-image-2001" - with MultiplatformImageBuilder() as mpib: - mpib._built_images = [ - BuiltImageInfo( - id="test-id1", - images_by_platform={ - "linux/arm64": BuiltTaggedImage( - repo=build_name, - tag="test", - digest="abcd12345", - platform="linux/arm64", - ), - "linux/amd64": BuiltTaggedImage( - repo=build_name, - tag="test", - digest="abcd12345", - platform="linux/amd64", - ), - }, - tagged_images=tagged_images, - ) - ] - - mpib.push() - - assert mock_docker.call_count == expected_call_count diff --git a/tests/test_push_artifact.py b/tests/test_push_artifact.py deleted file mode 100644 index 66fe12c6..00000000 --- a/tests/test_push_artifact.py +++ /dev/null @@ -1,190 +0,0 @@ -import os -import tempfile -import json -import pytest - - -from tests import test_runner - -test_dir_path = os.path.realpath(os.path.dirname(__file__)) -TEST_DIR = os.path.basename(os.path.dirname(__file__)) -top_dir_path = os.path.realpath(os.path.dirname(test_dir_path)) - - -def _test_buildrunner_file( - test_dir, file_name, args, exit_code, artifacts_in_file: dict -): - with tempfile.TemporaryDirectory(prefix="buildrunner.results-") as temp_dir: - command_line = [ - "buildrunner-tester", - "-d", - top_dir_path, - "-b", - temp_dir, - # Since we are using a fresh temp directory, don't delete it first - "--keep-step-artifacts", - "-f", - os.path.join(test_dir, file_name), - ] - if args: - command_line.extend(args) - - assert exit_code == test_runner.run_tests( - command_line, - master_config_file=f"{test_dir_path}/config-files/etc-buildrunner.yaml", - global_config_files=[ - f"{test_dir_path}/config-files/etc-buildrunner.yaml", - f"{test_dir_path}/config-files/dot-buildrunner.yaml", - ], - ) - - artifacts_file = f"{temp_dir}/artifacts.json" - assert os.path.exists(artifacts_file) - with open(artifacts_file, "r") as artifacts_file: - artifacts = json.load(artifacts_file) - - if "build.log" in artifacts.keys(): - del artifacts["build.log"] - - for artifact, is_present in artifacts_in_file.items(): - if is_present: - assert artifact in artifacts.keys() - del artifacts[artifact] - else: - assert artifact not in artifacts.keys() - - assert len(artifacts) == 0 - - -# Test legacy builder -@pytest.mark.parametrize( - "test_name, artifacts_in_file", - [ - ("test-no-artifacts", {}), - ( - "test-no-artifact-properties", - { - "test-no-artifact-properties/test-no-artifact-properties-dir/test1.txt": False, - "test-no-artifact-properties/test-no-artifacts-properties-dir/test2.txt": False, - "test-no-artifact-properties/test-no-artifact-properties.txt": False, - }, - ), - ( - "test-no-push-properties", - { - "test-no-push-properties/test-no-push-properties-dir/test1.txt": True, - "test-no-push-properties/test-no-push-properties-dir/test2.txt": True, - "test-no-push-properties/test-no-push-properties.txt": True, - }, - ), - ( - "test-push-true", - { - "test-push-true/test-push-true-dir/test1.txt": True, - "test-push-true/test-push-true-dir/test2.txt": True, - "test-push-true/test-push-true.txt": True, - }, - ), - ( - "test-push-false", - { - "test-push-false/test-push-false-dir/test1.txt": False, - "test-push-false/test-push-false-dir/test2.txt": False, - "test-push-false/test-push-false.txt": False, - }, - ), - ( - "single-file-rename", - { - "single-file-rename/hello-world.txt": True, - "single-file-rename/hello-world1.txt": True, - "single-file-rename/hello-world2.txt": True, - "single-file-rename/hello.txt": False, - }, - ), - ( - "archive-file-rename", - { - "archive-file-rename/dir1.tar.gz": True, - "archive-file-rename/dir1-dir2.tar.gz": True, - "archive-file-rename/dir3-dir2.tar.gz": True, - "archive-file-rename/dir2.tar.gz": False, - }, - ), - ], -) -def test_artifacts_with_legacy_builder(test_name, artifacts_in_file): - _test_buildrunner_file( - f"{TEST_DIR}/test-files", - "test-push-artifact-legacy.yaml", - ["-s", test_name], - 0, - artifacts_in_file, - ) - - -# Test buildx builder -@pytest.mark.parametrize( - "test_name, artifacts_in_file", - [ - ("test-no-artifacts", {}), - ( - "test-no-artifact-properties", - { - "test-no-artifact-properties/test-no-artifact-properties-dir/test1.txt": False, - "test-no-artifact-properties/test-no-artifacts-properties-dir/test2.txt": False, - "test-no-artifact-properties/test-no-artifact-properties.txt": False, - }, - ), - ( - "test-no-push-properties", - { - "test-no-push-properties/test-no-push-properties-dir/test1.txt": True, - "test-no-push-properties/test-no-push-properties-dir/test2.txt": True, - "test-no-push-properties/test-no-push-properties.txt": True, - }, - ), - ( - "test-push-true", - { - "test-push-true/test-push-true-dir/test1.txt": True, - "test-push-true/test-push-true-dir/test2.txt": True, - "test-push-true/test-push-true.txt": True, - }, - ), - ( - "test-push-false", - { - "test-push-false/test-push-false-dir/test1.txt": False, - "test-push-false/test-push-false-dir/test2.txt": False, - "test-push-false/test-push-false.txt": False, - }, - ), - ( - "single-file-rename", - { - "single-file-rename/hello-world.txt": True, - "single-file-rename/hello-world1.txt": True, - "single-file-rename/hello-world2.txt": True, - "single-file-rename/hello.txt": False, - }, - ), - ( - "archive-file-rename", - { - "archive-file-rename/dir1.tar.gz": True, - "archive-file-rename/dir1-dir2.tar.gz": True, - "archive-file-rename/dir3-dir2.tar.gz": True, - "archive-file-rename/dir2.tar.gz": False, - }, - ), - ], -) -def test_artifacts_with_buildx_builder(test_name, artifacts_in_file): - _test_buildrunner_file( - f"{TEST_DIR}/test-files", - "test-push-artifact-buildx.yaml", - ["-s", test_name], - 0, - artifacts_in_file, - ) diff --git a/tests/test_push_security_scan.py b/tests/test_push_security_scan.py deleted file mode 100644 index cc1a40bd..00000000 --- a/tests/test_push_security_scan.py +++ /dev/null @@ -1,471 +0,0 @@ -import os -from unittest import mock - -import pytest -import yaml - -from buildrunner.config.models import GlobalSecurityScanConfig -from buildrunner.errors import BuildRunnerProcessingError -from buildrunner.steprunner.tasks import push - - -@pytest.fixture(name="config_mock") -def fixture_config_mock(): - with mock.patch( - "buildrunner.steprunner.tasks.push.BuildRunnerConfig" - ) as buildrunner_config_mock: - config_mock = mock.MagicMock() - buildrunner_config_mock.get_instance.return_value = config_mock - yield config_mock - - -def _generate_built_image(num: int) -> mock.MagicMock(): - image = mock.MagicMock() - image.repo = f"repo{num}" - image.tag = f"tag{num}" - image.platform = f"platform{num}" - return image - - -def test__security_scan_mp(): - image_info = mock.MagicMock() - image_info.built_images = [_generate_built_image(num) for num in range(1, 4)] - - self_mock = mock.MagicMock() - self_mock._security_scan.side_effect = ( - lambda **kwargs: None - if kwargs["repository"] == "repo2" - else {"image": kwargs["repository"]} - ) - push_security_scan = mock.MagicMock() - - assert push.PushBuildStepRunnerTask._security_scan_mp( - self_mock, - image_info, - "image_ref1", - push_security_scan, - ) == { - "docker:security-scan": { - "platform1": {"image": "repo1"}, - "platform3": {"image": "repo3"}, - } - } - assert self_mock._security_scan.call_args_list == [ - mock.call( - repository="repo1", - tag="tag1", - log_image_ref="image_ref1:platform1", - pull=True, - push_security_scan=push_security_scan, - ), - mock.call( - repository="repo2", - tag="tag2", - log_image_ref="image_ref1:platform2", - pull=True, - push_security_scan=push_security_scan, - ), - mock.call( - repository="repo3", - tag="tag3", - log_image_ref="image_ref1:platform3", - pull=True, - push_security_scan=push_security_scan, - ), - ] - - -def test__security_scan_mp_empty(): - image_info = mock.MagicMock() - image_info.built_images = [_generate_built_image(num) for num in range(1, 4)] - - self_mock = mock.MagicMock() - self_mock._security_scan.return_value = None - - assert not push.PushBuildStepRunnerTask._security_scan_mp( - self_mock, - image_info, - "image_ref1", - None, - ) - - -@mock.patch( - "buildrunner.steprunner.tasks.push.MultiplatformImageBuilder.get_native_platform" -) -def test__security_scan_single(get_native_platform_mock): - get_native_platform_mock.return_value = "platform1" - self_mock = mock.MagicMock() - self_mock._security_scan.return_value = {"result": True} - push_security_scan = mock.MagicMock() - - assert push.PushBuildStepRunnerTask._security_scan_single( - self_mock, "repo1", "abc123", push_security_scan - ) == {"docker:security-scan": {"platform1": {"result": True}}} - self_mock._security_scan.assert_called_once_with( - repository="repo1", - tag="abc123", - log_image_ref="repo1:abc123", - pull=False, - push_security_scan=push_security_scan, - ) - - -@mock.patch( - "buildrunner.steprunner.tasks.push.MultiplatformImageBuilder.get_native_platform" -) -def test__security_scan_single_empty(get_native_platform_mock): - get_native_platform_mock.return_value = "platform1" - self_mock = mock.MagicMock() - self_mock._security_scan.return_value = None - assert ( - push.PushBuildStepRunnerTask._security_scan_single( - self_mock, "repo1", "abc123", None - ) - == {} - ) - self_mock._security_scan.assert_called_once() - - -def test__security_scan_scanner_disabled(config_mock): - config_mock.global_config.security_scan = GlobalSecurityScanConfig(enabled=False) - self_mock = mock.MagicMock() - assert not push.PushBuildStepRunnerTask._security_scan( - self_mock, - repository="repo1", - tag="tag1", - log_image_ref="image1", - pull=False, - push_security_scan=None, - ) - self_mock._security_scan_trivy.assert_not_called() - - -def test__security_scan_scanner_trivy(config_mock): - security_scan_mock = mock.MagicMock() - merged_config_mock = mock.MagicMock() - config_mock.global_config.security_scan = security_scan_mock - security_scan_mock.merge_scan_config.return_value = merged_config_mock - merged_config_mock.enabled = True - merged_config_mock.scanner = "trivy" - push_security_scan = mock.MagicMock() - - self_mock = mock.MagicMock() - self_mock._security_scan_trivy.return_value = {"result": True} - assert push.PushBuildStepRunnerTask._security_scan( - self_mock, - repository="repo1", - tag="tag1", - log_image_ref="image1", - pull=False, - push_security_scan=push_security_scan, - ) == {"result": True} - self_mock._security_scan_trivy.assert_called_once_with( - security_scan_config=merged_config_mock, - repository="repo1", - tag="tag1", - log_image_ref="image1 (repo1:tag1)", - pull=False, - ) - security_scan_mock.merge_scan_config.assert_called_once_with(push_security_scan) - - -def test__security_scan_scanner_unsupported(config_mock): - config_mock.global_config.security_scan = GlobalSecurityScanConfig( - enabled=True, scanner="bogus" - ) - self_mock = mock.MagicMock() - with pytest.raises(Exception) as exc_info: - assert push.PushBuildStepRunnerTask._security_scan( - self_mock, - repository="repo1", - tag="tag1", - log_image_ref="image1", - pull=False, - push_security_scan=None, - ) == {"result": True} - assert "Unsupported scanner" in str(exc_info.value) - self_mock._security_scan_trivy.assert_not_called() - - -@pytest.mark.parametrize( - "input_results, parsed_results", - [ - ({}, {"max_score": 0, "vulnerabilities": []}), - ({"Results": []}, {"max_score": 0, "vulnerabilities": []}), - ( - {"Results": [{"Vulnerabilities": [{}]}]}, - { - "max_score": 0, - "vulnerabilities": [ - { - "cvss_v3_score": None, - "severity": None, - "vulnerability_id": None, - "pkg_name": None, - "installed_version": None, - "primary_url": None, - } - ], - }, - ), - ( - { - "Results": [ - { - "Vulnerabilities": [ - { - "CVSS": {"nvd": {"V3Score": 1.0}}, - "Severity": "HIGH", - "VulnerabilityID": "CVE1", - "PkgName": "pkg1", - "InstalledVersion": "v1", - "PrimaryURL": "url1", - } - ] - } - ] - }, - { - "max_score": 1.0, - "vulnerabilities": [ - { - "cvss_v3_score": 1.0, - "severity": "HIGH", - "vulnerability_id": "CVE1", - "pkg_name": "pkg1", - "installed_version": "v1", - "primary_url": "url1", - } - ], - }, - ), - ], -) -def test__security_scan_trivy_parse_results(input_results, parsed_results): - security_scan_config = GlobalSecurityScanConfig() - assert ( - push.PushBuildStepRunnerTask._security_scan_trivy_parse_results( - security_scan_config, input_results - ) - == parsed_results - ) - - -@pytest.mark.parametrize( - "max_score_threshold, exception_raised", - [ - (None, False), - (2.11, False), - (2.1, True), - ], -) -def test__security_scan_trivy_parse_results_max_score_threshold( - max_score_threshold, exception_raised -): - security_scan_config = GlobalSecurityScanConfig( - **{"max-score-threshold": max_score_threshold} - ) - input_results = { - "Results": [ - { - "Vulnerabilities": [ - { - "CVSS": {"nvd": {"V3Score": 1.0}}, - }, - { - "CVSS": {"nvd": {"V3Score": None}}, - }, - { - "CVSS": {"nvd": {"V3Score": 0}}, - }, - ], - }, - { - "Vulnerabilities": [ - { - "CVSS": {"nvd": {"V3Score": 2.1}}, - }, - { - "CVSS": {"nvd": {"V3Score": 1.9}}, - }, - ], - }, - ] - } - if exception_raised: - with pytest.raises( - BuildRunnerProcessingError, match="is above the configured threshold" - ): - push.PushBuildStepRunnerTask._security_scan_trivy_parse_results( - security_scan_config, - input_results, - ) - else: - push.PushBuildStepRunnerTask._security_scan_trivy_parse_results( - security_scan_config, input_results - ) - - -@mock.patch("buildrunner.steprunner.tasks.push.DockerRunner") -@mock.patch("buildrunner.steprunner.tasks.push.tempfile") -@mock.patch("buildrunner.steprunner.tasks.push.os") -def test__security_scan_trivy( - os_mock, tempfile_mock, docker_runner_mock, config_mock, tmp_path -): - os_mock.makedirs = os.makedirs - os_mock.path = os.path - os_mock.getuid.return_value = "123" - os_mock.getgid.return_value = "234" - - run_path = tmp_path / "run" - run_path.mkdir() - tempfile_mock.TemporaryDirectory.return_value.__enter__.return_value = str(run_path) - - config_mock.global_config.docker_registry = "registry1" - security_scan_config = GlobalSecurityScanConfig() - self_mock = mock.MagicMock() - self_mock._security_scan_trivy_parse_results.return_value = {"parsed_results": True} - config_mock.global_config.temp_dir = str(tmp_path) - - def _call_run(command, **kwargs): - _ = kwargs - if command.startswith("trivy --config"): - (run_path / "results.json").write_text('{"results": True}') - return 0 - - docker_runner_mock.return_value.run.side_effect = _call_run - - assert push.PushBuildStepRunnerTask._security_scan_trivy( - self_mock, - security_scan_config=security_scan_config, - repository="repo1", - tag="tag1", - log_image_ref="image1", - pull=True, - ) == {"parsed_results": True} - - assert set(path.name for path in tmp_path.iterdir()) == { - "run", - "trivy-cache", - } - assert set(path.name for path in run_path.iterdir()) == { - "config.yaml", - "results.json", - } - assert yaml.safe_load((run_path / "config.yaml").read_text()) == { - "cache-dir": "/root/.cache/trivy", - **security_scan_config.config, - } - - docker_runner_mock.ImageConfig.assert_called_once_with( - "registry1/aquasec/trivy:latest", - pull_image=False, - ) - docker_runner_mock.assert_called_once_with( - docker_runner_mock.ImageConfig.return_value, - log=self_mock.step_runner.log, - ) - docker_runner_mock().start.assert_called_once_with( - entrypoint="/bin/sh", - volumes={ - str(run_path): "/trivy", - str(tmp_path / "trivy-cache"): "/root/.cache/trivy", - "/var/run/docker.sock": "/var/run/docker.sock", - }, - ) - assert docker_runner_mock().run.call_args_list == [ - mock.call("trivy --version", console=self_mock.step_runner.log), - mock.call( - "trivy --config /trivy/config.yaml image -f json -o /trivy/results.json repo1:tag1", - console=self_mock.step_runner.log, - ), - mock.call( - "chown -R 123:234 /trivy", - log=self_mock.step_runner.log, - ), - ] - docker_runner_mock().cleanup.assert_called_once_with() - self_mock._security_scan_trivy_parse_results.assert_called_once_with( - security_scan_config, {"results": True} - ) - - -@mock.patch("buildrunner.steprunner.tasks.push.DockerRunner") -def test__security_scan_trivy_failure(docker_runner_mock, config_mock, tmp_path): - config_mock.global_config.docker_registry = "registry1" - security_scan_config = GlobalSecurityScanConfig() - self_mock = mock.MagicMock() - config_mock.global_config.temp_dir = str(tmp_path) - docker_runner_mock.return_value.run.return_value = 1 - - with pytest.raises(BuildRunnerProcessingError, match="Could not scan"): - push.PushBuildStepRunnerTask._security_scan_trivy( - self_mock, - security_scan_config=security_scan_config, - repository="repo1", - tag="tag1", - log_image_ref="image1", - pull=True, - ) - - docker_runner_mock.ImageConfig.assert_called_once() - docker_runner_mock.assert_called_once() - docker_runner_mock().start.assert_called_once() - assert docker_runner_mock().run.call_count == 3 - docker_runner_mock().cleanup.assert_called_once() - self_mock._security_scan_trivy_parse_results.assert_not_called() - - -@mock.patch("buildrunner.steprunner.tasks.push.DockerRunner") -def test__security_scan_trivy_file_not_created( - docker_runner_mock, config_mock, tmp_path -): - config_mock.global_config.docker_registry = "registry1" - security_scan_config = GlobalSecurityScanConfig() - self_mock = mock.MagicMock() - config_mock.global_config.temp_dir = str(tmp_path) - docker_runner_mock.return_value.run.return_value = 0 - - with pytest.raises(BuildRunnerProcessingError, match="does not exist"): - push.PushBuildStepRunnerTask._security_scan_trivy( - self_mock, - security_scan_config=security_scan_config, - repository="repo1", - tag="tag1", - log_image_ref="image1", - pull=True, - ) - - -@mock.patch("buildrunner.steprunner.tasks.push.DockerRunner") -@mock.patch("buildrunner.steprunner.tasks.push.tempfile") -def test__security_scan_trivy_empty_file( - tempfile_mock, docker_runner_mock, config_mock, tmp_path -): - run_path = tmp_path / "run" - run_path.mkdir() - tempfile_mock.TemporaryDirectory.return_value.__enter__.return_value = str(run_path) - - config_mock.global_config.docker_registry = "registry1" - security_scan_config = GlobalSecurityScanConfig() - self_mock = mock.MagicMock() - config_mock.global_config.temp_dir = str(tmp_path) - - def _call_run(command, **kwargs): - _ = kwargs - if command.startswith("trivy --config"): - (run_path / "results.json").write_text("{}") - return 0 - - docker_runner_mock.return_value.run.side_effect = _call_run - - with pytest.raises(BuildRunnerProcessingError, match="Could not read results file"): - push.PushBuildStepRunnerTask._security_scan_trivy( - self_mock, - security_scan_config=security_scan_config, - repository="repo1", - tag="tag1", - log_image_ref="image1", - pull=True, - ) diff --git a/tests/test_util_checksum.py b/tests/test_util_checksum.py deleted file mode 100644 index c58394dc..00000000 --- a/tests/test_util_checksum.py +++ /dev/null @@ -1,21 +0,0 @@ -import os -from buildrunner.utils import checksum - -test_dir_path = os.path.realpath(os.path.dirname(__file__)) - - -def test_checksum(): - with open(f"{test_dir_path}/test-files/checksum/test1.test2.sha1") as checksum_file: - expected_checksum = checksum_file.read().strip() - - checksum_value = checksum( - f"{test_dir_path}/test-files/checksum/test1.txt", - f"{test_dir_path}/test-files/checksum/test2.txt", - ) - assert expected_checksum == checksum_value - - checksum_value = checksum( - f"{test_dir_path}/test-files/checksum/test2.txt", - f"{test_dir_path}/test-files/checksum/test1.txt", - ) - assert expected_checksum == checksum_value diff --git a/tests/test_utils_flock.py b/tests/test_utils_flock.py deleted file mode 100644 index 402d17a2..00000000 --- a/tests/test_utils_flock.py +++ /dev/null @@ -1,265 +0,0 @@ -from multiprocessing import Process, Event -import time -from unittest import mock - -import pytest -from buildrunner.utils import ( - FailureToAcquireLockException, - acquire_flock_open_read_binary, - acquire_flock_open_write_binary, - release_flock, -) - - -@pytest.fixture(name="mock_logger") -def fixture_mock_logger(): - mock_logger = mock.MagicMock() - mock_logger.info.side_effect = lambda message: print(f"[info] {message}") - mock_logger.warning.side_effect = lambda message: print(f"[warning] {message}") - return mock_logger - - -def _get_and_hold_lock( - ready_event: Event, done_event: Event, lock_file, exclusive=True, timeout_seconds=2 -): - fd = None - mock_logger = mock.MagicMock() - mock_logger.info.side_effect = lambda message: print( - f"[get-and-hold-info] {message}" - ) - mock_logger.warning.side_effect = lambda message: print( - f"[get-and-hold-warning] {message}" - ) - try: - if exclusive: - fd = acquire_flock_open_write_binary( - lock_file=lock_file, logger=mock_logger, timeout_seconds=timeout_seconds - ) - else: - fd = acquire_flock_open_read_binary( - lock_file=lock_file, logger=mock_logger, timeout_seconds=timeout_seconds - ) - assert fd is not None - print( - f"Acquired lock for file {lock_file} in background process (exclusive={exclusive})" - ) - ready_event.set() - done_event.wait() - finally: - release_flock(fd, mock_logger) - - -def _wait_and_set(event: Event, sleep_seconds: float): - time.sleep(sleep_seconds) - event.set() - - -def test_flock_acquire1(mock_logger, tmp_path): - try: - fd = None - lock_file = str(tmp_path / "mylock.file") - fd = acquire_flock_open_write_binary( - lock_file=lock_file, logger=mock_logger, timeout_seconds=1.0 - ) - assert fd is not None - - finally: - if fd: - release_flock(fd, mock_logger) - - -def test_flock_exclusive_acquire(mock_logger, tmp_path): - try: - lock_file = str(tmp_path / "mylock.file") - fd = None - - # Test exclusive followed by exclusive acquire - ready_event = Event() - done_event = Event() - p = Process( - target=_get_and_hold_lock, args=(ready_event, done_event, lock_file) - ) - p.start() - ready_event.wait() - - with pytest.raises(FailureToAcquireLockException): - fd = acquire_flock_open_write_binary( - lock_file, mock_logger, timeout_seconds=1.0 - ) - assert fd is None - done_event.set() - p.join() - - # Test shared followed by exclusive acquire - ready_event.clear() - done_event.clear() - p = Process( - target=_get_and_hold_lock, args=(ready_event, done_event, lock_file, False) - ) - p.start() - ready_event.wait() - - with pytest.raises(FailureToAcquireLockException): - fd = acquire_flock_open_write_binary( - lock_file, mock_logger, timeout_seconds=1.0 - ) - assert fd is None - done_event.set() - p.join() - - # Test exclusive followed by shared acquire - ready_event.clear() - done_event.clear() - p = Process( - target=_get_and_hold_lock, args=(ready_event, done_event, lock_file, True) - ) - p.start() - ready_event.wait() - - with pytest.raises(FailureToAcquireLockException): - fd = acquire_flock_open_read_binary( - lock_file, mock_logger, timeout_seconds=1.0 - ) - assert fd is None - done_event.set() - p.join() - - finally: - if fd: - release_flock(fd, mock_logger) - - -def test_flock_release(mock_logger, tmp_path): - fd = None - try: - lock_file = str(tmp_path / "mylock.file") - - ready_event = Event() - done_event = Event() - p = Process( - target=_get_and_hold_lock, args=(ready_event, done_event, lock_file) - ) - p.start() - ready_event.wait() - p2 = Process(target=_wait_and_set, args=(done_event, 1.0)) - p2.start() - - fd = acquire_flock_open_write_binary( - lock_file, mock_logger, timeout_seconds=5.0 - ) - assert fd is not None - p.join() - p2.join() - finally: - if fd: - release_flock(fd, mock_logger) - - -def test_flock_acquire_exclusive_timeout(mock_logger, tmp_path): - try: - lock_file = str(tmp_path / "mylock.file") - fd = None - - # Test exclusive followed by exclusive acquire - ready_event = Event() - done_event = Event() - p = Process( - target=_get_and_hold_lock, args=(ready_event, done_event, lock_file) - ) - p.start() - ready_event.wait() - - timeout_seconds = 5 - start_time = time.time() - with pytest.raises(FailureToAcquireLockException): - fd = acquire_flock_open_write_binary( - lock_file, mock_logger, timeout_seconds=timeout_seconds - ) - duration_seconds = time.time() - start_time - tolerance_seconds = 0.6 - assert ( - (timeout_seconds - tolerance_seconds) - <= duration_seconds - <= (timeout_seconds + tolerance_seconds) - ) - assert fd is None - done_event.set() - p.join() - - # Test exclusive followed by shared acquire - ready_event.clear() - done_event.clear() - p = Process( - target=_get_and_hold_lock, args=(ready_event, done_event, lock_file) - ) - p.start() - ready_event.wait() - - timeout_seconds = 5 - start_time = time.time() - with pytest.raises(FailureToAcquireLockException): - fd = acquire_flock_open_read_binary( - lock_file, mock_logger, timeout_seconds=timeout_seconds - ) - duration_seconds = time.time() - start_time - tolerance_seconds = 0.6 - assert ( - (timeout_seconds - tolerance_seconds) - <= duration_seconds - <= (timeout_seconds + tolerance_seconds) - ) - assert fd is None - done_event.set() - p.join() - - # Test shared followed by exclusive acquire - ready_event.clear() - done_event.clear() - p = Process( - target=_get_and_hold_lock, args=(ready_event, done_event, lock_file, False) - ) - p.start() - ready_event.wait() - - timeout_seconds = 5 - start_time = time.time() - with pytest.raises(FailureToAcquireLockException): - fd = acquire_flock_open_write_binary( - lock_file, mock_logger, timeout_seconds=timeout_seconds - ) - duration_seconds = time.time() - start_time - tolerance_seconds = 0.6 - assert ( - (timeout_seconds - tolerance_seconds) - <= duration_seconds - <= (timeout_seconds + tolerance_seconds) - ) - assert fd is None - done_event.set() - p.join() - finally: - if fd: - release_flock(fd, mock_logger) - - -def test_flock_shared_acquire(mock_logger, tmp_path): - try: - lock_file = str(tmp_path / "mylock.file") - # Lock file needs to be created - with open(lock_file, "w") as file: - file.write("Test file.") - fd = None - ready_event = Event() - done_event = Event() - p = Process( - target=_get_and_hold_lock, args=(ready_event, done_event, lock_file, False) - ) - p.start() - ready_event.wait() - fd = acquire_flock_open_read_binary(lock_file, mock_logger, timeout_seconds=1.0) - assert fd is not None - done_event.set() - p.join() - finally: - if fd: - release_flock(fd, mock_logger) diff --git a/tests/test_version.py b/tests/test_version.py deleted file mode 100644 index f33f31d5..00000000 --- a/tests/test_version.py +++ /dev/null @@ -1,80 +0,0 @@ -from collections import OrderedDict - -import pytest -from buildrunner.config import loader -from buildrunner.config.loader import ( - BuildRunnerVersionError, - ConfigVersionFormatError, - ConfigVersionTypeError, -) - -buildrunner_version = "2.0.701" -config_version = "2.0" - - -@pytest.fixture(name="config") -def fixture_config_file(): - config = OrderedDict({"version": config_version}) - yield config - - -@pytest.fixture(name="version_file", autouse=True) -def fixture_setup_version_file(tmp_path): - version_file = tmp_path / "version.py" - version_file.write_text(f"__version__ = '{buildrunner_version}'") - original_path = loader.VERSION_FILE_PATH - loader.VERSION_FILE_PATH = str(version_file) - yield str(version_file) - loader.VERSION_FILE_PATH = original_path - - -def test_valid_version_file(config): - loader._validate_version(config=config) - - -def test_missing_version_file(config): - # No exception for a missing version file it just prints a warning - loader.VERSION_FILE_PATH = "bogus" - loader._validate_version(config=config) - - -def test_missing_version_in_version_file(config, version_file): - with open(version_file, "w") as file: - file.truncate() - - with pytest.raises(BuildRunnerVersionError): - loader._validate_version(config=config) - - -def test_invalid_delim_version(config, version_file): - with open(version_file, "w") as file: - file.truncate() - file.write("__version__: '1.3.4'") - - with pytest.raises(ConfigVersionFormatError): - loader._validate_version(config=config) - - -def test_invalid_config_number_version(config, version_file): - with open(version_file, "w") as file: - file.truncate() - file.write("__version__ = '1'") - - with pytest.raises(ConfigVersionFormatError): - loader._validate_version(config=config) - - -def test_invalid_config_version_type(config, version_file): - with open(version_file, "w") as file: - file.truncate() - file.write("__version__ = 'two.zero.five'") - - with pytest.raises(ConfigVersionTypeError): - loader._validate_version(config=config) - - -def test_bad_version(config): - config = OrderedDict({"version": 2.1}) - - with pytest.raises(ConfigVersionFormatError): - loader._validate_version(config=config) diff --git a/tests/webservicecontainer/Dockerfile b/tests/webservicecontainer/Dockerfile deleted file mode 100644 index 5147821a..00000000 --- a/tests/webservicecontainer/Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -ARG DOCKER_REGISTRY -FROM $DOCKER_REGISTRY/rockylinux:8.5 -RUN \ - set -eux; \ - yum clean all; \ - yum -y install python3-setuptools python3-pip; \ - rm -rf /var/cache/yum -RUN pip3 install bottle pika -COPY . /root -EXPOSE 8080 -CMD ["/root/testserver.py"] diff --git a/tests/webservicecontainer/testserver.py b/tests/webservicecontainer/testserver.py deleted file mode 100755 index 17250a12..00000000 --- a/tests/webservicecontainer/testserver.py +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env python3 - -from bottle import route, run, HTTPError -import pika - -EXCHANGE = "test" - - -@route("/") -def index(message): - connection = pika.BlockingConnection( - pika.ConnectionParameters( - host="rabbitmq", - ) - ) - channel = connection.channel() - - channel.exchange_declare( - exchange=EXCHANGE, - exchange_type="topic", - ) - - channel.queue_declare(queue="message") - - channel.queue_bind( - queue="message", - exchange=EXCHANGE, - routing_key="#", - ) - - channel.basic_publish( - exchange=EXCHANGE, - routing_key="hello", - body=message, - ) - - method_frame, _, message_body = channel.basic_get( - queue="message", - ) - if not method_frame or method_frame.NAME == "Basic.GetEmpty": - raise HTTPError(500) - - return message_body - - -run(host="0.0.0.0", port=8080)