diff --git a/.semaphore/cp_dockerfile_build.yml b/.semaphore/cp_dockerfile_build.yml new file mode 100644 index 000000000..5feecf1a3 --- /dev/null +++ b/.semaphore/cp_dockerfile_build.yml @@ -0,0 +1,241 @@ +# This file is managed by ServiceBot plugin - Semaphore. The content in this file is created using a common +# template and configurations in service.yml. +# Any modifications made to ths file will be overwritten by the generated content in nightly runs. +# For more information, please refer to the page: +# https://confluentinc.atlassian.net/wiki/spaces/Foundations/pages/2871296194/Add+SemaphoreCI +version: v1.0 +name: build-test-release +agent: + machine: + type: s1-prod-ubuntu20-04-amd64-1 + +fail_fast: + cancel: + when: "true" + +execution_time_limit: + hours: 1 + +queue: + - when: "branch != 'master' and branch !~ '[0-9]+\\.[0-9]+\\.[0-9]+'" + processing: parallel + +global_job_config: + prologue: + commands: + - checkout + - sem-version java 8 + - . vault-setup + - . cache-maven restore + - export GIT_COMMIT=$(git rev-parse --verify HEAD --short) + - export BUILD_NUMBER=$(echo $SEMAPHORE_WORKFLOW_ID | cut -f1 -d"-") + - export BRANCH_TAG=$(echo $SEMAPHORE_GIT_BRANCH | tr / -) + # Semaphore does not accept empty values for parameters. + - if [[ "$PACKAGES_MAVEN_URL" = "NONE" ]]; then unset PACKAGES_MAVEN_URL; fi + - if [[ "$PACKAGING_BUILD_NUMBER" = "NONE" ]]; then unset PACKAGING_BUILD_NUMBER; fi + - export ALLOW_UNSIGNED=$(echo $ALLOW_UNSIGNED | awk '{ print tolower($0) }') + - >- + echo """ + Parameters: + ========================== + CONFLUENT_VERSION: $CONFLUENT_VERSION + PACKAGES_URL: $PACKAGES_URL + PACKAGES_MAVEN_URL: $PACKAGES_MAVEN_URL + PACKAGING_BUILD_NUMBER: $PACKAGING_BUILD_NUMBER + ALLOW_UNSIGNED: $ALLOW_UNSIGNED + CONFLUENT_DEB_VERSION: $CONFLUENT_DEB_VERSION + """ + - if [[ $BRANCH_TAG =~ "-rc" ]]; then export IS_RC="true"; fi + - if [[ $BRANCH_TAG =~ "-cp" ]]; then export IS_HOTFIX="true"; fi + - if [[ $BRANCH_TAG =~ "-post" ]]; then export IS_POST="true"; fi + - if [[ $BRANCH_TAG =~ "-beta" ]]; then export IS_BETA="true"; fi + - if [[ $BRANCH_TAG =~ "-alpha" ]]; then export IS_PREVIEW="true"; fi + - if [[ $IS_RC || $IS_HOTFIX || $IS_POST || $IS_BETA || $IS_PREVIEW ]]; then export IS_RELEASE="true"; fi + + - >- + if [[ $IS_BETA || $IS_HOTFIX || $IS_POST ]]; then + export PLATFORM_LABEL=$(echo $BRANCH_TAG | awk -F - '{print "~"$2}') + elif [[ ! $IS_RELEASE ]]; then + export PLATFORM_LABEL="~SNAPSHOT" + else + export PLATFORM_LABEL="" + fi + - export PACKAGING_BUILD_ARGS=" -DCONFLUENT_VERSION=$CONFLUENT_VERSION -DCONFLUENT_PLATFORM_LABEL=$PLATFORM_LABEL -DCONFLUENT_DEB_VERSION=$CONFLUENT_DEB_VERSION -DALLOW_UNSIGNED=$ALLOW_UNSIGNED" + - >- + if [[ $IS_RELEASE && $PACKAGING_BUILD_NUMBER ]]; then + if [[ $IS_BETA ]]; then + export MAVEN_PACKAGES_URL="https://s3.us-west-2.amazonaws.com/jenkins-confluent-packages-beta-maven/$BRANCH_TAG/$PACKAGING_BUILD_NUMBER/maven" + elif [[ $IS_PREVIEW ]]; then + export MAVEN_PACKAGES_URL="https://s3.us-west-2.amazonaws.com/jenkins-confluent-packages-alpha-maven/$BRANCH_NAME/$PACKAGING_BUILD_NUMBER/maven" + elif [[ $IS_RC ]]; then + export MAVEN_PACKAGES_URL="https://s3.us-west-2.amazonaws.com/staging-confluent-packages-maven/v$BRANCH_NAME/maven" + if [[ $PACKAGES_MAVEN_URL ]]; then + export MAVEN_PACKAGES_URL=$PACKAGES_MAVEN_URL + fi + fi + # Overwrite maven global configuration + . vault-sem-get-secret maven-settings-cp-dockerfile + else + echo "This job is not a isBetaJob, isPreviewJob, isHotfixJob, or isRcJob (What we know how to handle) - and we don't know how to handle it" + fi + - export DOCKER_DEV_REGISTRY="519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/dev/" + - export DOCKER_PROD_REGISTRY="519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/prod/" + - export DOCKER_UPSTREAM_REGISTRY=$DOCKER_PROD_REGISTRY + - export LATEST_TAG=$BRANCH_TAG-latest + - export DOCKER_UPSTREAM_TAG="$LATEST_TAG" + - export DOCKER_REPOS="unsynced-confluentinc/cp-kafka-mqtt" + - export DOCKER_DEV_TAG="dev-$BRANCH_TAG-$BUILD_NUMBER" + - export AMD_ARCH=.amd64 + - export ARM_ARCH=.arm64 +blocks: + - name: Build, Test, & Scan AMD + dependencies: [] + run: + # don't run the tests on non-functional changes... + when: "change_in('/', {exclude: ['/.deployed-versions/', '.github/']})" + task: + jobs: + - name: Build, Test, & Scan ubi8 + commands: + - export OS_TAG="-ubi8" + - export DOCKER_DEV_FULL_IMAGES=$DOCKER_DEV_REGISTRY${DOCKER_REPOS// /:$DOCKER_DEV_TAG$OS_TAG $DOCKER_DEV_REGISTRY}:$DOCKER_DEV_TAG$OS_TAG + - export AMD_DOCKER_DEV_FULL_IMAGES=${DOCKER_DEV_FULL_IMAGES// /$AMD_ARCH }$AMD_ARCH + - ci-tools ci-update-version + - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") + - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" + - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean package dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY + -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi8 + $PACKAGING_BUILD_ARGS + - . cache-maven store + - >- + for dev_image in $AMD_DOCKER_DEV_FULL_IMAGES; + do + cve-scan $dev_image + done + - for image in $AMD_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done + epilogue: + always: + commands: + - . publish-test-results + - artifact push workflow target/test-results + - artifact push workflow target --destination target-AMD + - name: Deploy AMD unsynced-confluentinc/cp-kafka-mqtt + dependencies: ["Build, Test, & Scan AMD"] + run: + when: "branch = 'master' or branch =~ '[0-9]+\\.[0-9]+\\.[0-9]+'" + task: + jobs: + - name: Deploy AMD unsynced-confluentinc/cp-kafka-mqtt ubi8 + commands: + - export OS_TAG="-ubi8" + - export PROD_IMAGE_NAME=${DOCKER_PROD_REGISTRY}unsynced-confluentinc/cp-kafka-mqtt + - export GIT_COMMIT_TAG=$GIT_COMMIT$OS_TAG$AMD_ARCH + - export BRANCH_BUILD_TAG=$BRANCH_TAG-$BUILD_NUMBER$OS_TAG$AMD_ARCH + - export DEV_IMAGE_FULL=${DOCKER_DEV_REGISTRY}unsynced-confluentinc/cp-kafka-mqtt:$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH + - docker pull $DEV_IMAGE_FULL + - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$GIT_COMMIT_TAG + - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG + - docker push $PROD_IMAGE_NAME:$GIT_COMMIT_TAG + - docker push $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG + - export PACKAGE_TAG=$BRANCH_TAG-$PACKAGING_BUILD_NUMBER$OS_TAG$AMD_ARCH + - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$PACKAGE_TAG + - docker push $PROD_IMAGE_NAME:$PACKAGE_TAG + - name: Build & Test ARM + dependencies: [] + run: + # don't run the tests on non-functional changes... + when: "change_in('/', {exclude: ['/.deployed-versions/', '.github/']})" + task: + agent: + machine: + type: s1-prod-ubuntu20-04-arm64-1 + jobs: + - name: Build & Test ubi8 + commands: + - export OS_TAG="-ubi8" + - export DOCKER_DEV_FULL_IMAGES=$DOCKER_DEV_REGISTRY${DOCKER_REPOS// /:$DOCKER_DEV_TAG$OS_TAG $DOCKER_DEV_REGISTRY}:$DOCKER_DEV_TAG$OS_TAG + - export ARM_DOCKER_DEV_FULL_IMAGES=${DOCKER_DEV_FULL_IMAGES// /$ARM_ARCH }$ARM_ARCH + - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") + - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" + - ci-tools ci-update-version + - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean package dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY + -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi8 + $PACKAGING_BUILD_ARGS + - . cache-maven store + - for image in $ARM_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done + epilogue: + always: + commands: + - . publish-test-results + - artifact push workflow target/test-results + - artifact push workflow target --destination target-ARM + - name: Deploy ARM unsynced-confluentinc/cp-kafka-mqtt + dependencies: ["Build & Test ARM"] + run: + when: "branch = 'master' or branch =~ '[0-9]+\\.[0-9]+\\.[0-9]+'" + task: + agent: + machine: + type: s1-prod-ubuntu20-04-arm64-1 + jobs: + - name: Deploy ARM unsynced-confluentinc/cp-kafka-mqtt ubi8 + commands: + - export OS_TAG="-ubi8" + - export PROD_IMAGE_NAME=${DOCKER_PROD_REGISTRY}unsynced-confluentinc/cp-kafka-mqtt + - export GIT_COMMIT_TAG=$GIT_COMMIT$OS_TAG$ARM_ARCH + - export BRANCH_BUILD_TAG=$BRANCH_TAG-$BUILD_NUMBER$OS_TAG$ARM_ARCH + - export DEV_IMAGE_FULL=${DOCKER_DEV_REGISTRY}unsynced-confluentinc/cp-kafka-mqtt:$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH + - docker pull $DEV_IMAGE_FULL + - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$GIT_COMMIT_TAG + - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG + - docker push $PROD_IMAGE_NAME:$GIT_COMMIT_TAG + - docker push $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG + - export PACKAGE_TAG=$BRANCH_TAG-$PACKAGING_BUILD_NUMBER$OS_TAG$ARM_ARCH + - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$PACKAGE_TAG + - docker push $PROD_IMAGE_NAME:$PACKAGE_TAG + - name: Create Manifest and Maven Deploy + dependencies: ["Deploy AMD unsynced-confluentinc/cp-kafka-mqtt", "Deploy ARM unsynced-confluentinc/cp-kafka-mqtt"] + run: + when: "branch = 'master' or branch =~ '[0-9]+\\.[0-9]+\\.[0-9]+'" + task: + jobs: + - name: Create Manifest and Maven Deploy + commands: + - export DOCKER_PROD_IMAGE_NAME=$DOCKER_PROD_REGISTRY${DOCKER_REPOS// / $DOCKER_PROD_REGISTRY} + - ci-tools ci-update-version + - ci-tools ci-push-tag + - echo "Skipping Maven Deploy" + # Create manifest + - >- + for image in $DOCKER_PROD_IMAGE_NAME; + do + export OS_TAG="-ubi8" + export GIT_TAG=$GIT_COMMIT$OS_TAG + docker manifest create $image:$GIT_TAG $image:$GIT_TAG$AMD_ARCH $image:$GIT_TAG$ARM_ARCH + docker manifest push $image:$GIT_TAG + export BRANCH_BUILD_TAG=$BRANCH_TAG-$BUILD_NUMBER$OS_TAG + docker manifest create $image:$BRANCH_BUILD_TAG $image:$BRANCH_BUILD_TAG$AMD_ARCH $image:$BRANCH_BUILD_TAG$ARM_ARCH + docker manifest push $image:$BRANCH_BUILD_TAG + export PACKAGE_TAG=$BRANCH_TAG-$PACKAGING_BUILD_NUMBER$OS_TAG + docker manifest create $image:$PACKAGE_TAG $image:$PACKAGE_TAG$AMD_ARCH $image:$PACKAGE_TAG$ARM_ARCH + docker manifest push $image:$PACKAGE_TAG + done +after_pipeline: + task: + agent: + machine: + type: s1-prod-ubuntu20-04-arm64-0 + jobs: + - name: Metrics + commands: + - emit-ci-metrics -p -a test-results + - name: Publish Test Results + commands: + - test-results gen-pipeline-report + - name: SonarQube + commands: + - checkout + - sem-version java 11 + - artifact pull workflow target-AMD + - artifact pull workflow target-ARM + - emit-sonarqube-data --run_only_sonar_scan diff --git a/.semaphore/cp_dockerfile_promote.yml b/.semaphore/cp_dockerfile_promote.yml new file mode 100644 index 000000000..eb7dccf09 --- /dev/null +++ b/.semaphore/cp_dockerfile_promote.yml @@ -0,0 +1,142 @@ +# This file is managed by ServiceBot plugin - Semaphore. The content in this file is created using a common +# template and configurations in service.yml. +# Any modifications made to ths file will be overwritten by the generated content in nightly runs. +# For more information, please refer to the page: +# https://confluentinc.atlassian.net/wiki/spaces/Foundations/pages/2871296194/Add+SemaphoreCI +version: v1.0 +name: build-test-release +agent: + machine: + type: s1-prod-ubuntu20-04-amd64-1 + +fail_fast: + cancel: + when: "true" + +execution_time_limit: + hours: 1 + +queue: + - when: "branch != 'master' and branch !~ '[0-9]+\\.[0-9]+\\.[0-9]+'" + processing: parallel + +global_job_config: + prologue: + commands: + - checkout + - . vault-setup + # Semaphore does not accept empty values for parameters. + - if [[ "$CONFLUENT_VERSION" = "NONE" ]]; then echo "Must specify CONFLUENT_VERSION" && exit 1; fi + - if [[ "$IMAGE_REVISION" = "NONE" ]]; then echo "Must specify IMAGE_REVISION" && exit 1; fi + - if [[ "$UPDATE_LATEST_TAG" = "NONE" ]]; then unset UPDATE_LATEST_TAG; fi + - if [[ "$PACKAGING_BUILD_NUMBER" = "NONE" ]]; then unset PACKAGING_BUILD_NUMBER; fi + - if [[ "$PROMOTE_OS_TYPE" = "NONE" ]]; then echo "Must specify PROMOTE_OS_TYPE" && exit 1; fi + - >- + echo """ + Parameters: + ========================== + CONFLUENT_VERSION: $CONFLUENT_VERSION + IMAGE_REVISION: $IMAGE_REVISION + UPDATE_LATEST_TAG: $UPDATE_LATEST_TAG + PACKAGING_BUILD_NUMBER: $PACKAGING_BUILD_NUMBER + PROMOTE_OS_TYPE: $PROMOTE_OS_TYPE + """ + - export BRANCH_TAG=$(echo $SEMAPHORE_GIT_BRANCH | tr / -) + - export DOCKER_PROD_REGISTRY="519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/prod/" + - export PROMOTED_TAG_PREFIX="$CONFLUENT_VERSION-$IMAGE_REVISION" + - if [[ ! "$PACKAGING_BUILD_NUMBER" ]]; then export $PACKAGING_BUILD_NUMBER="latest"; fi + - docker login --username $DOCKERHUB_USER --password $DOCKERHUB_APIKEY + - export AMD_ARCH=.amd64 + - export ARM_ARCH=.arm64 + +blocks: + - name: Promote AMD + dependencies: [] + task: + jobs: + - name: Promote unsynced-confluentinc/cp-kafka-mqtt ubi8 AMD + commands: + - export OS_TYPE="ubi8" + - export DOCKER_REPO="unsynced-confluentinc/cp-kafka-mqtt" + - if [[ "$OS_TYPE" =~ "$PROMOTE_OS_TYPE*" || ! "$OS_TYPE" ]]; then export OS_TAG=""; else export OS_TAG="-$OS_TYPE"; fi + - export INTERNAL_IMAGE_TAG="$BRANCH_TAG-$PACKAGING_BUILD_NUMBER$OS_TAG" + - export PROMOTED_TAG="$PROMOTED_TAG_PREFIX$OS_TAG$AMD_ARCH" + - docker pull $DOCKER_PROD_REGISTRY$DOCKER_REPO:$INTERNAL_IMAGE_TAG + - docker tag $DOCKER_PROD_REGISTRY$DOCKER_REPO:$INTERNAL_IMAGE_TAG $DOCKER_REPO:$PROMOTED_TAG + - docker run --rm $DOCKER_REPO:$PROMOTED_TAG sh -c "grep staging /etc/yum.repos.d/confluent.repo || grep staging /etc/apt/sources.list" || export STAGING_CHECK_SUCCEED="true" + - if [[ ! "$STAGING_CHECK_SUCCEED" ]]; then echo "Detected there was a staging repo in image $DOCKER_REPO:$PROMOTED_TAG refusing to promote." && exit 1; fi + - docker push $DOCKER_REPO:$PROMOTED_TAG + - >- + if [[ ! "$OS_TYPE" ]] || [[ "$OS_TYPE" =~ "ubi*" ]]; then + export APPLY_TAG=$CONFLUENT_VERSION$AMD_ARCH + docker tag $DOCKER_PROD_REGISTRY$DOCKER_REPO:$INTERNAL_IMAGE_TAG $DOCKER_REPO:$APPLY_TAG + docker push $DOCKER_REPO:$APPLY_TAG + export APPLIED="true" + fi + - >- + if [[ $UPDATE_LATEST_TAG ]]; then + if [[ $APPLIED ]]; then + docker tag $DOCKER_REPO:$PROMOTED_TAG $DOCKER_REPO:latest$AMD_ARCH + docker push $DOCKER_REPO:latest$AMD_ARCH + fi + docker tag $DOCKER_REPO:$PROMOTED_TAG $DOCKER_REPO:latest-$OS_TYPE$AMD_ARCH + docker push $DOCKER_REPO:latest-$OS_TYPE$AMD_ARCH + fi + - name: Promote ARM + dependencies: [] + task: + jobs: + - name: Promote unsynced-confluentinc/cp-kafka-mqtt ubi8 ARM + commands: + - export OS_TYPE="ubi8" + - export DOCKER_REPO="unsynced-confluentinc/cp-kafka-mqtt" + - if [[ "$OS_TYPE" =~ "$PROMOTE_OS_TYPE*" || ! "$OS_TYPE" ]]; then export OS_TAG=""; else export OS_TAG="-$OS_TYPE"; fi + - export INTERNAL_IMAGE_TAG="$BRANCH_TAG-$PACKAGING_BUILD_NUMBER$OS_TAG" + - export PROMOTED_TAG="$PROMOTED_TAG_PREFIX$OS_TAG$ARM_ARCH" + - docker pull $DOCKER_PROD_REGISTRY$DOCKER_REPO:$INTERNAL_IMAGE_TAG + - docker tag $DOCKER_PROD_REGISTRY$DOCKER_REPO:$INTERNAL_IMAGE_TAG $DOCKER_REPO:$PROMOTED_TAG + - docker run --rm $DOCKER_REPO:$PROMOTED_TAG sh -c "grep staging /etc/yum.repos.d/confluent.repo || grep staging /etc/apt/sources.list" || export STAGING_CHECK_SUCCEED="true" + - if [[ ! "$STAGING_CHECK_SUCCEED" ]]; then echo "Detected there was a staging repo in image $DOCKER_REPO:$PROMOTED_TAG refusing to promote." && exit 1; fi + - docker push $DOCKER_REPO:$PROMOTED_TAG + - >- + if [[ ! "$OS_TYPE" ]] || [[ "$OS_TYPE" =~ "ubi*" ]]; then + export APPLY_TAG=$CONFLUENT_VERSION$ARM_ARCH + docker tag $DOCKER_PROD_REGISTRY$DOCKER_REPO:$INTERNAL_IMAGE_TAG $DOCKER_REPO:$APPLY_TAG + docker push $DOCKER_REPO:$APPLY_TAG + export APPLIED="true" + fi + - >- + if [[ $UPDATE_LATEST_TAG ]]; then + if [[ $APPLIED ]]; then + docker tag $DOCKER_REPO:$PROMOTED_TAG $DOCKER_REPO:latest$ARM_ARCH + docker push $DOCKER_REPO:latest$ARM_ARCH + fi + docker tag $DOCKER_REPO:$PROMOTED_TAG $DOCKER_REPO:latest-$OS_TYPE$ARM_ARCH + docker push $DOCKER_REPO:latest-$OS_TYPE$ARM_ARCH + fi + - name: Create Manifest + dependencies: ["Promote AMD", "Promote ARM"] + task: + jobs: + - name: Create Manifest unsynced-confluentinc/cp-kafka-mqtt ubi8 + commands: + - export OS_TYPE="ubi8" + - export DOCKER_REPO="unsynced-confluentinc/cp-kafka-mqtt" + - export PROMOTED_TAG="$PROMOTED_TAG_PREFIX$OS_TAG" + - docker manifest create $DOCKER_REPO:$PROMOTED_TAG $DOCKER_REPO:$PROMOTED_TAG$AMD_ARCH $DOCKER_REPO:$PROMOTED_TAG$ARM_ARCH + - docker manifest push $DOCKER_REPO:$PROMOTED_TAG + - >- + if [[ ! "$OS_TYPE" ]] || [[ "$OS_TYPE" =~ "ubi*" ]]; then + docker manifest create $DOCKER_REPO:$CONFLUENT_VERSION $DOCKER_REPO:$CONFLUENT_VERSION$AMD_ARCH $DOCKER_REPO:$CONFLUENT_VERSION$ARM_ARCH + docker manifest push $DOCKER_REPO:$CONFLUENT_VERSION + export APPLIED="true" + fi + - >- + if [[ $UPDATE_LATEST_TAG ]]; then + if [[ $APPLIED ]]; then + docker manifest create $DOCKER_REPO:latest $DOCKER_REPO:latest$AMD_ARCH $DOCKER_REPO:latest$ARM_ARCH + docker manifest push $DOCKER_REPO:latest + fi + docker manifest create $DOCKER_REPO:latest-$OS_TYPE $DOCKER_REPO:latest-$OS_TYPE$AMD_ARCH $DOCKER_REPO:latest-$OS_TYPE$ARM_ARCH + docker manifest push $DOCKER_REPO:latest-$OS_TYPE + fi diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml new file mode 100644 index 000000000..2b80d3ee3 --- /dev/null +++ b/.semaphore/semaphore.yml @@ -0,0 +1,155 @@ +# This file is managed by ServiceBot plugin - Semaphore. The content in this file is created using a common +# template and configurations in service.yml. +# Any modifications made to ths file will be overwritten by the generated content in nightly runs. +# For more information, please refer to the page: +# https://confluentinc.atlassian.net/wiki/spaces/Foundations/pages/2871296194/Add+SemaphoreCI +version: v1.0 +name: build-test-release +agent: + machine: + type: s1-prod-ubuntu20-04-amd64-1 + +fail_fast: + cancel: + when: "true" + +execution_time_limit: + hours: 1 + +queue: + - when: "branch != 'master' and branch !~ '[0-9]+\\.[0-9]+\\.[0-9]+'" + processing: parallel + +global_job_config: + prologue: + commands: + - checkout + - sem-version java 8 + - . vault-setup + - . cache-maven restore + - export GIT_COMMIT=$(git rev-parse --verify HEAD --short) + - export BUILD_NUMBER=$(echo $SEMAPHORE_WORKFLOW_ID | cut -f1 -d"-") + - export BRANCH_TAG=$(echo $SEMAPHORE_GIT_BRANCH | tr / -) + # For PR Builds using Packaging + - pip install confluent-release-tools + - if [ $BRANCH_TAG == "master" ]; then export BUILD_KEY=$(pinto get-master-version); else export BUILD_KEY=$BRANCH_TAG; fi + - export LATEST_PACKAGING_BUILD_NUMBER=$(aws s3 ls s3://jenkins-confluent-packages/$BRANCH_TAG/ --no-paginate --recursive | grep "$BRANCH_TAG/[0-9]" | sort | tail -n 1 | awk '{print $4}' | awk -F + / '{print $2}') + - export CONFLUENT_VERSION=$(pinto get-version --build $BUILD_KEY --key confluent.version) + - export DEFAULT_OS_TYPE="ubi" + - export URL_CONFLUENT_VERSION=$(echo $CONFLUENT_VERSION | awk -F . '{print $1"."$2}') + - export PACKAGES_URL="https://s3-us-west-2.amazonaws.com/jenkins-confluent-packages/$BRANCH_TAG/$LATEST_PACKAGING_BUILD_NUMBER/PACKAGE_TYPE/$URL_CONFLUENT_VERSION" + - export PACKAGING_BUILD_NUMBER=$LATEST_PACKAGING_BUILD_NUMBER + - >- + if [[ $IS_BETA || $IS_HOTFIX || $IS_POST ]]; then + export PLATFORM_LABEL=$(echo $BRANCH_TAG | awk -F - '{print "~"$2}') + elif [[ ! $IS_RELEASE ]]; then + export PLATFORM_LABEL="~SNAPSHOT" + else + export PLATFORM_LABEL="" + fi + - export PACKAGING_BUILD_ARGS=" -DCONFLUENT_VERSION=$CONFLUENT_VERSION -DCONFLUENT_PLATFORM_LABEL=$PLATFORM_LABEL -DCONFLUENT_DEB_VERSION=$CONFLUENT_DEB_VERSION -DALLOW_UNSIGNED=$ALLOW_UNSIGNED" + - >- + if [[ $IS_RELEASE && $PACKAGING_BUILD_NUMBER ]]; then + if [[ $IS_BETA ]]; then + export MAVEN_PACKAGES_URL="https://s3.us-west-2.amazonaws.com/jenkins-confluent-packages-beta-maven/$BRANCH_TAG/$PACKAGING_BUILD_NUMBER/maven" + elif [[ $IS_PREVIEW ]]; then + export MAVEN_PACKAGES_URL="https://s3.us-west-2.amazonaws.com/jenkins-confluent-packages-alpha-maven/$BRANCH_NAME/$PACKAGING_BUILD_NUMBER/maven" + elif [[ $IS_RC ]]; then + export MAVEN_PACKAGES_URL="https://s3.us-west-2.amazonaws.com/staging-confluent-packages-maven/v$BRANCH_NAME/maven" + if [[ $PACKAGES_MAVEN_URL ]]; then + export MAVEN_PACKAGES_URL=$PACKAGES_MAVEN_URL + fi + fi + # Overwrite maven global configuration + . vault-sem-get-secret maven-settings-cp-dockerfile + else + echo "This job is not a isBetaJob, isPreviewJob, isHotfixJob, or isRcJob (What we know how to handle) - and we don't know how to handle it" + fi + - export DOCKER_DEV_REGISTRY="519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/dev/" + - export DOCKER_PROD_REGISTRY="519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/prod/" + - export DOCKER_UPSTREAM_REGISTRY=$DOCKER_PROD_REGISTRY + - export LATEST_TAG=$BRANCH_TAG-latest + - export DOCKER_UPSTREAM_TAG="$LATEST_TAG" + - export DOCKER_REPOS="unsynced-confluentinc/cp-kafka-mqtt" + - export DOCKER_DEV_TAG="dev-$BRANCH_TAG-$BUILD_NUMBER" + - export AMD_ARCH=.amd64 + - export ARM_ARCH=.arm64 +blocks: + - name: Build, Test, & Scan AMD + dependencies: [] + run: + when: "pull_request =~ '.*'" + task: + jobs: + - name: Build, Test, & Scan ubi8 + commands: + - export OS_TAG="-ubi8" + - export DOCKER_DEV_FULL_IMAGES=$DOCKER_DEV_REGISTRY${DOCKER_REPOS// /:$DOCKER_DEV_TAG$OS_TAG $DOCKER_DEV_REGISTRY}:$DOCKER_DEV_TAG$OS_TAG + - export AMD_DOCKER_DEV_FULL_IMAGES=${DOCKER_DEV_FULL_IMAGES// /$AMD_ARCH }$AMD_ARCH + - ci-tools ci-update-version + - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") + - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" + - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean package dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY + -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi8 + $PACKAGING_BUILD_ARGS + - . cache-maven store + - >- + for dev_image in $AMD_DOCKER_DEV_FULL_IMAGES; + do + cve-scan $dev_image + done + - for image in $AMD_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done + epilogue: + always: + commands: + - . publish-test-results + - artifact push workflow target/test-results + - artifact push workflow target --destination target-AMD + - name: Build & Test ARM + dependencies: [] + run: + when: "pull_request =~ '.*'" + task: + agent: + machine: + type: s1-prod-ubuntu20-04-arm64-1 + jobs: + - name: Build & Test ubi8 + commands: + - export OS_TAG="-ubi8" + - export DOCKER_DEV_FULL_IMAGES=$DOCKER_DEV_REGISTRY${DOCKER_REPOS// /:$DOCKER_DEV_TAG$OS_TAG $DOCKER_DEV_REGISTRY}:$DOCKER_DEV_TAG$OS_TAG + - export ARM_DOCKER_DEV_FULL_IMAGES=${DOCKER_DEV_FULL_IMAGES// /$ARM_ARCH }$ARM_ARCH + - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") + - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" + - ci-tools ci-update-version + - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean package dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY + -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi8 + $PACKAGING_BUILD_ARGS + - . cache-maven store + - for image in $ARM_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done + epilogue: + always: + commands: + - . publish-test-results + - artifact push workflow target/test-results + - artifact push workflow target --destination target-ARM +after_pipeline: + task: + agent: + machine: + type: s1-prod-ubuntu20-04-arm64-0 + jobs: + - name: Metrics + commands: + - emit-ci-metrics -p -a test-results + - name: Publish Test Results + commands: + - test-results gen-pipeline-report + - name: SonarQube + commands: + - checkout + - sem-version java 11 + - artifact pull workflow target-AMD + - artifact pull workflow target-ARM + - emit-sonarqube-data --run_only_sonar_scan diff --git a/service.yml b/service.yml index 518a9dec5..0cddcebb0 100644 --- a/service.yml +++ b/service.yml @@ -6,4 +6,13 @@ codeowners: enable: true semaphore: enable: true - pipeline_type: cp-dockerfile \ No newline at end of file + pipeline_type: cp-dockerfile + docker_repos: ['unsynced-confluentinc/cp-kafka-mqtt',] + maven_phase: 'package' + cve_scan : false + maven_skip_deploy: true + os_types: ['ubi8'] + build_arm: true + use_packages: true + cp_images: true + nano_version: true \ No newline at end of file