From 59c424b4b1a229c64def6c536cf845c9256d3fdb Mon Sep 17 00:00:00 2001 From: Evgeniy Patlan Date: Fri, 10 Jul 2026 16:25:10 +0300 Subject: [PATCH] PMM: build pmm-server for arm64 pmm3-server-autobuild becomes a parent that runs two new per-arch jobs (pmm3-server-autobuild-amd, pmm3-server-autobuild-arm) in parallel and merges the images into a multi-arch manifest, same layout as the client autobuild. Watchtower is built for both arches as well. API tests, image scanning, staging and package testing get arch parameters. --- pmm/v3/pmm3-api-tests.groovy | 7 +- pmm/v3/pmm3-aws-staging-start.groovy | 13 +- pmm/v3/pmm3-image-scanning.groovy | 16 +- pmm/v3/pmm3-package-testing-amd64.groovy | 5 + pmm/v3/pmm3-package-testing-arm64.groovy | 5 + pmm/v3/pmm3-package-tests-matrix-amd64.groovy | 5 + pmm/v3/pmm3-package-tests-matrix-arm64.groovy | 5 + pmm/v3/pmm3-server-autobuild-amd.groovy | 180 ++++++++++++++++ pmm/v3/pmm3-server-autobuild-arm.groovy | 169 +++++++++++++++ pmm/v3/pmm3-server-autobuild.groovy | 193 ++++++++---------- pmm/v3/pmm3-watchtower-autobuild.groovy | 42 ++-- pmm/v3/vars/runSpotInstance.groovy | 6 +- pmm/v3/vars/setupPMM3Client.groovy | 6 +- 13 files changed, 516 insertions(+), 136 deletions(-) create mode 100644 pmm/v3/pmm3-server-autobuild-amd.groovy create mode 100644 pmm/v3/pmm3-server-autobuild-arm.groovy diff --git a/pmm/v3/pmm3-api-tests.groovy b/pmm/v3/pmm3-api-tests.groovy index 181254b3eb..bcc5fcbbe6 100644 --- a/pmm/v3/pmm3-api-tests.groovy +++ b/pmm/v3/pmm3-api-tests.groovy @@ -5,9 +5,14 @@ library changelog: false, identifier: 'lib@master', retriever: modernSCM([ pipeline { agent { - label 'agent-amd64' + label params.AGENT_ARCH == 'arm64' ? 'agent-arm64' : 'agent-amd64' } parameters { + choice( + choices: ['amd64', 'arm64'], + description: 'CPU architecture of the agent that runs the server container and tests', + name: 'AGENT_ARCH' + ) string( defaultValue: 'https://github.com/percona/pmm', description: 'Url for pmm repository', diff --git a/pmm/v3/pmm3-aws-staging-start.groovy b/pmm/v3/pmm3-aws-staging-start.groovy index 8bce83385d..390e1f15a6 100644 --- a/pmm/v3/pmm3-aws-staging-start.groovy +++ b/pmm/v3/pmm3-aws-staging-start.groovy @@ -23,6 +23,11 @@ pipeline { description: 'WatchTower docker container version (image-name:version-tag, ex: perconalab/watchtower:dev-latest)', name: 'WATCHTOWER_VERSION' ) + choice( + choices: ['amd64', 'arm64'], + description: 'CPU architecture of the staging VM (arm64 = AWS Graviton t4g.xlarge)', + name: 'SERVER_ARCH' + ) string( defaultValue: '3-dev-latest', description: 'PMM Client version ("3-dev-latest" for main branch, "latest" or "X.X.X" for released version, "pmm3-rc" for Release Candidate, "http://..." for feature build)', @@ -174,7 +179,13 @@ pipeline { stage('Run VM') { steps { // This sets envvars: SPOT_PRICE, REQUEST_ID, IP, AMI_ID - runSpotInstance('t3.xlarge') + script { + if (params.SERVER_ARCH == 'arm64') { + runSpotInstance('t4g.xlarge', 'arm64') + } else { + runSpotInstance('t3.xlarge') + } + } withCredentials([sshUserPrivateKey(credentialsId: 'aws-jenkins', keyFileVariable: 'KEY_PATH', passphraseVariable: '', usernameVariable: 'USER')]) { sh ''' diff --git a/pmm/v3/pmm3-image-scanning.groovy b/pmm/v3/pmm3-image-scanning.groovy index 74d65b7208..1e501b49db 100644 --- a/pmm/v3/pmm3-image-scanning.groovy +++ b/pmm/v3/pmm3-image-scanning.groovy @@ -38,10 +38,12 @@ pipeline { steps { script { sh """ - trivy image --severity HIGH,CRITICAL --format table -o trivy-server-report.txt ${params.PMM_SERVER_IMAGE} - trivy image --severity HIGH,CRITICAL --format template --template "@html.tpl" -o trivy-server-report.html ${params.PMM_SERVER_IMAGE} + trivy image --platform linux/amd64 --severity HIGH,CRITICAL --format table -o trivy-server-report-amd64.txt ${params.PMM_SERVER_IMAGE} + trivy image --platform linux/amd64 --severity HIGH,CRITICAL --format template --template "@html.tpl" -o trivy-server-report-amd64.html ${params.PMM_SERVER_IMAGE} + trivy image --platform linux/arm64 --severity HIGH,CRITICAL --format table -o trivy-server-report-arm64.txt ${params.PMM_SERVER_IMAGE} + trivy image --platform linux/arm64 --severity HIGH,CRITICAL --format template --template "@html.tpl" -o trivy-server-report-arm64.html ${params.PMM_SERVER_IMAGE} """ - archiveArtifacts artifacts: 'trivy-server-report.*', allowEmptyArchive: true + archiveArtifacts artifacts: 'trivy-server-report-*.*', allowEmptyArchive: true } } } @@ -49,10 +51,12 @@ pipeline { steps { script { sh """ - trivy image --severity HIGH,CRITICAL --format table -o trivy-client-report.txt ${params.PMM_CLIENT_IMAGE} - trivy image --severity HIGH,CRITICAL --format template --template "@html.tpl" -o trivy-client-report.html ${params.PMM_CLIENT_IMAGE} + trivy image --platform linux/amd64 --severity HIGH,CRITICAL --format table -o trivy-client-report-amd64.txt ${params.PMM_CLIENT_IMAGE} + trivy image --platform linux/amd64 --severity HIGH,CRITICAL --format template --template "@html.tpl" -o trivy-client-report-amd64.html ${params.PMM_CLIENT_IMAGE} + trivy image --platform linux/arm64 --severity HIGH,CRITICAL --format table -o trivy-client-report-arm64.txt ${params.PMM_CLIENT_IMAGE} + trivy image --platform linux/arm64 --severity HIGH,CRITICAL --format template --template "@html.tpl" -o trivy-client-report-arm64.html ${params.PMM_CLIENT_IMAGE} """ - archiveArtifacts artifacts: 'trivy-client-report.*', allowEmptyArchive: true + archiveArtifacts artifacts: 'trivy-client-report-*.*', allowEmptyArchive: true } } } diff --git a/pmm/v3/pmm3-package-testing-amd64.groovy b/pmm/v3/pmm3-package-testing-amd64.groovy index fc0ef63c9b..4794a9c48e 100644 --- a/pmm/v3/pmm3-package-testing-amd64.groovy +++ b/pmm/v3/pmm3-package-testing-amd64.groovy @@ -6,6 +6,7 @@ library changelog: false, identifier: 'lib@master', retriever: modernSCM([ void runStaging(String DOCKER_VERSION, ADMIN_PASSWORD, CLIENTS) { stagingJob = build job: 'pmm3-aws-staging-start', parameters: [ string(name: 'DOCKER_VERSION', value: DOCKER_VERSION), + string(name: 'SERVER_ARCH', value: params.SERVER_ARCH), string(name: 'CLIENT_VERSION', value: '3-dev-latest'), string(name: 'DOCKER_ENV_VARIABLE', value: '-e PMM_ENABLE_TELEMETRY=0 -e PMM_DATA_RETENTION=48h -e PMM_PERCONA_PLATFORM_ADDRESS=https://check-dev.percona.com:443 -e PMM_ENABLE_NOMAD=1'), string(name: 'CLIENTS', value: CLIENTS), @@ -114,6 +115,10 @@ pipeline { description: 'PMM Server docker container version (image-name:version-tag)', name: 'DOCKER_VERSION', trim: true) + choice( + choices: ['amd64', 'arm64'], + description: 'Architecture of the PMM server staging VM', + name: 'SERVER_ARCH') string( defaultValue: latestVersion, description: 'PMM Version for testing', diff --git a/pmm/v3/pmm3-package-testing-arm64.groovy b/pmm/v3/pmm3-package-testing-arm64.groovy index e0fd33f646..693ce3a569 100644 --- a/pmm/v3/pmm3-package-testing-arm64.groovy +++ b/pmm/v3/pmm3-package-testing-arm64.groovy @@ -6,6 +6,7 @@ library changelog: false, identifier: 'lib@master', retriever: modernSCM([ void runStaging(String DOCKER_VERSION, ADMIN_PASSWORD, CLIENTS) { stagingJob = build job: 'pmm3-aws-staging-start', parameters: [ string(name: 'DOCKER_VERSION', value: DOCKER_VERSION), + string(name: 'SERVER_ARCH', value: params.SERVER_ARCH), string(name: 'CLIENT_VERSION', value: '3-dev-latest'), string(name: 'DOCKER_ENV_VARIABLE', value: '-e PMM_ENABLE_TELEMETRY=0 -e PMM_DATA_RETENTION=48h -e PMM_PERCONA_PLATFORM_ADDRESS=https://check-dev.percona.com:443 -e PMM_ENABLE_NOMAD=1'), string(name: 'CLIENTS', value: CLIENTS), @@ -114,6 +115,10 @@ pipeline { description: 'PMM Server docker container version (image-name:version-tag)', name: 'DOCKER_VERSION', trim: true) + choice( + choices: ['arm64', 'amd64'], + description: 'Architecture of the PMM server staging VM', + name: 'SERVER_ARCH') string( defaultValue: latestVersion, description: 'PMM Version for testing', diff --git a/pmm/v3/pmm3-package-tests-matrix-amd64.groovy b/pmm/v3/pmm3-package-tests-matrix-amd64.groovy index 2cf5283936..1507ab57f8 100644 --- a/pmm/v3/pmm3-package-tests-matrix-amd64.groovy +++ b/pmm/v3/pmm3-package-tests-matrix-amd64.groovy @@ -7,6 +7,7 @@ void runPackageTest(String GIT_BRANCH, DOCKER_VERSION, PMM_VERSION, TESTS, INSTA packageTestJob = build job: 'pmm3-package-testing', parameters: [ string(name: 'GIT_BRANCH', value: GIT_BRANCH), string(name: 'DOCKER_VERSION', value: DOCKER_VERSION), + string(name: 'SERVER_ARCH', value: params.SERVER_ARCH), string(name: 'PMM_VERSION', value: PMM_VERSION), string(name: 'TESTS', value: TESTS), string(name: 'INSTALL_REPO', value: INSTALL_REPO), @@ -38,6 +39,10 @@ pipeline { description: 'PMM Server docker container version (image-name:version-tag)', name: 'DOCKER_VERSION', trim: true) + choice( + choices: ['amd64', 'arm64'], + description: 'Architecture of the PMM server staging VM', + name: 'SERVER_ARCH') string( defaultValue: latestVersion, description: 'PMM Version for testing', diff --git a/pmm/v3/pmm3-package-tests-matrix-arm64.groovy b/pmm/v3/pmm3-package-tests-matrix-arm64.groovy index c2f00330b0..71f24aaad4 100644 --- a/pmm/v3/pmm3-package-tests-matrix-arm64.groovy +++ b/pmm/v3/pmm3-package-tests-matrix-arm64.groovy @@ -7,6 +7,7 @@ void runPackageTest(String GIT_BRANCH, DOCKER_VERSION, PMM_VERSION, TESTS, INSTA packageTestJob = build job: 'pmm3-package-testing-arm', parameters: [ string(name: 'GIT_BRANCH', value: GIT_BRANCH), string(name: 'DOCKER_VERSION', value: DOCKER_VERSION), + string(name: 'SERVER_ARCH', value: params.SERVER_ARCH), string(name: 'PMM_VERSION', value: PMM_VERSION), string(name: 'TESTS', value: TESTS), string(name: 'INSTALL_REPO', value: INSTALL_REPO), @@ -38,6 +39,10 @@ pipeline { description: 'PMM Server docker container version (image-name:version-tag)', name: 'DOCKER_VERSION', trim: true) + choice( + choices: ['arm64', 'amd64'], + description: 'Architecture of the PMM server staging VM', + name: 'SERVER_ARCH') string( defaultValue: latestVersion, description: 'PMM Version for testing', diff --git a/pmm/v3/pmm3-server-autobuild-amd.groovy b/pmm/v3/pmm3-server-autobuild-amd.groovy new file mode 100644 index 0000000000..fe8ee56f16 --- /dev/null +++ b/pmm/v3/pmm3-server-autobuild-amd.groovy @@ -0,0 +1,180 @@ +library changelog: false, identifier: 'lib@master', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +pipeline { + agent { + label params.USE_ONDEMAND ? 'agent-amd64-ondemand' : 'agent-amd64' + } + parameters { + string( + defaultValue: 'v3', + description: 'Tag/Branch for pmm-submodules repository', + name: 'GIT_BRANCH') + choice( + // default is 'experimental' + choices: ['experimental', 'testing'], + description: 'Repository to push packages to', + name: 'DESTINATION') + booleanParam( + defaultValue: false, + description: 'Use on-demand instances instead of spot (for RC/Release builds)', + name: 'USE_ONDEMAND' + ) + } + options { + buildDiscarder(logRotator(numToKeepStr: '30')) + skipDefaultCheckout() + disableConcurrentBuilds() + } + environment { + PATH_TO_SCRIPTS = 'sources/pmm/src/github.com/percona/pmm/build/scripts' + } + stages { + stage('Prepare') { + steps { + git poll: true, + branch: GIT_BRANCH, + url: 'http://github.com/Percona-Lab/pmm-submodules' + script { + env.VERSION = sh(returnStdout: true, script: "cat VERSION").trim() + } + sh ''' + set -o errexit + git submodule update --init --jobs 10 + git submodule status + + git rev-parse --short HEAD > shortCommit + echo "UPLOAD/pmm3-components/yum/${DESTINATION}/${JOB_NAME}/pmm/${VERSION}/${GIT_BRANCH}/$(cat shortCommit)/${BUILD_NUMBER}" > uploadPath + ''' + script { + if (params.DESTINATION == "testing") { + env.DOCKER_LATEST_TAG = "${VERSION}-rc${BUILD_NUMBER}-amd64" + env.DOCKER_RC_TAG = "${VERSION}-rc-amd64" + } else { + env.DOCKER_LATEST_TAG = "3-dev-latest-amd64" + } + } + + archiveArtifacts 'uploadPath' + stash includes: 'uploadPath', name: 'uploadPath' + archiveArtifacts 'shortCommit' + slackSend botUser: true, channel: '#pmm-notifications', color: '#0000FF', message: "[${JOB_NAME}]: build started - ${BUILD_URL}" + } + } + stage('Build client source') { + steps { + sh "${PATH_TO_SCRIPTS}/build-client-source" + stash includes: 'results/source_tarball/*.tar.*', name: 'source.tarball' + uploadTarball('source') + } + } + stage('Build client binary') { + steps { + sh "${PATH_TO_SCRIPTS}/build-client-binary" + stash includes: 'results/tarball/*.tar.*', name: 'binary.tarball' + uploadTarball('binary') + } + } + stage('Build client source rpm') { + steps { + sh "${PATH_TO_SCRIPTS}/build-client-srpm" + stash includes: 'results/srpm/pmm*-client-*.src.rpm', name: 'rpms' + uploadRPM() + } + } + stage('Build client binary rpm') { + steps { + sh ''' + set -o errexit + + ${PATH_TO_SCRIPTS}/build-client-rpm + + mkdir -p tmp/pmm-server/RPMS/ + cp results/rpm/pmm*-client-*.rpm tmp/pmm-server/RPMS/ + ''' + stash includes: 'tmp/pmm-server/RPMS/*.rpm', name: 'rpms' + uploadRPM() + } + } + stage('Build server packages') { + steps { + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'pmm-staging-slave', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + sh ''' + set -o errexit + + ${PATH_TO_SCRIPTS}/build-server-rpm-all + ''' + } + stash includes: 'tmp/pmm-server/RPMS/*/*/*.rpm', name: 'rpms' + uploadRPM() + } + } + stage('Build server docker') { + steps { + withCredentials([ + usernamePassword(credentialsId: 'hub.docker.com', passwordVariable: 'PASS', usernameVariable: 'USER')]) { + sh ''' + echo "${PASS}" | docker login -u "${USER}" --password-stdin + ''' + } + sh ''' + set -o errexit + + export DOCKER_TAG=perconalab/pmm-server:$(date -u '+%Y%m%d%H%M')-amd64 + export DOCKERFILE=Dockerfile.el9 + if [ -n "${DOCKER_RC_TAG}" ]; then + export PMM_PERCONA_PLATFORM_ADDRESS=https://check.percona.com + fi + + ${PATH_TO_SCRIPTS}/build-server-docker + + if [ -n "${DOCKER_RC_TAG}" ]; then + docker tag ${DOCKER_TAG} perconalab/pmm-server:${DOCKER_RC_TAG} + docker push perconalab/pmm-server:${DOCKER_RC_TAG} + fi + docker tag ${DOCKER_TAG} perconalab/pmm-server:${DOCKER_LATEST_TAG} + docker push ${DOCKER_TAG} + docker push perconalab/pmm-server:${DOCKER_LATEST_TAG} + echo "${DOCKER_LATEST_TAG}" > DOCKER_TAG + echo "${DOCKER_TAG}" > TIMESTAMP_TAG + ''' + script { + env.IMAGE = sh(returnStdout: true, script: "cat DOCKER_TAG").trim() + env.TIMESTAMP_TAG = sh(returnStdout: true, script: "cat TIMESTAMP_TAG").trim() + } + withCredentials([string(credentialsId: 'LAUNCHABLE_TOKEN', variable: 'LAUNCHABLE_TOKEN')]) { + sh ''' + set -o errexit + pip3 install --user --upgrade launchable~=1.0 || true + launchable verify || true + echo "$(git submodule status)" || true + + export DOCKER_IMAGE_ID=$(docker inspect perconalab/pmm-server:${IMAGE} -f "{{.Id}}") || true + + launchable record build --name "${DOCKER_IMAGE_ID}" --lineage "perconalab/pmm-server:${IMAGE}" || true + ''' + } + } + } + } + post { + success { + script { + slackSend botUser: true, channel: '#pmm-notifications', color: '#00FF00', message: "[${JOB_NAME}]: build finished - ${IMAGE}, URL: ${BUILD_URL}" + if (params.DESTINATION == "testing") { + currentBuild.description = "RC Build v3 (amd64), Image:" + env.IMAGE + slackSend botUser: true, channel: '#pmm-qa', color: '#00FF00', message: "[${JOB_NAME}]: RC build finished - ${IMAGE}, URL: ${BUILD_URL}" + } + } + } + failure { + script { + echo "Pipeline failed" + slackSend botUser: true, channel: '#pmm-notifications', color: '#FF0000', message: "[${JOB_NAME}]: build ${currentBuild.result}, URL: ${BUILD_URL}" + slackSend botUser: true, channel: '#pmm-qa', color: '#FF0000', message: "[${JOB_NAME}]: build ${currentBuild.result}, URL: ${BUILD_URL}" + } + } + } +} diff --git a/pmm/v3/pmm3-server-autobuild-arm.groovy b/pmm/v3/pmm3-server-autobuild-arm.groovy new file mode 100644 index 0000000000..cbfe24863f --- /dev/null +++ b/pmm/v3/pmm3-server-autobuild-arm.groovy @@ -0,0 +1,169 @@ +library changelog: false, identifier: 'lib@master', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +pipeline { + agent { + label params.USE_ONDEMAND ? 'agent-arm64-ondemand' : 'agent-arm64' + } + parameters { + string( + defaultValue: 'v3', + description: 'Tag/Branch for pmm-submodules repository', + name: 'GIT_BRANCH') + choice( + // default is 'experimental' + choices: ['experimental', 'testing'], + description: 'Repository to push packages to', + name: 'DESTINATION') + booleanParam( + defaultValue: false, + description: 'Use on-demand instances instead of spot (for RC/Release builds)', + name: 'USE_ONDEMAND' + ) + } + options { + buildDiscarder(logRotator(numToKeepStr: '30')) + skipDefaultCheckout() + disableConcurrentBuilds() + } + environment { + PATH_TO_SCRIPTS = 'sources/pmm/src/github.com/percona/pmm/build/scripts' + GOARCH = 'arm64' + } + stages { + stage('Prepare') { + steps { + git poll: true, + branch: GIT_BRANCH, + url: 'http://github.com/Percona-Lab/pmm-submodules' + script { + env.VERSION = sh(returnStdout: true, script: "cat VERSION").trim() + } + sh ''' + set -o errexit + git submodule update --init --jobs 10 + git submodule status + + git rev-parse --short HEAD > shortCommit + echo "UPLOAD/pmm3-components/yum/${DESTINATION}/${JOB_NAME}/pmm/${VERSION}/${GIT_BRANCH}/$(cat shortCommit)/${BUILD_NUMBER}" > uploadPath + ''' + script { + if (params.DESTINATION == "testing") { + env.DOCKER_LATEST_TAG = "${VERSION}-rc${BUILD_NUMBER}-arm64" + env.DOCKER_RC_TAG = "${VERSION}-rc-arm64" + } else { + env.DOCKER_LATEST_TAG = "3-dev-latest-arm64" + } + } + + archiveArtifacts 'uploadPath' + stash includes: 'uploadPath', name: 'uploadPath' + archiveArtifacts 'shortCommit' + slackSend botUser: true, channel: '#pmm-notifications', color: '#0000FF', message: "[${JOB_NAME}]: build started - ${BUILD_URL}" + } + } + stage('Build client source') { + steps { + sh "${PATH_TO_SCRIPTS}/build-client-source" + stash includes: 'results/source_tarball/*.tar.*', name: 'source.tarball' + uploadTarball('source') + } + } + stage('Build client binary') { + steps { + sh "${PATH_TO_SCRIPTS}/build-client-binary" + stash includes: 'results/tarball/*.tar.*', name: 'binary.tarball' + uploadTarball('binary') + } + } + stage('Build client source rpm') { + steps { + sh "${PATH_TO_SCRIPTS}/build-client-srpm" + stash includes: 'results/srpm/pmm*-client-*.src.rpm', name: 'rpms' + uploadRPM() + } + } + stage('Build client binary rpm') { + steps { + sh ''' + set -o errexit + + ${PATH_TO_SCRIPTS}/build-client-rpm + + mkdir -p tmp/pmm-server/RPMS/ + cp results/rpm/pmm*-client-*.rpm tmp/pmm-server/RPMS/ + ''' + stash includes: 'tmp/pmm-server/RPMS/*.rpm', name: 'rpms' + uploadRPM() + } + } + stage('Build server packages') { + steps { + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'pmm-staging-slave', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + sh ''' + set -o errexit + + ${PATH_TO_SCRIPTS}/build-server-rpm-all + ''' + } + stash includes: 'tmp/pmm-server/RPMS/*/*/*.rpm', name: 'rpms' + uploadRPM() + } + } + stage('Build server docker') { + steps { + withCredentials([ + usernamePassword(credentialsId: 'hub.docker.com', passwordVariable: 'PASS', usernameVariable: 'USER')]) { + sh ''' + echo "${PASS}" | docker login -u "${USER}" --password-stdin + ''' + } + sh ''' + set -o errexit + + export DOCKER_TAG=perconalab/pmm-server:$(date -u '+%Y%m%d%H%M')-arm64 + export DOCKERFILE=Dockerfile.el9 + if [ -n "${DOCKER_RC_TAG}" ]; then + export PMM_PERCONA_PLATFORM_ADDRESS=https://check.percona.com + fi + + ${PATH_TO_SCRIPTS}/build-server-docker + + if [ -n "${DOCKER_RC_TAG}" ]; then + docker tag ${DOCKER_TAG} perconalab/pmm-server:${DOCKER_RC_TAG} + docker push perconalab/pmm-server:${DOCKER_RC_TAG} + fi + docker tag ${DOCKER_TAG} perconalab/pmm-server:${DOCKER_LATEST_TAG} + docker push ${DOCKER_TAG} + docker push perconalab/pmm-server:${DOCKER_LATEST_TAG} + echo "${DOCKER_LATEST_TAG}" > DOCKER_TAG + echo "${DOCKER_TAG}" > TIMESTAMP_TAG + ''' + script { + env.IMAGE = sh(returnStdout: true, script: "cat DOCKER_TAG").trim() + env.TIMESTAMP_TAG = sh(returnStdout: true, script: "cat TIMESTAMP_TAG").trim() + } + } + } + } + post { + success { + script { + slackSend botUser: true, channel: '#pmm-notifications', color: '#00FF00', message: "[${JOB_NAME}]: build finished - ${IMAGE}, URL: ${BUILD_URL}" + if (params.DESTINATION == "testing") { + currentBuild.description = "RC Build v3 (arm64), Image:" + env.IMAGE + slackSend botUser: true, channel: '#pmm-qa', color: '#00FF00', message: "[${JOB_NAME}]: RC build finished - ${IMAGE}, URL: ${BUILD_URL}" + } + } + } + failure { + script { + echo "Pipeline failed" + slackSend botUser: true, channel: '#pmm-notifications', color: '#FF0000', message: "[${JOB_NAME}]: build ${currentBuild.result}, URL: ${BUILD_URL}" + slackSend botUser: true, channel: '#pmm-qa', color: '#FF0000', message: "[${JOB_NAME}]: build ${currentBuild.result}, URL: ${BUILD_URL}" + } + } + } +} diff --git a/pmm/v3/pmm3-server-autobuild.groovy b/pmm/v3/pmm3-server-autobuild.groovy index df85df1684..7aa5825070 100644 --- a/pmm/v3/pmm3-server-autobuild.groovy +++ b/pmm/v3/pmm3-server-autobuild.groovy @@ -5,7 +5,7 @@ library changelog: false, identifier: 'lib@master', retriever: modernSCM([ pipeline { agent { - label params.USE_ONDEMAND ? 'agent-amd64-ondemand' : 'agent-amd64' + label params.USE_ONDEMAND ? 'cli-ondemand' : 'cli' } parameters { string( @@ -13,9 +13,8 @@ pipeline { description: 'Tag/Branch for pmm-submodules repository', name: 'GIT_BRANCH') choice( - // default is 'experimental' choices: ['experimental', 'testing'], - description: 'Repository to push packages to', + description: 'Publish packages to repositories: testing (for RC), experimental (for dev-latest)', name: 'DESTINATION') booleanParam( defaultValue: false, @@ -25,140 +24,92 @@ pipeline { } options { buildDiscarder(logRotator(numToKeepStr: '30')) - skipDefaultCheckout() disableConcurrentBuilds() + parallelsAlwaysFailFast() } triggers { upstream upstreamProjects: 'pmm3-submodules-rewind', threshold: hudson.model.Result.SUCCESS } - environment { - PATH_TO_SCRIPTS = 'sources/pmm/src/github.com/percona/pmm/build/scripts' - } stages { stage('Prepare') { steps { - git poll: true, - branch: GIT_BRANCH, - url: 'http://github.com/Percona-Lab/pmm-submodules' - script { - env.VERSION = sh(returnStdout: true, script: "cat VERSION").trim() - } - sh ''' - set -o errexit - git submodule update --init --jobs 10 - git submodule status - - git rev-parse --short HEAD > shortCommit - echo "UPLOAD/pmm3-components/yum/${DESTINATION}/${JOB_NAME}/pmm/${VERSION}/${GIT_BRANCH}/$(cat shortCommit)/${BUILD_NUMBER}" > uploadPath - ''' + checkout([$class: 'GitSCM', + branches: [[name: "*/${params.GIT_BRANCH}"]], + extensions: [[$class: 'CloneOption', + noTags: true, + reference: '', + shallow: true]], + userRemoteConfigs: [[url: 'https://github.com/Percona-Lab/pmm-submodules']] + ]) script { + def versionTag = sh(returnStdout: true, script: "cat VERSION").trim() if (params.DESTINATION == "testing") { - env.DOCKER_LATEST_TAG = "${VERSION}-rc${BUILD_NUMBER}" - env.DOCKER_RC_TAG = "${VERSION}-rc" + env.DOCKER_RC_TAG = "${versionTag}-rc" } else { - env.DOCKER_LATEST_TAG = "3-dev-latest" + env.DOCKER_LATEST_TAG = "3-dev-latest" } } - - archiveArtifacts 'uploadPath' - stash includes: 'uploadPath', name: 'uploadPath' - archiveArtifacts 'shortCommit' slackSend botUser: true, channel: '#pmm-notifications', color: '#0000FF', message: "[${JOB_NAME}]: build started - ${BUILD_URL}" } } - stage('Build client source') { - steps { - sh "${PATH_TO_SCRIPTS}/build-client-source" - stash includes: 'results/source_tarball/*.tar.*', name: 'source.tarball' - uploadTarball('source') - } - } - stage('Build client binary') { - steps { - sh "${PATH_TO_SCRIPTS}/build-client-binary" - stash includes: 'results/tarball/*.tar.*', name: 'binary.tarball' - uploadTarball('binary') - } - } - stage('Build client source rpm') { - steps { - sh "${PATH_TO_SCRIPTS}/build-client-srpm" - stash includes: 'results/srpm/pmm*-client-*.src.rpm', name: 'rpms' - uploadRPM() - } - } - stage('Build client binary rpm') { - steps { - sh ''' - set -o errexit - - ${PATH_TO_SCRIPTS}/build-client-rpm - - mkdir -p tmp/pmm-server/RPMS/ - cp results/rpm/pmm*-client-*.rpm tmp/pmm-server/RPMS/ - ''' - stash includes: 'tmp/pmm-server/RPMS/*.rpm', name: 'rpms' - uploadRPM() + stage('Build pmm3 server') { + parallel { + stage('Build pmm3 server for amd64') { + steps { + script { + def pmmServerAmd64 = build job: 'pmm3-server-autobuild-amd', parameters: [ + string(name: 'GIT_BRANCH', value: params.GIT_BRANCH), + string(name: 'DESTINATION', value: params.DESTINATION), + booleanParam(name: 'USE_ONDEMAND', value: params.USE_ONDEMAND) + ] + env.TIMESTAMP_TAG_AMD64 = pmmServerAmd64.buildVariables.TIMESTAMP_TAG + } + } + } + stage('Build pmm3 server for arm64') { + steps { + script { + def pmmServerArm64 = build job: 'pmm3-server-autobuild-arm', parameters: [ + string(name: 'GIT_BRANCH', value: params.GIT_BRANCH), + string(name: 'DESTINATION', value: params.DESTINATION), + booleanParam(name: 'USE_ONDEMAND', value: params.USE_ONDEMAND) + ] + env.TIMESTAMP_TAG_ARM64 = pmmServerArm64.buildVariables.TIMESTAMP_TAG + } + } + } } } - stage('Build server packages') { + stage('Push pmm3 server multi-arch images') { steps { - withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'pmm-staging-slave', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + withCredentials([usernamePassword(credentialsId: 'hub.docker.com', passwordVariable: 'PASS', usernameVariable: 'USER')]) { sh ''' - set -o errexit + echo "${PASS}" | docker login -u "${USER}" --password-stdin + set -o xtrace + + TIMESTAMP_TAG=perconalab/pmm-server:$(date -u '+%Y%m%d%H%M') + docker buildx imagetools create -t ${TIMESTAMP_TAG} \ + ${TIMESTAMP_TAG_AMD64} \ + ${TIMESTAMP_TAG_ARM64} + echo "${TIMESTAMP_TAG}" > TIMESTAMP_TAG - ${PATH_TO_SCRIPTS}/build-server-rpm-all + if [ -n "${DOCKER_RC_TAG}" ]; then + docker buildx imagetools create -t perconalab/pmm-server:${DOCKER_RC_TAG} \ + perconalab/pmm-server:${DOCKER_RC_TAG}-amd64 \ + perconalab/pmm-server:${DOCKER_RC_TAG}-arm64 + echo "${DOCKER_RC_TAG}" > DOCKER_TAG + else + docker buildx imagetools create -t perconalab/pmm-server:${DOCKER_LATEST_TAG} \ + perconalab/pmm-server:${DOCKER_LATEST_TAG}-amd64 \ + perconalab/pmm-server:${DOCKER_LATEST_TAG}-arm64 + echo "${DOCKER_LATEST_TAG}" > DOCKER_TAG + fi ''' } - stash includes: 'tmp/pmm-server/RPMS/*/*/*.rpm', name: 'rpms' - uploadRPM() - } - } - stage('Build server docker') { - steps { - withCredentials([ - usernamePassword(credentialsId: 'hub.docker.com', passwordVariable: 'PASS', usernameVariable: 'USER')]) { - sh ''' - echo "${PASS}" | docker login -u "${USER}" --password-stdin - ''' - } - sh ''' - set -o errexit - - export DOCKER_TAG=perconalab/pmm-server:$(date -u '+%Y%m%d%H%M') - export DOCKERFILE=Dockerfile.el9 - if [ -n "${DOCKER_RC_TAG}" ]; then - export PMM_PERCONA_PLATFORM_ADDRESS=https://check.percona.com - fi - - ${PATH_TO_SCRIPTS}/build-server-docker - - if [ -n "${DOCKER_RC_TAG}" ]; then - docker tag ${DOCKER_TAG} perconalab/pmm-server:${DOCKER_RC_TAG} - docker push perconalab/pmm-server:${DOCKER_RC_TAG} - fi - docker tag ${DOCKER_TAG} perconalab/pmm-server:${DOCKER_LATEST_TAG} - docker push ${DOCKER_TAG} - docker push perconalab/pmm-server:${DOCKER_LATEST_TAG} - echo "${DOCKER_LATEST_TAG}" > DOCKER_TAG - echo "${DOCKER_TAG}" > TIMESTAMP_TAG - ''' script { env.IMAGE = sh(returnStdout: true, script: "cat DOCKER_TAG").trim() env.TIMESTAMP_TAG = sh(returnStdout: true, script: "cat TIMESTAMP_TAG").trim() } - withCredentials([string(credentialsId: 'LAUNCHABLE_TOKEN', variable: 'LAUNCHABLE_TOKEN')]) { - sh ''' - set -o errexit - pip3 install --user --upgrade launchable~=1.0 || true - launchable verify || true - echo "$(git submodule status)" || true - - export DOCKER_IMAGE_ID=$(docker inspect perconalab/pmm-server:${IMAGE} -f "{{.Id}}") || true - - launchable record build --name "${DOCKER_IMAGE_ID}" --lineage "perconalab/pmm-server:${IMAGE}" || true - ''' - } } } stage('Trigger a devcontainer build') { @@ -180,8 +131,23 @@ pipeline { } } stage('Start API Tests') { - steps { - build job: 'pmm3-api-tests', propagate: false + parallel { + stage('API tests on amd64') { + steps { + build job: 'pmm3-api-tests', propagate: false, parameters: [ + string(name: 'AGENT_ARCH', value: 'amd64'), + string(name: 'DOCKER_VERSION', value: "perconalab/pmm-server:${env.IMAGE}") + ] + } + } + stage('API tests on arm64') { + steps { + build job: 'pmm3-api-tests', propagate: false, parameters: [ + string(name: 'AGENT_ARCH', value: 'arm64'), + string(name: 'DOCKER_VERSION', value: "perconalab/pmm-server:${env.IMAGE}") + ] + } + } } } } @@ -202,5 +168,8 @@ pipeline { slackSend botUser: true, channel: '#pmm-qa', color: '#FF0000', message: "[${JOB_NAME}]: build ${currentBuild.result}, URL: ${BUILD_URL}" } } + cleanup { + deleteDir() + } } } diff --git a/pmm/v3/pmm3-watchtower-autobuild.groovy b/pmm/v3/pmm3-watchtower-autobuild.groovy index 907bdd7651..6aabc439da 100644 --- a/pmm/v3/pmm3-watchtower-autobuild.groovy +++ b/pmm/v3/pmm3-watchtower-autobuild.groovy @@ -72,14 +72,17 @@ pipeline { } } } - stage('Build watchtower binary') { + stage('Build watchtower binaries') { steps { withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'pmm-staging-slave', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { sh ''' - docker run --rm \ - -v ${WORKSPACE}/${PATH_TO_WATCHTOWER}:/watchtower \ - public.ecr.aws/e7j3v3n0/rpmbuild:3 \ - make -C /watchtower build + for arch in amd64 arm64; do + docker run --rm \ + -v ${WORKSPACE}/${PATH_TO_WATCHTOWER}:/watchtower \ + -e GOARCH=${arch} \ + public.ecr.aws/e7j3v3n0/rpmbuild:3 \ + bash -c "make -C /watchtower build && mv /watchtower/watchtower /watchtower/watchtower-${arch}" + done ''' } } @@ -95,17 +98,32 @@ pipeline { } sh ''' set -o xtrace + set -o errexit cd ${PATH_TO_WATCHTOWER} - docker build -t ${TIMESTAMP_TAG} -f dockerfiles/Dockerfile . - docker push ${TIMESTAMP_TAG} + docker buildx create --name multiarch-wt --driver docker-container --use || docker buildx use multiarch-wt + + for arch in amd64 arm64; do + rm -rf ctx-${arch} + mkdir -p ctx-${arch}/dockerfiles + cp watchtower-${arch} ctx-${arch}/watchtower + cp dockerfiles/Dockerfile ctx-${arch}/dockerfiles/Dockerfile + docker buildx build \ + --platform linux/${arch} \ + -t ${TIMESTAMP_TAG}-${arch} \ + -f ctx-${arch}/dockerfiles/Dockerfile \ + --push ctx-${arch} + done + + docker buildx imagetools create -t ${TIMESTAMP_TAG} \ + ${TIMESTAMP_TAG}-amd64 ${TIMESTAMP_TAG}-arm64 if [ -n "${WATCHTOWER_RC_TAG}" ]; then - docker tag ${TIMESTAMP_TAG} ${WATCHTOWER_RC_TAG} - docker push ${WATCHTOWER_RC_TAG} + docker buildx imagetools create -t ${WATCHTOWER_RC_TAG} \ + ${TIMESTAMP_TAG}-amd64 ${TIMESTAMP_TAG}-arm64 fi - docker tag ${TIMESTAMP_TAG} ${WATCHTOWER_LATEST_TAG} - docker push ${WATCHTOWER_LATEST_TAG} + docker buildx imagetools create -t ${WATCHTOWER_LATEST_TAG} \ + ${TIMESTAMP_TAG}-amd64 ${TIMESTAMP_TAG}-arm64 echo "${WATCHTOWER_LATEST_TAG}" > WATCHTOWER_LATEST_TAG ''' script { @@ -114,7 +132,7 @@ pipeline { } } } - post { + post { success { script { slackSend botUser: true, channel: '#pmm-notifications', color: '#00FF00', message: "[${JOB_NAME}]: build finished - ${IMAGE}, URL: ${BUILD_URL}" diff --git a/pmm/v3/vars/runSpotInstance.groovy b/pmm/v3/vars/runSpotInstance.groovy index 8381d4a9ed..22f97c1fcb 100644 --- a/pmm/v3/vars/runSpotInstance.groovy +++ b/pmm/v3/vars/runSpotInstance.groovy @@ -1,5 +1,5 @@ -def call(String INSTANCE_TYPE) { - withEnv(["INSTANCE_TYPE=${INSTANCE_TYPE}"]) { +def call(String INSTANCE_TYPE, String ARCH = 'x86_64') { + withEnv(["INSTANCE_TYPE=${INSTANCE_TYPE}", "ARCH=${ARCH}"]) { withCredentials([aws(credentialsId: 'pmm-staging-slave')]) { sh ''' set -o xtrace @@ -8,7 +8,7 @@ def call(String INSTANCE_TYPE) { IMAGE_ID=$( aws ec2 describe-images \ --owners self \ - --filters "Name=tag:iit-billing-tag,Values=pmm-worker-3" "Name=architecture,Values=x86_64" \ + --filters "Name=tag:iit-billing-tag,Values=pmm-worker-3" "Name=architecture,Values=${ARCH}" \ --region us-east-2 \ --output text \ --query 'Images[0].ImageId' diff --git a/pmm/v3/vars/setupPMM3Client.groovy b/pmm/v3/vars/setupPMM3Client.groovy index 420348ecb3..f673585a7f 100644 --- a/pmm/v3/vars/setupPMM3Client.groovy +++ b/pmm/v3/vars/setupPMM3Client.groovy @@ -58,7 +58,11 @@ def call(String SERVER_IP, String CLIENT_VERSION, String PMM_VERSION, String ENA fi if [[ "${CLIENT_VERSION}" == "latest-tarball" ]]; then - CLIENT_VERSION="https://pmm-build-cache.s3.us-east-2.amazonaws.com/PR-BUILDS/pmm-client/pmm-client-latest.tar.gz" + if [ "$(uname -m)" = "aarch64" ]; then + CLIENT_VERSION="https://pmm-build-cache.s3.us-east-2.amazonaws.com/PR-BUILDS/pmm-client-arm/pmm-client-latest.tar.gz" + else + CLIENT_VERSION="https://pmm-build-cache.s3.us-east-2.amazonaws.com/PR-BUILDS/pmm-client/pmm-client-latest.tar.gz" + fi fi if [ "${CLIENT_VERSION}" = 3-dev-latest ]; then