Skip to content

Commit 1edc5a2

Browse files
vitalykorolevvitalykorolev
authored
Merge pull request #289 from marklogic/release/docker-2.0.0
Release/docker 2.0.0
2 parents b483021 + 293b5d0 commit 1edc5a2

18 files changed

+1702
-586
lines changed

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,3 +27,4 @@ test/test_results/*
2727
test/log.html
2828
test/report.html
2929
test/output.xml
30+
test/python_env/*

‎Jenkinsfile‎

Lines changed: 94 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,8 @@ JIRA_ID_PATTERN = /(?i)(MLE)-\d{3,6}/
1515
JIRA_ID = ''
1616
LINT_OUTPUT = ''
1717
SCAN_OUTPUT = ''
18-
IMAGE_INFO = 0
18+
IMAGE_SIZE = 0
19+
RPMversion = ''
1920

2021
// Define local funtions
2122
void preBuildCheck() {
@@ -103,9 +104,9 @@ void resultNotification(message) {
103104
} else {
104105
emailList = params.emailList
105106
}
106-
jira_link = "https://project.marklogic.com/jira/browse/${JIRA_ID}"
107-
email_body = "<b>Jenkins pipeline for</b> ${env.JOB_NAME} <br><b>Build Number: </b>${env.BUILD_NUMBER} <b><br><br>Lint Output: <br></b><pre><code>${LINT_OUTPUT}</code></pre><br><b>Vulnerabilities: </b><pre><code>${SCAN_OUTPUT}</code></pre> <br><b>Image Details: <br></b>${IMAGE_INFO} <br><br><b>Build URL: </b><br>${env.BUILD_URL}"
108-
jira_email_body = "${email_body} <br><br><b>Jira URL: </b><br>${jira_link}"
107+
jira_link = "https://progresssoftware.atlassian.net/browse/${JIRA_ID}"
108+
email_body = "<b>Jenkins pipeline for</b> ${env.JOB_NAME} <br><b>Build Number: </b>${env.BUILD_NUMBER} <b><br><br>Lint Output: <br></b><pre><code>${LINT_OUTPUT}</code></pre><br><b>Vulnerabilities: </b><pre><code>${SCAN_OUTPUT}</code></pre> <br><b>Image Size: <br></b>${IMAGE_SIZE} <br><pre><code>docker pull ${dockerRegistry}/${latestTag}</code></pre><br><br><b>Build URL: </b><br><a href='${env.BUILD_URL}'>${env.BUILD_URL}</a>"
109+
jira_email_body = "${email_body} <br><br><b>Jira URL: </b><br><a href='${jira_link}'>${jira_link}</a>"
109110

110111
if (JIRA_ID) {
111112
def comment = [ body: "Jenkins pipeline build result: ${message}" ]
@@ -125,7 +126,7 @@ void copyRPMs() {
125126
else if (marklogicVersion == "11") {
126127
RPMsuffix = ".nightly-rhel"
127128
RPMbranch = "b11"
128-
RPMversion = "11.2"
129+
RPMversion = "11.3"
129130
}
130131
else if (marklogicVersion == "12") {
131132
RPMsuffix = ".nightly-rhel"
@@ -136,7 +137,7 @@ void copyRPMs() {
136137
error "Invalid value in marklogicVersion parameter."
137138
}
138139
sh """
139-
cd src/centos
140+
cd src
140141
if [ -z ${env.ML_RPM} ]; then
141142
wget --no-verbose https://bed-artifactory.bedford.progress.com:443/artifactory/ml-rpm-tierpoint/${RPMbranch}/server/MarkLogic-${RPMversion}${RPMsuffix}.x86_64.rpm
142143
else
@@ -149,43 +150,74 @@ void copyRPMs() {
149150
fi
150151
"""
151152
script {
152-
RPM = sh(returnStdout: true, script: 'cd src/centos;file MarkLogic-*.rpm | cut -d: -f1').trim()
153-
CONVERTERS = sh(returnStdout: true, script: 'cd src/centos;file MarkLogicConverters-*.rpm | cut -d: -f1').trim()
154-
mlVersion = sh(returnStdout: true, script: "echo ${RPM}| awk -F \"MarkLogic-\" '{print \$2;}' | awk -F \".x86_64.rpm\" '{print \$1;}' | awk -F \"-rhel\" '{print \$1;}' ").trim()
153+
// Get the RPM and Converters file names
154+
RPM = sh(returnStdout: true, script: 'cd src;file MarkLogic-*.rpm | cut -d: -f1').trim()
155+
CONVERTERS = sh(returnStdout: true, script: 'cd src;file MarkLogicConverters-*.rpm | cut -d: -f1').trim()
156+
// Extract MarkLogic version from RPM file name
157+
marklogicVersion = sh(returnStdout: true, script: "echo ${RPM}| awk -F \"MarkLogic-\" '{print \$2;}' | awk -F \".x86_64.rpm\" '{print \$1;}' | awk -F \"-rhel\" '{print \$1;}' ").trim()
158+
}
159+
}
160+
161+
void buildDockerImage() {
162+
builtImage="marklogic/marklogic-server-${dockerImageType}:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion}"
163+
publishImage="marklogic/marklogic-server-${dockerImageType}:${marklogicVersion}-${env.dockerImageType}"
164+
mlVerShort=marklogicVersion.split("\\.")[0]
165+
latestTag="marklogic/marklogic-server-${dockerImageType}:latest-${mlVerShort}"
166+
sh "make build docker_image_type=${dockerImageType} dockerTag=${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} marklogicVersion=${marklogicVersion} dockerVersion=${env.dockerVersion} build_branch=${env.BRANCH_NAME} package=${RPM} converters=${CONVERTERS}"
167+
currentBuild.displayName = "#${BUILD_NUMBER}: ${marklogicVersion}-${env.dockerImageType} (${env.dockerVersion})"
168+
}
169+
170+
void pullUpgradeDockerImage() {
171+
if (dockerImageType == "ubi-rootless" ) {
172+
sh """
173+
echo 'dockerImageType is set to ubi-rootless, skipping this stage and Docker upgrade test.'
174+
"""
175+
} else {
176+
if (upgradeDockerImage != "" ) {
177+
sh """
178+
echo 'upgradeDockerImage: ${upgradeDockerImage}'
179+
docker pull ${upgradeDockerImage}
180+
"""
181+
} else {
182+
upgradeDockerImage = "${dockerRegistry}/marklogic/marklogic-server-ubi:${marklogicVersion}-ubi-${env.dockerVersion}"
183+
sh """
184+
echo 'upgradeDockerImage is not specified, using ${upgradeDockerImage} for upgrade test.'
185+
docker pull ${dockerRegistry}/marklogic/marklogic-server-ubi:${marklogicVersion}-ubi-${env.dockerVersion}
186+
"""
187+
}
155188
}
156189
}
157190

158191
void structureTests() {
159192
sh """
160-
cd test
161-
#insert current version
162-
sed -i -e 's^VERSION_PLACEHOLDER^${mlVersion}-${env.platformString}-${env.dockerVersion}^g' -e 's^BRANCH_PLACEHOLDER^${env.BRANCH_NAME}^g' ./structure-test.yaml
163-
cd ..
164-
curl -s -LO https://storage.googleapis.com/container-structure-test/v1.11.0/container-structure-test-linux-amd64 && chmod +x container-structure-test-linux-amd64 && mv container-structure-test-linux-amd64 container-structure-test
165-
make structure-test version=${mlVersion}-${env.platformString}-${env.dockerVersion} Jenkins=true
166-
#fix junit output
167-
sed -i -e 's/<\\/testsuites>//' -e 's/<testsuite>//' -e 's/<testsuites/<testsuite name="container-structure-test"/' ./container-structure-test.xml
193+
#install container-structure-test 1.16.0 binary
194+
curl -s -LO https://storage.googleapis.com/container-structure-test/v1.16.0/container-structure-test-linux-amd64 && chmod +x container-structure-test-linux-amd64 && mv container-structure-test-linux-amd64 container-structure-test
195+
make structure-test current_image=marklogic/marklogic-server-${dockerImageType}:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} marklogicVersion=${marklogicVersion} dockerVersion=${env.dockerVersion} build_branch=${env.BRANCH_NAME} docker_image_type=${env.dockerImageType} Jenkins=true
168196
"""
169197
}
170198

199+
void dockerTests() {
200+
sh "make docker-tests current_image=marklogic/marklogic-server-${dockerImageType}:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} upgrade_image=${upgradeDockerImage} marklogicVersion=${marklogicVersion} build_branch=${env.BRANCH_NAME} dockerVersion=${env.dockerVersion} docker_image_type=${dockerImageType}"
201+
}
202+
171203
void lint() {
172-
IMAGE_INFO = sh(returnStdout: true, script: 'docker images | grep \"marklogic-server-centos\"')
204+
IMAGE_SIZE = sh(returnStdout: true, script: "docker images marklogic/marklogic-server-${dockerImageType}:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} --format '{{.Repository}}:{{.Tag}}\t{{.Size}}'")
173205

174-
sh '''
206+
sh """
175207
make lint Jenkins=true
176-
cat start-marklogic-lint.txt marklogic-deps-centos-base-lint.txt marklogic-server-centos-base-lint.txt
177-
'''
208+
cat start-scripts-lint.txt dockerfile-lint.txt
209+
"""
178210

179-
LINT_OUTPUT = sh(returnStdout: true, script: 'echo start-marklogic.sh: ;echo; cat start-marklogic-lint.txt; echo dockerfile-marklogic-server-centos: ; echo marklogic-deps-centos:base: ;echo; cat marklogic-deps-centos-base-lint.txt; echo marklogic-server-centos:base: ;echo; cat marklogic-server-centos-base-lint.txt').trim()
211+
LINT_OUTPUT = sh(returnStdout: true, script: "echo start-scripts-lint.txt: ;echo; cat start-scripts-lint.txt; echo; echo dockerfile-lint.txt: ; cat dockerfile-lint.txt; echo").trim()
180212

181-
sh '''
182-
rm -f start-marklogic-lint.txt marklogic-deps-centos-base-lint.txt marklogic-server-centos-base-lint.txt
183-
'''
213+
sh """
214+
rm -f start-scripts-lint.txt dockerfile-lint.txt
215+
"""
184216
}
185217

186-
void scan() {
218+
void vulnerabilityScan() {
187219
sh """
188-
make scan version=${mlVersion}-${env.platformString}-${env.dockerVersion} Jenkins=true
220+
make scan current_image=marklogic/marklogic-server-${dockerImageType}:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} Jenkins=true
189221
grep \'High\\|Critical\' scan-server-image.txt
190222
"""
191223

@@ -198,11 +230,15 @@ void scan() {
198230
}
199231

200232
void publishToInternalRegistry() {
201-
publishTag="${mlVersion}-${env.platformString}-${env.dockerVersion}"
202233
withCredentials([usernamePassword(credentialsId: 'builder-credentials-artifactory', passwordVariable: 'docker_password', usernameVariable: 'docker_user')]) {
203234
sh """
204235
echo "${docker_password}" | docker login --username ${docker_user} --password-stdin ${dockerRegistry}
205-
make push-mlregistry version=${publishTag}
236+
docker tag ${builtImage} ${dockerRegistry}/${builtImage}
237+
docker tag ${builtImage} ${dockerRegistry}/${publishImage}
238+
docker tag ${builtImage} ${dockerRegistry}/${latestTag}
239+
docker push ${dockerRegistry}/${builtImage}
240+
docker push ${dockerRegistry}/${publishImage}
241+
docker push ${dockerRegistry}/${latestTag}
206242
"""
207243

208244
}
@@ -216,13 +252,15 @@ void publishToInternalRegistry() {
216252
]]) {
217253
sh """
218254
aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin 713759029616.dkr.ecr.us-west-2.amazonaws.com
219-
docker tag local-dev/marklogic-server-centos:${publishTag} 713759029616.dkr.ecr.us-west-2.amazonaws.com/ml-docker-nightly:${publishTag}
220-
docker push 713759029616.dkr.ecr.us-west-2.amazonaws.com/ml-docker-nightly:${publishTag}
255+
docker tag ${builtImage} 713759029616.dkr.ecr.us-west-2.amazonaws.com/ml-docker-nightly:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion}
256+
docker tag ${builtImage} 713759029616.dkr.ecr.us-west-2.amazonaws.com/ml-docker-nightly:${marklogicVersion}-${env.dockerImageType}
257+
docker push 713759029616.dkr.ecr.us-west-2.amazonaws.com/ml-docker-nightly:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion}
258+
docker push 713759029616.dkr.ecr.us-west-2.amazonaws.com/ml-docker-nightly:${marklogicVersion}-${env.dockerImageType}
221259
"""
222260
}
223261
}
224262

225-
currentBuild.description = "Publish ${publishTag}"
263+
currentBuild.description = "Published"
226264
}
227265

228266
void publishTestResults() {
@@ -242,18 +280,25 @@ pipeline {
242280
skipStagesAfterUnstable()
243281
}
244282
triggers {
245-
parameterizedCron( env.BRANCH_NAME == 'develop' ? '''00 03 * * * % marklogicVersion=10
246-
00 04 * * * % marklogicVersion=11
247-
00 05 * * * % marklogicVersion=12''' : '')
283+
parameterizedCron( env.BRANCH_NAME == 'develop' ? '''00 02 * * * % marklogicVersion=11;dockerImageType=centos
284+
00 02 * * * % marklogicVersion=11;dockerImageType=ubi
285+
00 02 * * * % marklogicVersion=11;dockerImageType=ubi-rootless
286+
30 02 * * * % marklogicVersion=10;dockerImageType=centos
287+
30 02 * * * % marklogicVersion=10;dockerImageType=ubi
288+
30 02 * * * % marklogicVersion=10;dockerImageType=ubi-rootless
289+
00 03 * * * % marklogicVersion=12;dockerImageType=centos
290+
00 03 * * * % marklogicVersion=12;dockerImageType=ubi
291+
00 03 * * * % marklogicVersion=12;dockerImageType=ubi-rootless''' : '')
248292
}
249293
environment {
250294
QA_LICENSE_KEY = credentials('QA_LICENSE_KEY')
251295
}
252296

253297
parameters {
254298
string(name: 'emailList', defaultValue: emailList, description: 'List of email for build notification', trim: true)
255-
string(name: 'dockerVersion', defaultValue: '1.1.2', description: 'ML Docker version. This version along with ML rpm package version will be the image tag as {ML_Version}_{dockerVersion}', trim: true)
256-
string(name: 'platformString', defaultValue: 'centos', description: 'Platform string for Docker image version. Will be made part of the docker image tag', trim: true)
299+
string(name: 'dockerVersion', defaultValue: '2.0.0', description: 'ML Docker version. This version along with ML rpm package version will be the image tag as {ML_Version}_{dockerVersion}', trim: true)
300+
choice(name: 'dockerImageType', choices: 'ubi-rootless\nubi\ncentos', description: 'Platform type for Docker image. Will be made part of the docker image tag')
301+
string(name: 'upgradeDockerImage', defaultValue: '', description: 'Docker image for testing upgrades. Defaults to ubi image if left blank.\n Currently upgrading to ubi-rotless is not supported hence the test is skipped when ubi-rootless image is provided.', trim: true)
257302
choice(name: 'marklogicVersion', choices: '11\n12\n10', description: 'MarkLogic Server Branch. used to pick appropriate rpm')
258303
string(name: 'ML_RPM', defaultValue: '', description: 'URL for RPM to be used for Image creation. \n If left blank nightly ML rpm will be used.\n Please provide Jenkins accessible path e.g. /project/engineering or /project/qa', trim: true)
259304
string(name: 'ML_CONVERTERS', defaultValue: '', description: 'URL for the converters RPM to be included in the image creation \n If left blank the nightly ML Converters Package will be used.', trim: true)
@@ -277,7 +322,13 @@ pipeline {
277322

278323
stage('Build-Image') {
279324
steps {
280-
sh "make build version=${mlVersion}-${env.platformString}-${env.dockerVersion} build_branch=${env.BRANCH_NAME} package=${RPM} converters=${CONVERTERS}"
325+
buildDockerImage()
326+
}
327+
}
328+
329+
stage('Pull-Upgrade-Image') {
330+
steps {
331+
pullUpgradeDockerImage()
281332
}
282333
}
283334

@@ -289,7 +340,7 @@ pipeline {
289340

290341
stage('Scan') {
291342
steps {
292-
scan()
343+
vulnerabilityScan()
293344
}
294345
}
295346

@@ -307,7 +358,7 @@ pipeline {
307358
expression { return params.DOCKER_TESTS }
308359
}
309360
steps {
310-
sh "make docker-tests test_image=local-dev/marklogic-server-centos:${mlVersion}-${env.platformString}-${env.dockerVersion} version=${mlVersion}-${env.platformString}-${env.dockerVersion} build_branch=${env.BRANCH_NAME}"
361+
dockerTests()
311362
}
312363
}
313364

@@ -320,15 +371,17 @@ pipeline {
320371
}
321372
steps {
322373
publishToInternalRegistry()
374+
build job: 'MarkLogic-Docker-Kubernetes/docker/docker-nightly-builds-qa', wait: false, parameters: [string(name: 'dockerImageType', value: "${dockerImageType}"), string(name: 'marklogicVersion', value: "${RPMversion}")]
323375
}
324376
}
325377
}
326378

327379
post {
328380
always {
329381
sh '''
330-
cd src/centos
382+
cd src
331383
rm -rf *.rpm
384+
docker rm -f $(docker ps -a -q) || true
332385
docker system prune --force --filter "until=720h"
333386
docker volume prune --force
334387
docker image prune --force --all

0 commit comments

Comments
 (0)