|
1 | | -def githubUsernameToSlackName(github_author) { |
2 | | - // Add team members as necessary |
3 | | - def mapping = [ |
4 | | - "Jim Albright": "albrja", |
5 | | - "Steve Bachmeier": "sbachmei", |
6 | | - "Hussain Jafari": "hjafari", |
7 | | - "Patrick Nast": "pnast", |
8 | | - "Rajan Mudambi": "rmudambi", |
9 | | - ] |
10 | | - return mapping.get(github_author, "channel") |
11 | | -} |
12 | | - |
13 | | -pipeline_name="gbd_mapping" |
14 | | -conda_env_name="${pipeline_name}-${BRANCH_NAME}-${BUILD_NUMBER}" |
15 | | -conda_env_path="/tmp/${conda_env_name}" |
16 | | -// defaults for conda and pip are a local directory /svc-simsci for improved speed. |
17 | | -// In the past, we used /ihme/code/* on the NFS (which is slower) |
18 | | -shared_path="/svc-simsci" |
19 | | -// comma separated string list of branches to run periodic builds on |
20 | | -scheduled_branches = "main" |
21 | | -CRON_SETTINGS = scheduled_branches.split(',').collect{it.trim()}.contains(BRANCH_NAME) ? 'H H(20-23) * * *' : '' |
22 | | - |
23 | | -pipeline { |
24 | | - // This agent runs as svc-simsci on node simsci-ci-coordinator-01. |
25 | | - // It has access to standard IHME filesystems and singularity |
26 | | - agent { label "coordinator" } |
27 | | - |
28 | | - options { |
29 | | - // Keep 100 old builds. |
30 | | - buildDiscarder logRotator(numToKeepStr: "100") |
31 | | - |
32 | | - // Wait 60 seconds before starting the build. |
33 | | - // If another commit enters the build queue in this time, the first build will be discarded. |
34 | | - quietPeriod(60) |
35 | | - |
36 | | - // Fail immediately if any part of a parallel stage fails |
37 | | - parallelsAlwaysFailFast() |
38 | | - } |
39 | | - |
40 | | - parameters { |
41 | | - booleanParam( |
42 | | - name: "DEPLOY_OVERRIDE", |
43 | | - defaultValue: false, |
44 | | - description: "Whether to deploy despite building a non-default branch. Builds of the default branch are always deployed." |
45 | | - ) |
46 | | - booleanParam( |
47 | | - name: "IS_CRON", |
48 | | - defaultValue: true, |
49 | | - description: "Indicates a recurring build. Used to skip deployment steps." |
50 | | - ) |
51 | | - string( |
52 | | - name: "SLACK_TO", |
53 | | - defaultValue: "simsci-ci-status-test", |
54 | | - description: "The Slack channel to send messages to." |
55 | | - ) |
56 | | - booleanParam( |
57 | | - name: "DEBUG", |
58 | | - defaultValue: false, |
59 | | - description: "Used as needed for debugging purposes." |
60 | | - ) |
61 | | - } |
62 | | - triggers { |
63 | | - cron(CRON_SETTINGS) |
64 | | - } |
65 | | - stages { |
66 | | - stage("Initialization") { |
67 | | - steps { |
68 | | - script { |
69 | | - // Use the name of the branch in the build name |
70 | | - currentBuild.displayName = "#${BUILD_NUMBER} ${GIT_BRANCH}" |
71 | | - } |
72 | | - } |
73 | | - } |
74 | | - |
75 | | - stage("Python version matrix") { |
76 | | - // we don't want to go to deployment if any of the matrix branches fail |
77 | | - failFast true |
78 | | - matrix { |
79 | | - // customWorkspace setting must be ran within a node |
80 | | - agent { |
81 | | - node { |
82 | | - // Run child tasks on simsci-jenkinsagent-ci-p01. |
83 | | - label "matrix-tasks" |
84 | | - } |
85 | | - } |
86 | | - axes { |
87 | | - axis { |
88 | | - // parallelize by python minor version |
89 | | - name 'PYTHON_VERSION' |
90 | | - values "3.10", "3.11" |
91 | | - } |
92 | | - } |
93 | | - |
94 | | - environment { |
95 | | - // Get the branch being built and strip everything but the text after the last "/" |
96 | | - BRANCH = sh(script: "echo ${GIT_BRANCH} | rev | cut -d '/' -f1 | rev", returnStdout: true).trim() |
97 | | - TIMESTAMP = sh(script: 'date', returnStdout: true) |
98 | | - // Specify the path to the .condarc file via environment variable. |
99 | | - // This file configures the shared conda package cache. |
100 | | - CONDARC = "${shared_path}/miniconda3/.condarc" |
101 | | - CONDA_BIN_PATH = "${shared_path}/miniconda3/bin" |
102 | | - // Specify conda env by build number so that we don't have collisions if builds from |
103 | | - // different branches happen concurrently. |
104 | | - PYTHON_DEPLOY_VERSION = "3.11" |
105 | | - CONDA_ENV_NAME = "${conda_env_name}" |
106 | | - CONDA_ENV_PATH = "${conda_env_path}_${PYTHON_VERSION}" |
107 | | - // Set the Pip cache. |
108 | | - XDG_CACHE_HOME = "${shared_path}/pip-cache" |
109 | | - // Jenkins commands run in separate processes, so need to activate the environment every |
110 | | - // time we run pip, poetry, etc. |
111 | | - ACTIVATE = "source ${CONDA_BIN_PATH}/activate ${CONDA_ENV_PATH} &> /dev/null" |
112 | | - } |
113 | | - |
114 | | - stages { |
115 | | - stage("Debug Info") { |
116 | | - steps { |
117 | | - echo "Jenkins pipeline run timestamp: ${TIMESTAMP}" |
118 | | - // Display parameters used. |
119 | | - echo """Parameters: |
120 | | - DEPLOY_OVERRIDE: ${params.DEPLOY_OVERRIDE}""" |
121 | | - |
122 | | - // Display environment variables from Jenkins. |
123 | | - echo """Environment: |
124 | | - ACTIVATE: '${ACTIVATE}' |
125 | | - BUILD_NUMBER: '${BUILD_NUMBER}' |
126 | | - BRANCH: '${BRANCH}' |
127 | | - CONDARC: '${CONDARC}' |
128 | | - CONDA_BIN_PATH: '${CONDA_BIN_PATH}' |
129 | | - CONDA_ENV_NAME: '${CONDA_ENV_NAME}' |
130 | | - CONDA_ENV_PATH: '${CONDA_ENV_PATH}' |
131 | | - GIT_BRANCH: '${GIT_BRANCH}' |
132 | | - JOB_NAME: '${JOB_NAME}' |
133 | | - WORKSPACE: '${WORKSPACE}' |
134 | | - XDG_CACHE_HOME: '${XDG_CACHE_HOME}'""" |
135 | | - } |
136 | | - } |
137 | | - |
138 | | - stage("Build Environment") { |
139 | | - environment { |
140 | | - // Command for activating the base environment. Activating the base environment sets |
141 | | - // the correct path to the conda binary which is used to create a new conda env. |
142 | | - ACTIVATE_BASE = "source ${CONDA_BIN_PATH}/activate &> /dev/null" |
143 | | - } |
144 | | - steps { |
145 | | - // The env should have been cleaned out after the last build, but delete it again |
146 | | - // here just to be safe. |
147 | | - sh "rm -rf ${CONDA_ENV_PATH}" |
148 | | - sh "${ACTIVATE_BASE} && make build-env PYTHON_VERSION=${PYTHON_VERSION}" |
149 | | - // open permissions for test users to create file in workspace |
150 | | - sh "chmod 777 ${WORKSPACE}" |
151 | | - } |
152 | | - } |
153 | | - |
154 | | - stage("Install Package") { |
155 | | - steps { |
156 | | - sh "${ACTIVATE} && make install \"ARGS=${GIT_BRANCH}\"" |
157 | | - } |
158 | | - } |
159 | | - |
160 | | - // Quality Checks |
161 | | - stage("Format") { |
162 | | - steps { |
163 | | - sh "${ACTIVATE} && make format" |
164 | | - } |
165 | | - } |
166 | | - |
167 | | - // stage("Lint") { |
168 | | - // steps { |
169 | | - // sh "${ACTIVATE} && make lint" |
170 | | - // } |
171 | | - // } |
172 | | - |
173 | | - // Tests |
174 | | - // removable, if passwords can be exported to env. securely without bash indirection |
175 | | - stage("Run Integration Tests") { |
176 | | - steps { |
177 | | - sh "${ACTIVATE} && make integration" |
178 | | - publishHTML([ |
179 | | - allowMissing: true, |
180 | | - alwaysLinkToLastBuild: false, |
181 | | - keepAll: true, |
182 | | - reportDir: "output/htmlcov_integration", |
183 | | - reportFiles: "index.html", |
184 | | - reportName: "Coverage Report - Integration tests", |
185 | | - reportTitles: '' |
186 | | - ]) |
187 | | - } |
188 | | - } |
189 | | - |
190 | | - // Build |
191 | | - stage('Build and Deploy') { |
192 | | - when { |
193 | | - expression { "${PYTHON_DEPLOY_VERSION}" == "${PYTHON_VERSION}" } |
194 | | - } |
195 | | - stages { |
196 | | - stage("Build Docs") { |
197 | | - steps { |
198 | | - sh "${ACTIVATE} && make build-doc" |
199 | | - } |
200 | | - } |
201 | | - stage("Build Package") { |
202 | | - steps { |
203 | | - sh "${ACTIVATE} && make build-package" |
204 | | - } |
205 | | - } |
206 | | - } // stages within build and deploy |
207 | | - } // build and deploy stage |
208 | | - } // stages bracket within Python matrix |
209 | | - post { |
210 | | - always { |
211 | | - // Generate a message to send to Slack. |
212 | | - script { |
213 | | - if (env.BRANCH == "main") { |
214 | | - channelName = "simsci-ci-status" |
215 | | - } else { |
216 | | - channelName = "simsci-ci-status-test" |
217 | | - } |
218 | | - // Run git command to get the author of the last commit |
219 | | - developerID = sh( |
220 | | - script: "git log -1 --pretty=format:'%an'", |
221 | | - returnStdout: true |
222 | | - ).trim() |
223 | | - slackID = githubUsernameToSlackName(developerID) |
224 | | - slackMessage = """ |
225 | | - Job: *${env.JOB_NAME}* |
226 | | - Build number: #${env.BUILD_NUMBER} |
227 | | - Build status: *${currentBuild.result}* |
228 | | - Author: @${slackID} |
229 | | - Build details: <${env.BUILD_URL}/console|See in web console> |
230 | | - """.stripIndent() |
231 | | - } |
232 | | - } |
233 | | - failure { |
234 | | - echo "This build triggered by ${developerID} failed on ${GIT_BRANCH}. Sending a failure message to Slack." |
235 | | - slackSend channel: "#${channelName}", |
236 | | - message: slackMessage, |
237 | | - teamDomain: "ihme", |
238 | | - tokenCredentialId: "slack" |
239 | | - } |
240 | | - success { |
241 | | - script { |
242 | | - if (params.DEBUG) { |
243 | | - echo 'Debug is enabled. Sending a success message to Slack.' |
244 | | - slackSend channel: "#${channelName}", |
245 | | - message: slackMessage, |
246 | | - teamDomain: "ihme", |
247 | | - tokenCredentialId: "slack" |
248 | | - } else { |
249 | | - echo 'Debug is not enabled. No success message will be sent to Slack.' |
250 | | - } |
251 | | - } |
252 | | - } |
253 | | - cleanup { // cleanup for python matrix workspaces |
254 | | - sh "${ACTIVATE} && make clean" |
255 | | - sh "rm -rf ${CONDA_ENV_PATH}" |
256 | | - cleanWs() |
257 | | - dir("${WORKSPACE}@tmp"){ |
258 | | - deleteDir() |
259 | | - } |
260 | | - } |
261 | | - } // post bracket |
262 | | - } // Python matrix bracket |
263 | | - } // Python matrix stage bracket |
264 | | - } // stages bracket |
265 | | - post { |
266 | | - cleanup { // cleanup for outer workspace |
267 | | - cleanWs() |
268 | | - dir("${WORKSPACE}@tmp"){ |
269 | | - deleteDir() |
270 | | - } |
271 | | - } |
272 | | - } |
273 | | -} |
| 1 | +@Library("vivarium_build_utils") _ |
| 2 | +reusable_pipeline(scheduled_branches: ["main"], test_types: ["integration"], python_versions: ["3.10", "3.11"]) |
0 commit comments