|
1 | | -def githubUsernameToSlackName(github_author) { |
2 | | - // Add team members as necessary |
3 | | - def mapping = [ |
4 | | - "Jim Albright": "albrja", |
5 | | - "Steve Bachmeier": "sbachmei", |
6 | | - "Hussain Jafari": "hjafari", |
7 | | - "Patrick Nast": "pnast", |
8 | | - "Rajan Mudambi": "rmudambi", |
9 | | - ] |
10 | | - return mapping.get(github_author, "channel") |
11 | | -} |
12 | | - |
13 | | -pipeline_name="vivarium" |
14 | | -conda_env_name="${pipeline_name}-${BRANCH_NAME}-${BUILD_NUMBER}" |
15 | | -conda_env_path="/tmp/${conda_env_name}" |
16 | | -// defaults for conda and pip are a local directory /svc-simsci for improved speed. |
17 | | -// In the past, we used /ihme/code/* on the NFS (which is slower) |
18 | | -shared_path="/svc-simsci" |
19 | | -// comma separated string list of branches to run periodic builds on |
20 | | -scheduled_branches = "main" |
21 | | -CRON_SETTINGS = scheduled_branches.split(',').collect{it.trim()}.contains(BRANCH_NAME) ? 'H H(20-23) * * *' : '' |
22 | | - |
23 | | -pipeline { |
24 | | - // This agent runs as svc-simsci on node simsci-ci-coordinator-01. |
25 | | - // It has access to standard IHME filesystems and singularity |
26 | | - agent { label "coordinator" } |
27 | | - triggers { |
28 | | - cron(CRON_SETTINGS) |
29 | | - } |
30 | | - |
31 | | - options { |
32 | | - // Keep 100 old builds. |
33 | | - buildDiscarder logRotator(numToKeepStr: "100") |
34 | | - |
35 | | - // Wait 60 seconds before starting the build. |
36 | | - // If another commit enters the build queue in this time, the first build will be discarded. |
37 | | - quietPeriod(60) |
38 | | - |
39 | | - // Fail immediately if any part of a parallel stage fails |
40 | | - parallelsAlwaysFailFast() |
41 | | - } |
42 | | - |
43 | | - parameters { |
44 | | - booleanParam( |
45 | | - name: "DEPLOY_OVERRIDE", |
46 | | - defaultValue: false, |
47 | | - description: "Whether to deploy despite building a non-default branch. Builds of the default branch are always deployed." |
48 | | - ) |
49 | | - booleanParam( |
50 | | - name: "IS_CRON", |
51 | | - defaultValue: true, |
52 | | - description: "Indicates a recurring build. Used to skip deployment steps." |
53 | | - ) |
54 | | - string( |
55 | | - name: "SLACK_TO", |
56 | | - defaultValue: "simsci-ci-status", |
57 | | - description: "The Slack channel to send messages to." |
58 | | - ) |
59 | | - booleanParam( |
60 | | - name: "DEBUG", |
61 | | - defaultValue: false, |
62 | | - description: "Used as needed for debugging purposes." |
63 | | - ) |
64 | | - } |
65 | | - |
66 | | - stages { |
67 | | - stage("Initialization") { |
68 | | - steps { |
69 | | - script { |
70 | | - // Use the name of the branch in the build name |
71 | | - currentBuild.displayName = "#${BUILD_NUMBER} ${GIT_BRANCH}" |
72 | | - } |
73 | | - } |
74 | | - } |
75 | | - |
76 | | - stage("Python version matrix") { |
77 | | - // we don't want to go to deployment if any of the matrix branches fail |
78 | | - failFast true |
79 | | - matrix { |
80 | | - // customWorkspace setting must be ran within a node |
81 | | - agent { |
82 | | - node { |
83 | | - // Run child tasks on simsci-jenkinsagent-ci-p01. |
84 | | - label "matrix-tasks" |
85 | | - } |
86 | | - } |
87 | | - axes { |
88 | | - axis { |
89 | | - // parallelize by python minor version |
90 | | - name 'PYTHON_VERSION' |
91 | | - values "3.10", "3.11" |
92 | | - } |
93 | | - } |
94 | | - |
95 | | - environment { |
96 | | - // Get the branch being built and strip everything but the text after the last "/" |
97 | | - BRANCH = sh(script: "echo ${GIT_BRANCH} | rev | cut -d '/' -f1 | rev", returnStdout: true).trim() |
98 | | - TIMESTAMP = sh(script: 'date', returnStdout: true) |
99 | | - // Specify the path to the .condarc file via environment variable. |
100 | | - // This file configures the shared conda package cache. |
101 | | - CONDARC = "${shared_path}/miniconda3/.condarc" |
102 | | - CONDA_BIN_PATH = "${shared_path}/miniconda3/bin" |
103 | | - // Specify conda env by build number so that we don't have collisions if builds from |
104 | | - // different branches happen concurrently. |
105 | | - PYTHON_DEPLOY_VERSION = "3.11" |
106 | | - CONDA_ENV_NAME = "${conda_env_name}" |
107 | | - CONDA_ENV_PATH = "${conda_env_path}_${PYTHON_VERSION}" |
108 | | - // Set the Pip cache. |
109 | | - XDG_CACHE_HOME = "${shared_path}/pip-cache" |
110 | | - // Jenkins commands run in separate processes, so need to activate the environment every |
111 | | - // time we run pip, poetry, etc. |
112 | | - ACTIVATE = "source ${CONDA_BIN_PATH}/activate ${CONDA_ENV_PATH} &> /dev/null" |
113 | | - } |
114 | | - |
115 | | - stages { |
116 | | - stage("Debug Info") { |
117 | | - steps { |
118 | | - echo "Jenkins pipeline run timestamp: ${TIMESTAMP}" |
119 | | - // Display parameters used. |
120 | | - echo """Parameters: |
121 | | - DEPLOY_OVERRIDE: ${params.DEPLOY_OVERRIDE}""" |
122 | | - |
123 | | - // Display environment variables from Jenkins. |
124 | | - echo """Environment: |
125 | | - ACTIVATE: '${ACTIVATE}' |
126 | | - BUILD_NUMBER: '${BUILD_NUMBER}' |
127 | | - BRANCH: '${BRANCH}' |
128 | | - CONDARC: '${CONDARC}' |
129 | | - CONDA_BIN_PATH: '${CONDA_BIN_PATH}' |
130 | | - CONDA_ENV_NAME: '${CONDA_ENV_NAME}' |
131 | | - CONDA_ENV_PATH: '${CONDA_ENV_PATH}' |
132 | | - GIT_BRANCH: '${GIT_BRANCH}' |
133 | | - JOB_NAME: '${JOB_NAME}' |
134 | | - WORKSPACE: '${WORKSPACE}' |
135 | | - XDG_CACHE_HOME: '${XDG_CACHE_HOME}'""" |
136 | | - } |
137 | | - } |
138 | | - |
139 | | - stage("Build Environment") { |
140 | | - environment { |
141 | | - // Command for activating the base environment. Activating the base environment sets |
142 | | - // the correct path to the conda binary which is used to create a new conda env. |
143 | | - ACTIVATE_BASE = "source ${CONDA_BIN_PATH}/activate &> /dev/null" |
144 | | - } |
145 | | - steps { |
146 | | - // The env should have been cleaned out after the last build, but delete it again |
147 | | - // here just to be safe. |
148 | | - sh "rm -rf ${CONDA_ENV_PATH}" |
149 | | - sh "${ACTIVATE_BASE} && make build-env PYTHON_VERSION=${PYTHON_VERSION}" |
150 | | - // open permissions for test users to create file in workspace |
151 | | - sh "chmod 777 ${WORKSPACE}" |
152 | | - } |
153 | | - } |
154 | | - |
155 | | - stage("Install Package") { |
156 | | - steps { |
157 | | - sh "${ACTIVATE} && make install \"ARGS=${GIT_BRANCH}\"" |
158 | | - } |
159 | | - } |
160 | | - |
161 | | - // Quality Checks |
162 | | - stage("Format") { |
163 | | - steps { |
164 | | - sh "${ACTIVATE} && make format" |
165 | | - } |
166 | | - } |
167 | | - |
168 | | - // stage("Lint") { |
169 | | - // steps { |
170 | | - // sh "${ACTIVATE} && make lint" |
171 | | - // } |
172 | | - // } |
173 | | - |
174 | | - // Tests |
175 | | - // removable, if passwords can be exported to env. securely without bash indirection |
176 | | - stage("Run Integration Tests") { |
177 | | - steps { |
178 | | - sh "${ACTIVATE} && make integration" |
179 | | - publishHTML([ |
180 | | - allowMissing: true, |
181 | | - alwaysLinkToLastBuild: false, |
182 | | - keepAll: true, |
183 | | - reportDir: "output/htmlcov_integration", |
184 | | - reportFiles: "index.html", |
185 | | - reportName: "Coverage Report - Integration tests", |
186 | | - reportTitles: '' |
187 | | - ]) |
188 | | - } |
189 | | - } |
190 | | - |
191 | | - // Build |
192 | | - stage('Build and Deploy') { |
193 | | - when { |
194 | | - expression { "${PYTHON_DEPLOY_VERSION}" == "${PYTHON_VERSION}" } |
195 | | - } |
196 | | - stages { |
197 | | - stage("Build Docs") { |
198 | | - steps { |
199 | | - sh "${ACTIVATE} && make build-doc" |
200 | | - } |
201 | | - } |
202 | | - stage("Build Package") { |
203 | | - steps { |
204 | | - sh "${ACTIVATE} && make build-package" |
205 | | - } |
206 | | - } |
207 | | - } // stages within build and deploy |
208 | | - } // build and deploy stage |
209 | | - } // stages bracket within Python matrix |
210 | | - post { |
211 | | - always { |
212 | | - sh "${ACTIVATE} && make clean" |
213 | | - sh "rm -rf ${CONDA_ENV_PATH}" |
214 | | - // Generate a message to send to Slack. |
215 | | - script { |
216 | | - if (env.BRANCH == "main") { |
217 | | - channelName = "simsci-ci-status" |
218 | | - } else { |
219 | | - channelName = "simsci-ci-status-test" |
220 | | - } |
221 | | - // Run git command to get the author of the last commit |
222 | | - developerID = sh( |
223 | | - script: "git log -1 --pretty=format:'%an'", |
224 | | - returnStdout: true |
225 | | - ).trim() |
226 | | - slackID = githubUsernameToSlackName(developerID) |
227 | | - slackMessage = """ |
228 | | - Job: *${env.JOB_NAME}* |
229 | | - Build number: #${env.BUILD_NUMBER} |
230 | | - Build status: *${currentBuild.result}* |
231 | | - Author: @${slackID} |
232 | | - Build details: <${env.BUILD_URL}/console|See in web console> |
233 | | - """.stripIndent() |
234 | | - } |
235 | | - |
236 | | - // Delete the workspace directory. |
237 | | - deleteDir() |
238 | | - } |
239 | | - failure { |
240 | | - echo "This build triggered by ${developerID} failed on ${GIT_BRANCH}. Sending a failure message to Slack." |
241 | | - slackSend channel: "#${channelName}", |
242 | | - message: slackMessage, |
243 | | - teamDomain: "ihme", |
244 | | - tokenCredentialId: "slack" |
245 | | - } |
246 | | - success { |
247 | | - script { |
248 | | - if (params.DEBUG) { |
249 | | - echo 'Debug is enabled. Sending a success message to Slack.' |
250 | | - slackSend channel: "#${channelName}", |
251 | | - message: slackMessage, |
252 | | - teamDomain: "ihme", |
253 | | - tokenCredentialId: "slack" |
254 | | - } else { |
255 | | - echo 'Debug is not enabled. No success message will be sent to Slack.' |
256 | | - } |
257 | | - } |
258 | | - } |
259 | | - } // post bracket |
260 | | - } // Python matrix bracket |
261 | | - } // Python matrix stage bracket |
262 | | - } // stages bracket |
263 | | -} |
| 1 | +@Library("vivarium_build_utils") _ |
| 2 | +reusable_pipeline(scheduled_branches: ["main"], test_types: ["integration"], python_versions: ["3.10", "3.11"]) |
0 commit comments