Skip to content

Commit f73034b

Browse files
adamjohnwrightadamjohnwright
authored
Merge pull request #9 from reactome/develop
Merge develop to main
2 parents e7154e9 + fdddd60 commit f73034b

File tree

3 files changed

+62
-82
lines changed

3 files changed

+62
-82
lines changed

‎Jenkinsfile

+55-76
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
1-
import groovy.json.JsonSlurper
21
// This Jenkinsfile is used by Jenkins to run the UpdateDOIs step of Reactome's release.
32
// It requires that the UpdateStableIdentifiers step has been run successfully before it can be run.
4-
def currentRelease
5-
def previousRelease
3+
import org.reactome.release.jenkins.utilities.Utilities
4+
5+
// Shared library maintained at 'release-jenkins-utils' repository.
6+
def utils = new Utilities()
7+
68
pipeline {
79
agent any
810

@@ -11,36 +13,19 @@ pipeline {
1113
stage('Check if UpdateStableIdentifiers build succeeded'){
1214
steps{
1315
script{
14-
currentRelease = (pwd() =~ /Releases\/(\d+)\//)[0][1];
15-
previousRelease = (pwd() =~ /Releases\/(\d+)\//)[0][1].toInteger() - 1;
16-
// This queries the Jenkins API to confirm that the most recent build of UpdateStableIdentifiers was successful.
17-
def updateStIdsStatusUrl = httpRequest authentication: 'jenkinsKey', validResponseCodes: "${env.VALID_RESPONSE_CODES}", url: "${env.JENKINS_JOB_URL}/job/$currentRelease/job/UpdateStableIdentifiers/lastBuild/api/json"
18-
if (updateStIdsStatusUrl.getStatus() == 404) {
19-
error("UpdateStableIdentifiers has not yet been run. Please complete a successful build.")
20-
} else {
21-
def updateStIdsStatusJson = new JsonSlurper().parseText(updateStIdsStatusUrl.getContent())
22-
if(updateStIdsStatusJson['result'] != "SUCCESS"){
23-
error("Most recent UpdateStableIdentifiers build status: " + updateStIdsStatusJson['result'])
24-
}
25-
}
16+
utils.checkUpstreamBuildsSucceeded("Relational-Database-Updates/job/UpdateStableIdentifiers")
2617
}
2718
}
2819
}
29-
// This stage backs up the gk_central and release_current databases before they are modified.
20+
// This stage backs up the gk_central (on curator server) and release_current databases before they are modified.
3021
stage('Setup: Back up DBs'){
3122
steps{
3223
script{
33-
dir('update-dois'){
34-
withCredentials([usernamePassword(credentialsId: 'mySQLUsernamePassword', passwordVariable: 'pass', usernameVariable: 'user')]){
35-
withCredentials([usernamePassword(credentialsId: 'mySQLCuratorUsernamePassword', passwordVariable: 'curPass', usernameVariable: 'curUser')]){
36-
def release_current_before_update_dois_dump = "${env.RELEASE_CURRENT}_${currentRelease}_before_update_dois.dump"
37-
def central_before_update_dois_dump = "${env.GK_CENTRAL}_${currentRelease}_before_update_dois.dump"
38-
sh "mysqldump -u$user -p$pass ${env.RELEASE_CURRENT} > $release_current_before_update_dois_dump"
39-
sh "gzip -f $release_current_before_update_dois_dump"
40-
sh "mysqldump -u$curUser -p$curPass -h${env.CURATOR_SERVER} ${env.GK_CENTRAL} > $central_before_update_dois_dump"
41-
sh "gzip -f $central_before_update_dois_dump"
42-
}
43-
}
24+
withCredentials([usernamePassword(credentialsId: 'mySQLUsernamePassword', passwordVariable: 'pass', usernameVariable: 'user')]){
25+
utils.takeDatabaseDumpAndGzip("${env.RELEASE_CURRENT_DB}", "update_dois", "before", "${env.RELEASE_SERVER}")
26+
}
27+
withCredentials([usernamePassword(credentialsId: 'mySQLCuratorUsernamePassword', passwordVariable: 'pass', usernameVariable: 'user')]){
28+
utils.takeDatabaseDumpAndGzip("${env.GK_CENTRAL_DB}", "update_dois", "before", "${env.CURATOR_SERVER}")
4429
}
4530
}
4631
}
@@ -49,9 +34,7 @@ pipeline {
4934
stage('Setup: Build jar file'){
5035
steps{
5136
script{
52-
dir('update-dois'){
53-
sh "mvn clean compile assembly:single"
54-
}
37+
utils.buildJarFile()
5538
}
5639
}
5740
}
@@ -60,47 +43,39 @@ pipeline {
6043
stage('Main: UpdateDOIs Test Run'){
6144
steps{
6245
script{
63-
dir('update-dois'){
64-
withCredentials([file(credentialsId: 'Config', variable: 'ConfigFile')]) {
65-
sh "touch src/main/resources/UpdateDOIs.report"
66-
sh "java -jar target/update-dois-${env.UPDATE_DOIS_VERSION}-jar-with-dependencies.jar $ConfigFile"
67-
}
46+
withCredentials([file(credentialsId: 'Config', variable: 'ConfigFile')]) {
47+
sh "touch src/main/resources/UpdateDOIs.report"
48+
sh "java -jar target/update-dois-*-jar-with-dependencies.jar $ConfigFile"
6849
}
6950
}
7051
}
7152
}
7253
// This stage takes the generated report file and sends it to the curator overseeing release.
7354
// Before moving onto the next stage of UpdateDOIs, their confirmation that the contents of the report file are correct is needed.
74-
/*
75-
stage('Send email of updateable DOIs to curator'){
55+
stage('Main: Send email of updateable DOIs to curator'){
7656
steps{
7757
script{
78-
emailext (
79-
body: "This is an automated message. Please review the attached file of Pathway DOIs to be updated and confirm they are correct with the developer running release. Thanks!",
80-
to: '$DEFAULT_RECIPIENTS',
81-
subject: "UpdateDOIs List for v${currentRelease}",
82-
attachmentsPattern: "**///doisToBeUpdated-v${currentRelease}.txt" <- This pattern (**/) messses up multi-line comments. When uncommenting, remove 2/3 backslashes before 'doisToBeUpdated'
83-
/* )
58+
def releaseVersion = utils.getReleaseVersion()
59+
def doisToBeUpdatedFile = "doisToBeUpdated-v${releaseVersion}.txt"
60+
def emailSubject = "UpdateStableIdentifiers complete & UpdateDOIs List for v${releaseVersion}"
61+
def emailBody = "This is an automated message: UpdateDOIs has completed a test run to determine which Pathway DOIs will be updated in the \'${env.RELEASE_CURRENT_DB}\' and \'${env.GK_CENTRAL_DB}\' databases. Please review the attached ${doisToBeUpdatedFile} file and let the developer running Release know if they look correct. \n\nThanks!"
62+
def emailAttachments = "${doisToBeUpdatedFile}"
63+
64+
utils.sendEmailWithAttachment("$emailSubject", "$emailBody", "$emailAttachments")
8465
}
8566
}
8667
}
87-
*/
8868
// UpdateDOIs should pause at this stage until the curator confirms the report file is correct. Once they do, respond with 'yes' to the user input form that Jenkins brings up.
8969
stage('User Input Required: Confirm DOIs'){
9070
steps{
9171
script{
92-
// This brings up a user input form, asking for confirmation that the curator overseeing release approves of the report file they received.
72+
def releaseVersion = utils.getReleaseVersion()
73+
// This asks user to confirm that the UpdateDOIs.report file has been approved by Curation.
9374
def userInput = input(
94-
id: 'userInput', message: "Has a curator confirmed that the list of DOIs to be updated in doisToBeUpdated-v${currentRelease}.txt is correct? (yes/no)",
75+
id: 'userInput', message: "Proceed once \'doisToBeUpdated-v${releaseVersion}.txt\' has been approved by curation. It should have been sent in an email after the test run step.",
9576
parameters: [
96-
[$class: 'TextParameterDefinition', defaultValue: '', description: 'Confirmation of updateable DOIs', name: 'response']
77+
[$class: 'BooleanParameterDefinition', defaultValue: true, name: 'response']
9778
])
98-
99-
if (userInput.toLowerCase().startsWith("y")) {
100-
echo("Proceeding to UpdateDOIs step.")
101-
} else {
102-
error("Please confirm output of UpdateDOIs Test Run matches DOIs that should be updated")
103-
}
10479
}
10580
}
10681
}
@@ -109,10 +84,9 @@ pipeline {
10984
stage('Main: UpdateDOIs'){
11085
steps{
11186
script{
112-
dir('update-dois'){
113-
withCredentials([file(credentialsId: 'Config', variable: 'ConfigFile')]) {
114-
sh "java -jar target/update-dois-${env.UPDATE_DOIS_VERSION}-jar-with-dependencies.jar $ConfigFile doisToBeUpdated-v${currentRelease}.txt"
115-
}
87+
withCredentials([file(credentialsId: 'Config', variable: 'ConfigFile')]) {
88+
def releaseVersion = utils.getReleaseVersion()
89+
sh "java -jar target/update-dois-*-jar-with-dependencies.jar $ConfigFile doisToBeUpdated-v${releaseVersion}.txt"
11690
}
11791
}
11892
}
@@ -121,33 +95,38 @@ pipeline {
12195
stage('Post: Backup DBs'){
12296
steps{
12397
script{
124-
dir('update-dois'){
125-
withCredentials([usernamePassword(credentialsId: 'mySQLUsernamePassword', passwordVariable: 'pass', usernameVariable: 'user')]){
126-
withCredentials([usernamePassword(credentialsId: 'mySQLCuratorUsernamePassword', passwordVariable: 'curPass', usernameVariable: 'curUser')]){
127-
def release_current_after_update_dois_dump = "${env.RELEASE_CURRENT}_${currentRelease}_after_update_dois.dump"
128-
def central_before_after_dois_dump = "${env.GK_CENTRAL}_${currentRelease}_after_update_dois.dump"
129-
sh "mysqldump -u$user -p$pass ${env.RELEASE_CURRENT} > $release_current_after_update_dois_dump"
130-
sh "gzip -f $release_current_after_update_dois_dump"
131-
sh "mysqldump -u$curUser -p$curPass -h${env.CURATOR_SERVER} ${env.GK_CENTRAL} > $central_before_after_dois_dump"
132-
sh "gzip -f $central_before_after_dois_dump"
133-
}
134-
}
98+
withCredentials([usernamePassword(credentialsId: 'mySQLUsernamePassword', passwordVariable: 'pass', usernameVariable: 'user')]){
99+
utils.takeDatabaseDumpAndGzip("${env.RELEASE_CURRENT_DB}", "update_dois", "after", "${env.RELEASE_SERVER}")
100+
}
101+
withCredentials([usernamePassword(credentialsId: 'mySQLCuratorUsernamePassword', passwordVariable: 'pass', usernameVariable: 'user')]){
102+
utils.takeDatabaseDumpAndGzip("${env.GK_CENTRAL_DB}", "update_dois", "after", "${env.CURATOR_SERVER}")
135103
}
136104
}
137105
}
138106
}
139-
// This stage archives all logs and database backups produced by UpdateDOIs.
140-
stage('Archive logs and backups'){
107+
// All databases, logs, and data files generated by this step are compressed before moving them to the Reactome S3 bucket. All files are then deleted.
108+
stage('Post: Archive Outputs'){
141109
steps{
142110
script{
143-
dir('update-dois'){
144-
sh "mkdir -p archive/${currentRelease}/logs"
145-
sh "mv --backup=numbered *_${currentRelease}_*.dump.gz archive/${currentRelease}/"
146-
sh "gzip logs/*"
147-
sh "mv logs/* archive/${currentRelease}/logs/"
148-
}
111+
def releaseVersion = utils.getReleaseVersion()
112+
def dataFiles = ["doisToBeUpdated-v${releaseVersion}.txt"]
113+
// Log files appear in the 'logs' folder by default, and so don't need to be specified here.
114+
def logFiles = []
115+
def foldersToDelete = []
116+
utils.cleanUpAndArchiveBuildFiles("update_dois", dataFiles, logFiles, foldersToDelete)
149117
}
150118
}
151119
}
120+
// This sends an email notifying the mailing list that both the UpdateStableIdentifiers and UpdateDOIs steps have completed. This indicates that gk_central can be reopened.
121+
stage('Post: Send completion email') {
122+
steps{
123+
script{
124+
def releaseVersion = utils.getReleaseVersion()
125+
def emailSubject = "UpdateStableIdentifier and UpdateDOIs complete for v${releaseVersion}"
126+
def emailBody = "Hello,\n\nThis is an automated message from Jenkins regarding an update for v${releaseVersion}: Both UpdateStableIdentifiers and UpdateDOIs steps have completed. ${env.GK_CENTRAL_DB} can likely be reopened, but Curation should get \'Human\' confirmation before doing so. \n\nThanks!"
127+
utils.sendEmail("${emailSubject}", "${emailBody}")
128+
}
129+
}
130+
}
152131
}
153132
}

‎README.md

+3-3
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,8 @@ More technically, it will find all 'Pathway' instances with a 'doi' attribute th
88
from the instance's 'stableIdentifier' attribute.
99

1010
New features and tests have been implemented for this iteration of UpdateDOIs:
11-
- Runtime tests ensure conchordance of the instance's being updated in the Test Reactome and GK Central databases
12-
- Users can provide a report file (UpdateDOIs.report) of DOIs that are expected to be updated. It can be created from the data found <a href="https://docs.google.com/spreadsheets/d/1KtZ_Z3rvBELroubmeO1ai5otbsS26QpXZn6au-oSCWw/edit#gid=1011530219">here</a>. This is explained further in <a href="https://github.com/reactome/data-release-pipeline/new/develop/update-dois#updatedoisreport">UpdateDOIs.report</a> section.
11+
- Runtime tests ensure concordance of the instance's being updated in the Test Reactome and GK Central databases
12+
- Users can provide a report file (UpdateDOIs.report) of DOIs that are expected to be updated. It can be created from the data found <a href="https://docs.google.com/spreadsheets/d/1KtZ_Z3rvBELroubmeO1ai5otbsS26QpXZn6au-oSCWw/edit#gid=1011530219">here</a>. This is explained further in <a href="#updatedoisreport">UpdateDOIs.report</a> section.
1313
- If the report has been provided, the script will report any unexpected behaviour and attempt to suggest to the user why it happened
1414

1515
<h2>Configuration</h2>
@@ -66,7 +66,7 @@ To use this feature, follow these steps:
6666
2) Create a new 'UpdateDOIs.report' file or remove the contents of the old one found at `src/main/resources`
6767
3) For each row of the spreadsheet that contains a new curation:
6868
- Prepend each 'stableID' with "10.3180/" to create the new DOI (eg: 10.3180/R-HSA-123456789.1)
69-
- Copy into UpdateDOIs.report the new DOI and 'name', seperated by a comma (eg: 10.3180/R-HSA-123456789.1,Reactome Annotation Example)
69+
- Copy into UpdateDOIs.report the new DOI and 'name', separated by a comma (eg: 10.3180/R-HSA-123456789.1,Reactome Annotation Example)
7070
4) If you created a new new UpdateDOIs.report file, move it to the `src/main/resources` folder.
7171

7272
<b>Example</b>:

‎pom.xml

+4-3
Original file line numberDiff line numberDiff line change
@@ -100,9 +100,10 @@
100100
<version>3.2.0</version>
101101
<configuration>
102102
<archive>
103-
<manifest>
104-
<mainClass>org.reactome.release.updateDOIs.Main</mainClass>
105-
</manifest>
103+
<manifestEntries>
104+
<Multi-Release>true</Multi-Release>
105+
<Main-Class>org.reactome.release.updateDOIs.Main</Main-Class>
106+
</manifestEntries>
106107
</archive>
107108
<descriptorRefs>
108109
<descriptorRef>jar-with-dependencies</descriptorRef>

0 commit comments

Comments
 (0)