Skip to content

Commit bedfd51

Browse files
adamjohnwrightadamjohnwright
authored
Merge pull request #11 from reactome/feature/verifier
Feature/verifier
2 parents c187b61 + e42c943 commit bedfd51

File tree

7 files changed

+341
-38
lines changed

7 files changed

+341
-38
lines changed

Diff for: ‎Dockerfile

+2-2
Original file line numberDiff line numberDiff line change
@@ -24,15 +24,15 @@ CMD mvn -B -q checkstyle:check | \
2424
# ===== stage 2 =====
2525
FROM setup-env AS build-jar
2626

27-
RUN mvn clean compile assembly:single
27+
RUN mvn clean package
2828

2929

3030
# ===== stage 3 =====
3131
FROM eclipse-temurin:11-jre-focal
3232

3333
ARG REPO_DIR
3434

35-
ARG JAR_FILE=target/update-dois-*-jar-with-dependencies.jar
35+
ARG JAR_FILE=target/update-dois-jar-with-dependencies.jar
3636

3737
WORKDIR ${REPO_DIR}
3838

Diff for: ‎Jenkinsfile

+29-7
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ pipeline {
1717
}
1818
}
1919
}
20+
2021
// This stage backs up the gk_central (on curator server) and release_current databases before they are modified.
2122
stage('Setup: Back up DBs'){
2223
steps{
@@ -30,26 +31,29 @@ pipeline {
3031
}
3132
}
3233
}
34+
3335
// This stage builds the jar file using maven.
3436
stage('Setup: Build jar file'){
3537
steps{
3638
script{
37-
utils.buildJarFile()
39+
sh "mvn clean package -DskipTests"
3840
}
3941
}
4042
}
43+
4144
// This stage executes UpdateDOIs without specifying a 'report' file, which should contain a list of updateable DOIS, as one of the arguments.
4245
// This results in test mode behaviour, which includes generating the report file that contains updateable DOIS for release.
4346
stage('Main: UpdateDOIs Test Run'){
4447
steps{
4548
script{
4649
withCredentials([file(credentialsId: 'Config', variable: 'ConfigFile')]) {
4750
sh "touch src/main/resources/UpdateDOIs.report"
48-
sh "java -jar target/update-dois.jar $ConfigFile"
51+
sh "java -jar target/update-dois-jar-with-dependencies.jar $ConfigFile"
4952
}
5053
}
5154
}
5255
}
56+
5357
// This stage takes the generated report file and sends it to the curator overseeing release.
5458
// Before moving onto the next stage of UpdateDOIs, their confirmation that the contents of the report file are correct is needed.
5559
stage('Main: Send email of updateable DOIs to curator'){
@@ -65,6 +69,7 @@ pipeline {
6569
}
6670
}
6771
}
72+
6873
// UpdateDOIs should pause at this stage until the curator confirms the report file is correct. Once they do, respond with 'yes' to the user input form that Jenkins brings up.
6974
stage('User Input Required: Confirm DOIs'){
7075
steps{
@@ -79,18 +84,33 @@ pipeline {
7984
}
8085
}
8186
}
87+
8288
// Now that you have curator approval regarding the report file, this step executes the same jar file again -- this time providing the report file as an argument.
8389
// With the report file as the second argument, UpdateDOIs executes database modifications.
8490
stage('Main: UpdateDOIs'){
8591
steps{
8692
script{
8793
withCredentials([file(credentialsId: 'Config', variable: 'ConfigFile')]) {
8894
def releaseVersion = utils.getReleaseVersion()
89-
sh "java -jar target/update-dois-*-jar-with-dependencies.jar $ConfigFile doisToBeUpdated-v${releaseVersion}.txt"
95+
sh "java -jar target/update-dois-jar-with-dependencies.jar $ConfigFile doisToBeUpdated-v${releaseVersion}.txt"
9096
}
9197
}
9298
}
9399
}
100+
101+
stage('Post: Verify UpdateDOIs') {
102+
steps {
103+
script {
104+
withCredentials([usernamePassword(credentialsId: 'mySQLUsernamePassword', passwordVariable: 'releasePass', usernameVariable: 'releaseUser')]){
105+
withCredentials([usernamePassword(credentialsId: 'mySQLCuratorUsernamePassword', passwordVariable: 'curatorPass', usernameVariable: 'curatorUser')]){
106+
def releaseVersion = utils.getReleaseVersion()
107+
sh "java -jar target/update-dois-verifier-jar-with-dependencies.jar --r $releaseVersion --cu $curatorUser --cp $curatorPass --ru $releaseUser --rp $releasePass --ch curator.reactome.org"
108+
}
109+
}
110+
}
111+
}
112+
}
113+
94114
// This stage backs up the gk_central and release_current databases after they are modified.
95115
stage('Post: Backup DBs'){
96116
steps{
@@ -104,6 +124,7 @@ pipeline {
104124
}
105125
}
106126
}
127+
107128
// All databases, logs, and data files generated by this step are compressed before moving them to the Reactome S3 bucket. All files are then deleted.
108129
stage('Post: Archive Outputs'){
109130
steps{
@@ -117,16 +138,17 @@ pipeline {
117138
}
118139
}
119140
}
141+
120142
// This sends an email notifying the mailing list that both the UpdateStableIdentifiers and UpdateDOIs steps have completed. This indicates that gk_central can be reopened.
121143
stage('Post: Send completion email') {
122144
steps{
123-
script{
124-
def releaseVersion = utils.getReleaseVersion()
145+
script{
146+
def releaseVersion = utils.getReleaseVersion()
125147
def emailSubject = "UpdateStableIdentifier and UpdateDOIs complete for v${releaseVersion}"
126148
def emailBody = "Hello,\n\nThis is an automated message from Jenkins regarding an update for v${releaseVersion}: Both UpdateStableIdentifiers and UpdateDOIs steps have completed. ${env.GK_CENTRAL_DB} can likely be reopened, but Curation should get \'Human\' confirmation before doing so. \n\nThanks!"
127149
utils.sendEmail("${emailSubject}", "${emailBody}")
128-
}
129-
}
150+
}
151+
}
130152
}
131153
}
132154
}

Diff for: ‎checkstyle.xml

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
"https://checkstyle.org/dtds/configuration_1_3.dtd">
44
<module name="Checker">
55
<module name="LineLength">
6-
<property name="max" value="150"/>
6+
<property name="max" value="175"/>
77
</module>
88
<!-- Add more modules as needed -->
99
</module>

Diff for: ‎pom.xml

+7-2
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,11 @@
4949
<artifactId>log4j-core</artifactId>
5050
<version>2.17.1</version>
5151
</dependency>
52+
<dependency>
53+
<groupId>org.jcommander</groupId>
54+
<artifactId>jcommander</artifactId>
55+
<version>1.83</version>
56+
</dependency>
5257
<dependency>
5358
<groupId>org.reactome.release</groupId>
5459
<artifactId>release-common-lib</artifactId>
@@ -118,7 +123,7 @@
118123
<descriptorRefs>
119124
<descriptorRef>jar-with-dependencies</descriptorRef>
120125
</descriptorRefs>
121-
<finalName>update-dois.jar</finalName>
126+
<finalName>update-dois</finalName>
122127
</configuration>
123128
<phase>package</phase>
124129
<goals>
@@ -138,7 +143,7 @@
138143
<Multi-Release>true</Multi-Release>
139144
</manifestEntries>
140145
</archive>
141-
<finalName>update-dois-verifier.jar</finalName>
146+
<finalName>update-dois-verifier</finalName>
142147
</configuration>
143148
<phase>package</phase>
144149
<goals>

Diff for: ‎src/main/java/org/reactome/release/updateDOIs/ReportTests.java

+29-12
Original file line numberDiff line numberDiff line change
@@ -19,21 +19,32 @@ public static boolean verifyDOIMatches( GKInstance trDOI, GKInstance gkDOI, Stri
1919
if (trDOI.getDBID().equals(gkDOI.getDBID()) && trDOI.getDisplayName().equals(gkDOI.getDisplayName())) {
2020
return true;
2121
} else if (trDOI.getDBID().equals(gkDOI.getDBID()) && !trDOI.getDisplayName().equals(gkDOI.getDisplayName())) {
22-
warningsLog.warn("[" + newDOI + "] Display names do not match: [Test Reactome]: " + trDOI.getDisplayName() + " ~ [GK Central]: " + gkDOI.getDisplayName());
22+
warningsLog.warn("[" + newDOI + "] Display names do not match: [Test Reactome]: " + trDOI.getDisplayName() +
23+
" ~ [GK Central]: " + gkDOI.getDisplayName());
2324
return false;
2425
} else if (!trDOI.getDBID().equals(gkDOI.getDBID()) && trDOI.getDisplayName().equals(gkDOI.getDisplayName())) {
25-
warningsLog.warn("[" + newDOI + "] DB IDs do not match: [Test Reactome]: " + trDOI.getDBID() + " ~ [GK Central]: " + gkDOI.getDBID());
26+
warningsLog.warn("[" + newDOI + "] DB IDs do not match: [Test Reactome]: " + trDOI.getDBID() +
27+
" ~ [GK Central]: " + gkDOI.getDBID());
2628
return false;
2729
} else {
28-
warningsLog.warn("DB ID and display name do not match: [Test Reactome]: " + trDOI + " ~ [GK Central]: " + gkDOI);
30+
warningsLog.warn("DB ID and display name do not match: [Test Reactome]: " + trDOI + " ~ [GK Central]: " +
31+
gkDOI);
2932
return false;
3033
}
3134

3235
}
3336

34-
public static void expectedUpdatesTests(Map<String,Map<String,String>> expectedUpdatedDOIs, List<String> updated, List<String> notUpdated, int expectedNumberOfUpdatedDOIs, String REACTOME_DOI_PREFIX) {
35-
// Checking if provided list matched updated instances. Any that don't, it attempts to determine why they might not of been updated.
36-
// This entails comparing the DB ID, display name and the stable ID version of the provided list (UpdateDOIs.report) with the actual updated instances
37+
public static void expectedUpdatesTests(
38+
Map<String,Map<String,String>> expectedUpdatedDOIs,
39+
List<String> updated,
40+
List<String> notUpdated,
41+
int expectedNumberOfUpdatedDOIs,
42+
String REACTOME_DOI_PREFIX
43+
) {
44+
// Checking if provided list matched updated instances. Any that don't, it attempts to determine why they
45+
// might not of been updated.
46+
// This entails comparing the DB ID, display name and the stable ID version of the provided list
47+
// (UpdateDOIs.report) with the actual updated instances
3748
if (notUpdated.size() > 0)
3849
{
3950
warningsLog.warn("Some DOIs from UpdateDOIs.report were not updated");
@@ -46,25 +57,30 @@ public static void expectedUpdatesTests(Map<String,Map<String,String>> expectedU
4657
String missedStableId = missedClean.split("\\.")[0];
4758
String missedStableIdVersion = missedClean.split("\\.")[1];
4859
int resolved = 0;
49-
// Iterate through each of the DOI's provided in UpdateDOIs.report, trying to find a match for the DOI that wasn't updated.
50-
// If it finds a match of either DB ID, stable ID version, or the display name, it will try to determine which of those 3 fields don't match.
60+
// Iterate through each of the DOI's provided in UpdateDOIs.report, trying to find a match for the DOI
61+
// that wasn't updated.
62+
// If it finds a match of either DB ID, stable ID version, or the display name, it will try to
63+
// determine which of those 3 fields don't match.
5164
// Once it finds the field that doesn't match, it logs it and then ends the current iteration.
5265
for (String key : expectedUpdatedDOIs.keySet())
5366
{
5467
if (expectedUpdatedDOIs.get(key).get("stableId").equals(missedStableId))
5568
{
5669
if (!expectedUpdatedDOIs.get(key).get("stableIdVersion").equals(missedStableIdVersion))
5770
{
58-
warningsLog.warn("[" + key + "] StableID 'version' in DB different from expected: [DB] " + missedDoi + "* ~ [Expected] " + key + "*");
71+
warningsLog.warn("[" + key + "] StableID 'version' in DB different from expected: [DB] " +
72+
missedDoi + "* ~ [Expected] " + key + "*");
5973
resolved++;
6074
continue;
6175
} else if (!expectedUpdatedDOIs.get(key).get("displayName").equals(missedName)) {
62-
warningsLog.warn("[" + key + "] 'Display name' in DB different from expected: [DB] " + missedName + " ~ [Expected] " + expectedUpdatedDOIs.get(key).get("displayName"));
76+
warningsLog.warn("[" + key + "] 'Display name' in DB different from expected: [DB] " +
77+
missedName + " ~ [Expected] " + expectedUpdatedDOIs.get(key).get("displayName"));
6378
resolved++;
6479
continue;
6580
}
6681
} else if (expectedUpdatedDOIs.get(key).get("displayName").equals(missedName)) {
67-
warningsLog.warn("[" + key + "] 'DB ID' from DB different from expected, but found matching display name: ~ [DB] " + missed + " [Expected] " + key + ":" + missedName);
82+
warningsLog.warn("[" + key + "] 'DB ID' from DB different from expected, but found matching " +
83+
"display name: ~ [DB] " + missed + " [Expected] " + key + ":" + missedName);
6884
resolved++;
6985
continue;
7086
}
@@ -78,7 +94,8 @@ public static void expectedUpdatesTests(Map<String,Map<String,String>> expectedU
7894
{
7995
for (String unresolvedDOI : unresolvedDOIs)
8096
{
81-
warningsLog.warn("[" + unresolvedDOI + "]" + "DOI does not match any DOIs expected to be updated -- Could not match display name or DB ID");
97+
warningsLog.warn("[" + unresolvedDOI + "]" + "DOI does not match any DOIs expected to be updated " +
98+
"-- Could not match display name or DB ID");
8299
}
83100
}
84101
} else if (expectedUpdatedDOIs.size() != 0 && expectedNumberOfUpdatedDOIs > expectedUpdatedDOIs.size()) {

Diff for: ‎src/main/java/org/reactome/release/updateDOIs/UpdateDOIs.java

+28-14
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,8 @@ public static void setAdaptors(MySQLAdaptor adaptorTR, MySQLAdaptor adaptorGK) {
3535
}
3636

3737
@SuppressWarnings("unchecked")
38-
public static void findAndUpdateDOIs(long personId, Path pathToReport, int releaseNumber, boolean testMode) throws IOException {
38+
public static void findAndUpdateDOIs(long personId, Path pathToReport, int releaseNumber, boolean testMode)
39+
throws IOException {
3940

4041
Path doisListFilepath = Paths.get("doisToBeUpdated-v" + releaseNumber + ".txt");
4142
if (testMode) {
@@ -61,14 +62,17 @@ public static void findAndUpdateDOIs(long personId, Path pathToReport, int relea
6162
expectedUpdatedDOIs = UpdateDOIs.getExpectedUpdatedDOIs(pathToReport.toString());
6263
}
6364
if (expectedUpdatedDOIs.size() == 0) {
64-
logger.warn("No DOIs listed in UpdateDOIs.report. Please add expected DOI and displayName to UpdateDOIs.report.");
65+
logger.warn("No DOIs listed in UpdateDOIs.report. " +
66+
"Please add expected DOI and displayName to UpdateDOIs.report.");
6567
}
6668
List<String> updated = new ArrayList<>();
6769
List<String> notUpdated = new ArrayList<>();
6870
try
6971
{
70-
// Get all instances in Test Reactome in the Pathway table that don't have a 'doi' attribute starting with 10.3180, the Reactome DOI standard
71-
doisTR = dbaTestReactome.fetchInstanceByAttribute(ReactomeJavaConstants.Pathway, "doi", "NOT REGEXP", "^" + REACTOME_DOI_PREFIX);
72+
// Get all instances in Test Reactome in the Pathway table that don't have a 'doi' attribute starting
73+
// with 10.3180, the Reactome DOI standard
74+
doisTR = dbaTestReactome.fetchInstanceByAttribute(
75+
ReactomeJavaConstants.Pathway, "doi", "NOT REGEXP", "^" + REACTOME_DOI_PREFIX);
7276
logger.info("Found " + doisTR.size() + " Pathway instances that need a DOI");
7377
// GKCentral should require transactional support
7478
if (dbaGkCentral.supportsTransactions())
@@ -78,14 +82,17 @@ public static void findAndUpdateDOIs(long personId, Path pathToReport, int relea
7882
outerloop:
7983
for (GKInstance trDOI : doisTR)
8084
{
81-
// The dois are constructed from the instances 'stableIdentifier', which should be in the db already
82-
String stableIdFromDb = ((GKInstance) trDOI.getAttributeValue(ReactomeJavaConstants.stableIdentifier)).getDisplayName();
85+
// The dois are constructed from the instances 'stableIdentifier',
86+
// which should be in the db already
87+
String stableIdFromDb = ((GKInstance)
88+
trDOI.getAttributeValue(ReactomeJavaConstants.stableIdentifier)).getDisplayName();
8389
String nameFromDb = trDOI.getAttributeValue(ReactomeJavaConstants.name).toString();
8490
String updatedDoi = REACTOME_DOI_PREFIX + "/" + stableIdFromDb;
8591
String dbId = trDOI.getAttributeValue(ReactomeJavaConstants.DB_ID).toString();
8692

8793
// Used to verify that report contents are as expected, based on provided list from curators
88-
if (expectedUpdatedDOIs.get(updatedDoi) != null && expectedUpdatedDOIs.get(updatedDoi).get("displayName").equals(nameFromDb))
94+
if (expectedUpdatedDOIs.get(updatedDoi) != null &&
95+
expectedUpdatedDOIs.get(updatedDoi).get("displayName").equals(nameFromDb))
8996
{
9097
updated.add(updatedDoi);
9198
} else {
@@ -95,14 +102,16 @@ public static void findAndUpdateDOIs(long personId, Path pathToReport, int relea
95102
continue;
96103
}
97104
}
98-
// This updates the 'modified' field for Pathways instances, keeping track of when changes happened for each instance
105+
// This updates the 'modified' field for Pathways instances, keeping track of when changes
106+
// happened for each instance
99107
trDOI.getAttributeValuesList(ReactomeJavaConstants.modified);
100108
trDOI.addAttributeValue(ReactomeJavaConstants.modified, instanceEditTR);
101109
trDOI.setAttributeValue("doi", updatedDoi);
102110

103-
// Grabs instance from GKCentral based on DB_ID taken from Test Reactome and updates it's DOI
111+
// Grabs instance from GKCentral based on DB_ID taken from Test Reactome and updates its DOI
104112
dbaGkCentral.startTransaction();
105-
doisGK = dbaGkCentral.fetchInstanceByAttribute(ReactomeJavaConstants.Pathway, ReactomeJavaConstants.DB_ID, "=", dbId);
113+
doisGK = dbaGkCentral.fetchInstanceByAttribute(
114+
ReactomeJavaConstants.Pathway, ReactomeJavaConstants.DB_ID, "=", dbId);
106115
if (!doisGK.isEmpty())
107116
{
108117
for (GKInstance gkDOI : doisGK)
@@ -136,7 +145,8 @@ public static void findAndUpdateDOIs(long personId, Path pathToReport, int relea
136145
dbaTestReactome.updateInstanceAttribute(trDOI, "doi");
137146
}
138147
}
139-
ReportTests.expectedUpdatesTests(expectedUpdatedDOIs, updated, notUpdated, doisTR.size(), REACTOME_DOI_PREFIX);
148+
ReportTests.expectedUpdatesTests(
149+
expectedUpdatedDOIs, updated, notUpdated, doisTR.size(), REACTOME_DOI_PREFIX);
140150
} else {
141151
logger.info("No DOIs to update");
142152
}
@@ -177,7 +187,10 @@ public static Map<String, Map<String,String>> getExpectedUpdatedDOIs(String path
177187
String reactomeDoi = commaSplit[0];
178188
String displayName = commaSplit[1];
179189
int lastPeriodIndex = commaSplit[0].lastIndexOf(".");
180-
String[] versionSplit = {reactomeDoi.substring(0, lastPeriodIndex), reactomeDoi.substring(lastPeriodIndex+1)};
190+
String[] versionSplit = {
191+
reactomeDoi.substring(0, lastPeriodIndex),
192+
reactomeDoi.substring(lastPeriodIndex + 1)
193+
};
181194
String stableId = versionSplit[0].replace(REACTOME_DOI_PREFIX + "/", "");
182195
String stableIdVersion = versionSplit[1];
183196
doiAttributes.put("displayName", displayName);
@@ -233,8 +246,9 @@ public static GKInstance createInstanceEdit(MySQLAdaptor dbAdaptor, long personI
233246
* @return an InstanceEdit object.
234247
* @throws Exception
235248
*/
236-
public static GKInstance createDefaultIE(MySQLAdaptor dba, Long defaultPersonId, boolean needStore, String note)
237-
throws Exception {
249+
public static GKInstance createDefaultIE(
250+
MySQLAdaptor dba, Long defaultPersonId, boolean needStore, String note) throws Exception {
251+
238252
GKInstance defaultPerson = dba.fetchInstance(defaultPersonId);
239253
if (defaultPerson != null) {
240254
GKInstance newIE = UpdateDOIs.createDefaultInstanceEdit(defaultPerson);

0 commit comments

Comments
 (0)