Skip to content

Commit f311a2a

Browse files
author
George Nash
committed
assertion analysis script for assertions missing orcid ids
1 parent 479965a commit f311a2a

File tree

7 files changed

+122
-49
lines changed

7 files changed

+122
-49
lines changed

‎assertion-service/src/main/java/org/orcid/memberportal/service/assertion/config/dbmigrations/AssertionServiceDbChanges.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -93,10 +93,10 @@ public void removeOrcidIdsFromAssertionsWithNoTokenAssociated(MongoTemplate mong
9393
Query assertionsQuery = new Query();
9494
assertionsQuery.addCriteria(Criteria.where("orcid_id").exists(true));
9595
List<Assertion> assertionsWithOrcidId = mongoTemplate.find(assertionsQuery, Assertion.class, "assertion");
96-
96+
9797
LOG.info("Found {} assertions with orcid id populated", assertionsWithOrcidId.size());
9898
int numRemoved = 0;
99-
99+
100100
for (Assertion a : assertionsWithOrcidId) {
101101
Query orcidRecordQuery = new Query();
102102
orcidRecordQuery.addCriteria(Criteria.where("email").is(a.getEmail()));
@@ -109,7 +109,7 @@ public void removeOrcidIdsFromAssertionsWithNoTokenAssociated(MongoTemplate mong
109109
numRemoved++;
110110
}
111111
}
112-
112+
113113
LOG.info("{} orcid ids removed from affiliations", numRemoved);
114114
}
115115

‎assertion-service/src/main/java/org/orcid/memberportal/service/assertion/repository/AssertionRepository.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ public interface AssertionRepository extends MongoRepository<Assertion, String>,
2121
List<Assertion> findAllByOwnerId(String ownerId, Sort sort);
2222

2323
Page<Assertion> findBySalesforceId(String salesforceId, Pageable pageable);
24-
24+
2525
List<Assertion> findByStatus(String status, Pageable pageable);
2626

2727
Page<Assertion> findBySalesforceIdAndAffiliationSectionContainingIgnoreCaseOrSalesforceIdAndDepartmentNameContainingIgnoreCaseOrSalesforceIdAndOrgNameContainingIgnoreCaseOrSalesforceIdAndDisambiguatedOrgIdContainingIgnoreCaseOrSalesforceIdAndEmailContainingIgnoreCaseOrSalesforceIdAndOrcidIdContainingIgnoreCaseOrSalesforceIdAndRoleTitleContainingIgnoreCase(
@@ -51,4 +51,5 @@ Page<Assertion> findBySalesforceIdAndAffiliationSectionContainingIgnoreCaseOrSal
5151
@Query("{ addedToORCID: { $exists: false }, $or: [ { notificationSent: { $exists: true } }, { invitationSent: { $exists: true } } ] }")
5252
Page<Assertion> findNotificationResendCandidates(Pageable pageable);
5353

54+
List<Assertion> findByStatusAndOrcidIdIsNull(String status);
5455
}

‎assertion-service/src/main/java/org/orcid/memberportal/service/assertion/scripts/ExamineAssertionsMissingOrcid.java‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
package org.orcid.memberportal.service.assertion.scripts;
2+
3+
import org.orcid.memberportal.service.assertion.AssertionServiceApp;
4+
import org.orcid.memberportal.service.assertion.config.*;
5+
import org.orcid.memberportal.service.assertion.domain.Assertion;
6+
import org.orcid.memberportal.service.assertion.domain.OrcidRecord;
7+
import org.orcid.memberportal.service.assertion.domain.OrcidToken;
8+
import org.orcid.memberportal.service.assertion.domain.enumeration.AssertionStatus;
9+
import org.orcid.memberportal.service.assertion.repository.AssertionRepository;
10+
import org.orcid.memberportal.service.assertion.repository.OrcidRecordRepository;
11+
import org.orcid.memberportal.service.assertion.services.AssertionService;
12+
import org.slf4j.Logger;
13+
import org.slf4j.LoggerFactory;
14+
import org.springframework.beans.factory.annotation.Autowired;
15+
import org.springframework.boot.SpringApplication;
16+
import org.springframework.boot.WebApplicationType;
17+
import org.springframework.boot.autoconfigure.SpringBootApplication;
18+
import org.springframework.context.ApplicationContext;
19+
import org.springframework.context.annotation.AnnotationConfigApplicationContext;
20+
import org.springframework.context.support.ClassPathXmlApplicationContext;
21+
22+
import java.util.List;
23+
import java.util.Optional;
24+
25+
public class ExamineAssertionsMissingOrcid {
26+
27+
private static final Logger LOG = LoggerFactory.getLogger(ExamineAssertionsMissingOrcid.class);
28+
29+
private static int missingTokenCount = 0;
30+
31+
private static int matchingTokenCount = 0;
32+
33+
private static int revokedTokenCount = 0;
34+
35+
private static int salesforceIdSubstrings = 0;
36+
37+
private static int missingOrcidRecordCount = 0;
38+
39+
public static void main(String[] args) {
40+
// to avoid startup errors involving user service client
41+
System.setProperty("jhipster.security.client-authorization.client-id", "not needed");
42+
43+
SpringApplication application = new SpringApplication(AssertionServiceApp.class);
44+
application.setWebApplicationType(WebApplicationType.NONE);
45+
ApplicationContext context = application.run(args);
46+
47+
AssertionRepository assertionRepository = context.getBean(AssertionRepository.class);
48+
OrcidRecordRepository orcidRecordRepository = context.getBean(OrcidRecordRepository.class);
49+
50+
List<Assertion> assertions = assertionRepository.findByStatusAndOrcidIdIsNull(AssertionStatus.IN_ORCID.name());
51+
LOG.info("Found {} IN_ORCID assertions with no orcid id", assertions.size());
52+
53+
assertions.forEach(a -> {
54+
Optional<OrcidRecord> optional = orcidRecordRepository.findOneByEmail(a.getEmail());
55+
if (optional.isEmpty()) {
56+
LOG.info("No orcid record found for assertion {} - email {}, salesforce id {}", a.getId(), a.getEmail(), a.getSalesforceId());
57+
missingOrcidRecordCount++;
58+
} else {
59+
OrcidRecord orcidRecord = optional.get();
60+
if (orcidRecord.getToken(a.getSalesforceId(), false) != null) {
61+
LOG.info("Found matching token for assertion {} - email {}, salesforce id {}", a.getId(), a.getEmail(), a.getSalesforceId());
62+
matchingTokenCount++;
63+
} else if (orcidRecord.getToken(a.getSalesforceId(), true) != null) {
64+
LOG.info("Found REVOKED token for assertion {} - email {}, salesforce id {}", a.getId(), a.getEmail(), a.getSalesforceId());
65+
revokedTokenCount++;
66+
} else {
67+
LOG.info("No token found for assertion {} - email {}, salesforce id {}", a.getId(), a.getEmail(), a.getSalesforceId());
68+
missingTokenCount++;
69+
70+
List<OrcidToken> possibleMatches = orcidRecord.getTokens();
71+
possibleMatches.forEach(t -> {
72+
if (a.getSalesforceId().contains(t.getSalesforceId())) {
73+
LOG.info("Found likely match. Token salesforce id might need updating from {} to {}", t.getSalesforceId(), a.getSalesforceId());
74+
salesforceIdSubstrings++;
75+
}
76+
});
77+
}
78+
}
79+
});
80+
81+
LOG.info("------ SUMMARY -------");
82+
LOG.info("Found {} IN_ORCID assertions with no orcid id", assertions.size());
83+
LOG.info("Found {} missing OrcidRecord", missingOrcidRecordCount);
84+
LOG.info("Found {} with OrcidRecord and matching token", matchingTokenCount);
85+
LOG.info("Found {} with revoked matching token", revokedTokenCount);
86+
LOG.info("Found {} with no matching token", missingTokenCount);
87+
LOG.info("Found {} possible broken salesforce id matches", salesforceIdSubstrings);
88+
89+
System.exit(0);
90+
}
91+
92+
93+
}

‎assertion-service/src/main/java/org/orcid/memberportal/service/assertion/services/AssertionService.java‎

Lines changed: 12 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,35 +1,13 @@
11
package org.orcid.memberportal.service.assertion.services;
22

3-
import java.io.File;
4-
import java.io.FileInputStream;
5-
import java.io.IOException;
6-
import java.io.InputStream;
7-
import java.time.Instant;
8-
import java.time.ZoneId;
9-
import java.time.format.DateTimeFormatter;
10-
import java.time.format.FormatStyle;
11-
import java.util.ArrayList;
12-
import java.util.HashMap;
13-
import java.util.List;
14-
import java.util.Locale;
15-
import java.util.Map;
16-
import java.util.Optional;
17-
18-
import javax.xml.bind.JAXBException;
19-
3+
import com.google.common.base.Objects;
204
import org.apache.commons.lang3.StringUtils;
215
import org.apache.http.client.ClientProtocolException;
226
import org.json.JSONException;
237
import org.json.JSONObject;
248
import org.orcid.memberportal.service.assertion.client.OrcidAPIClient;
259
import org.orcid.memberportal.service.assertion.csv.CsvWriter;
26-
import org.orcid.memberportal.service.assertion.domain.Assertion;
27-
import org.orcid.memberportal.service.assertion.domain.AssertionServiceUser;
28-
import org.orcid.memberportal.service.assertion.domain.CsvReport;
29-
import org.orcid.memberportal.service.assertion.domain.MemberAssertionStatusCount;
30-
import org.orcid.memberportal.service.assertion.domain.OrcidRecord;
31-
import org.orcid.memberportal.service.assertion.domain.OrcidToken;
32-
import org.orcid.memberportal.service.assertion.domain.StoredFile;
10+
import org.orcid.memberportal.service.assertion.domain.*;
3311
import org.orcid.memberportal.service.assertion.domain.enumeration.AffiliationSection;
3412
import org.orcid.memberportal.service.assertion.domain.enumeration.AssertionStatus;
3513
import org.orcid.memberportal.service.assertion.domain.normalization.AssertionNormalizer;
@@ -50,7 +28,16 @@
5028
import org.springframework.stereotype.Service;
5129
import org.springframework.web.multipart.MultipartFile;
5230

53-
import com.google.common.base.Objects;
31+
import javax.xml.bind.JAXBException;
32+
import java.io.File;
33+
import java.io.FileInputStream;
34+
import java.io.IOException;
35+
import java.io.InputStream;
36+
import java.time.Instant;
37+
import java.time.ZoneId;
38+
import java.time.format.DateTimeFormatter;
39+
import java.time.format.FormatStyle;
40+
import java.util.*;
5441

5542
@Service
5643
public class AssertionService {

‎assertion-service/src/main/resources/config/application-dev.yml‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -36,10 +36,6 @@ spring:
3636
jackson:
3737
serialization:
3838
indent-output: true
39-
data:
40-
mongodb:
41-
uri: mongodb://localhost:27017
42-
database: assertionservice
4339
mail:
4440
host: localhost
4541
port: 25
@@ -59,9 +55,6 @@ spring:
5955
discovery:
6056
enabled: true
6157

62-
server:
63-
port: 8092
64-
6558
# ===================================================================
6659
# JHipster specific properties
6760
#

‎assertion-service/src/main/resources/config/application-prod.yml‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -32,10 +32,6 @@ spring:
3232
enabled: false
3333
livereload:
3434
enabled: false
35-
data:
36-
mongodb:
37-
uri: mongodb://localhost:27017
38-
database: assertionservice
3935
mail:
4036
host: localhost
4137
port: 25
@@ -52,12 +48,6 @@ spring:
5248
locator:
5349
discovery:
5450
enabled: true
55-
server:
56-
port: 8092
57-
compression:
58-
enabled: true
59-
mime-types: text/html,text/xml,text/plain,text/css, application/javascript, application/json
60-
min-response-size: 1024
6151

6252
# ===================================================================
6353
# JHipster specific properties

‎assertion-service/src/main/resources/config/application.yml‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,10 @@ management:
8585
auto-time-requests: true
8686

8787
spring:
88+
data:
89+
mongodb:
90+
uri: mongodb://localhost:27017
91+
database: assertionservice
8892
servlet:
8993
multipart:
9094
max-file-size: 15MB
@@ -128,6 +132,11 @@ security:
128132
filter-order: 3
129133

130134
server:
135+
port: 8092
136+
compression:
137+
enabled: true
138+
mime-types: text/html,text/xml,text/plain,text/css, application/javascript, application/json
139+
min-response-size: 1024
131140
servlet:
132141
session:
133142
cookie:
@@ -177,14 +186,14 @@ jhipster:
177186
# https://www.jhipster.tech/common-application-properties/
178187
# ===================================================================
179188

180-
application:
189+
application:
181190
orcidAPIEndpoint: ${APPLICATION_ORCIDAPIENDPOINT}
182191
jwtSignatureUrl: ${APPLICATION_JWT_SIGNATURE_URL}
183192
landingPageUrl: ${APPLICATION_LANDING_PAGE_URL}
184193
mailDomain: ${APPLICATION_MAIL_DOMAIN}
185194
mailApiKey: ${APPLICATION_MAIL_API_KEY}
186195
mailApiUrl: ${APPLICATION_MAIL_API_URL}
187-
mailFromAddress: ${APPLICATION_MAIL_FROM_ADDRESS}
196+
mailFromAddress: ${APPLICATION_MAIL_FROM_ADDRESS}
188197
mailFromName: ${APPLICATION_MAIL_FROM_NAME}
189198
memberAssertionStatsDirectory: files/stats
190199
assertionsCsvUploadDirectory: files/uploads
@@ -205,5 +214,5 @@ application:
205214
grantType: ${APPLICATION_TOKEN_EXCHANGE_GRANT_TYPE}
206215
subjectTokenType: ${APPLICATION_TOKEN_EXCHANGE_SUBJECT_TOKEN_TYPE}
207216
requestedTokenType: ${APPLICATION_TOKEN_EXCHANGE_REQUESTED_TOKEN_TYPE}
208-
clientId: ${APPLICATION_TOKEN_EXCHANGE_CLIENT_ID}
217+
clientId: ${APPLICATION_TOKEN_EXCHANGE_CLIENT_ID}
209218
clientSecret: ${APPLICATION_TOKEN_EXCHANGE_CLIENT_SECRET}

0 commit comments

Comments
 (0)