Skip to content

Commit b9ce7c4

Browse files
lbowniklbownik
lbownik
and
lbownik
authored
Closes #2783 Changes in <Identifier> for OAI-PMH records
--------- Co-authored-by: lbownik <lbownik@icm.edu.pl>
1 parent ecef9fd commit b9ce7c4

6 files changed

Lines changed: 249 additions & 125 deletions

File tree

‎dataverse-webapp/src/main/java/edu/harvard/iq/dataverse/harvest/server/web/servlet/OAIServlet.java‎

Lines changed: 72 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,30 @@
11
package edu.harvard.iq.dataverse.harvest.server.web.servlet;
22

3-
import com.lyncode.xml.exceptions.XmlWriteException;
4-
import edu.harvard.iq.dataverse.DatasetDao;
5-
import edu.harvard.iq.dataverse.DataverseDao;
6-
import edu.harvard.iq.dataverse.export.ExportService;
7-
import edu.harvard.iq.dataverse.export.ExporterType;
8-
import edu.harvard.iq.dataverse.export.spi.Exporter;
9-
import edu.harvard.iq.dataverse.harvest.server.OAIRecordServiceBean;
10-
import edu.harvard.iq.dataverse.harvest.server.OAISetServiceBean;
11-
import edu.harvard.iq.dataverse.harvest.server.xoai.XdataProvider;
12-
import edu.harvard.iq.dataverse.harvest.server.xoai.XgetRecord;
13-
import edu.harvard.iq.dataverse.harvest.server.xoai.XitemRepository;
14-
import edu.harvard.iq.dataverse.harvest.server.xoai.XlistRecords;
15-
import edu.harvard.iq.dataverse.harvest.server.xoai.XsetRepository;
16-
import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
17-
import edu.harvard.iq.dataverse.util.SystemConfig;
18-
import org.apache.commons.lang.StringUtils;
3+
import static edu.harvard.iq.dataverse.settings.SettingsServiceBean.Key.OAIServerEnabled;
4+
import static edu.harvard.iq.dataverse.settings.SettingsServiceBean.Key.SystemEmail;
5+
import static javax.servlet.http.HttpServletResponse.SC_SERVICE_UNAVAILABLE;
6+
import static org.apache.commons.lang.StringUtils.isEmpty;
7+
import static org.dspace.xoai.dataprovider.model.MetadataFormat.metadataFormat;
8+
import static org.dspace.xoai.model.oaipmh.OAIPMH.NAMESPACE_URI;
9+
import static org.dspace.xoai.model.oaipmh.OAIPMH.SCHEMA_LOCATION;
10+
import static org.dspace.xoai.model.oaipmh.Verb.Type.GetRecord;
11+
import static org.dspace.xoai.model.oaipmh.Verb.Type.ListRecords;
12+
import static org.dspace.xoai.xml.XmlWriter.defaultContext;
13+
14+
import java.io.ByteArrayOutputStream;
15+
import java.io.IOException;
16+
import java.io.OutputStream;
17+
import java.util.Date;
18+
import java.util.logging.Logger;
19+
20+
import javax.inject.Inject;
21+
import javax.servlet.ServletConfig;
22+
import javax.servlet.ServletException;
23+
import javax.servlet.http.HttpServlet;
24+
import javax.servlet.http.HttpServletRequest;
25+
import javax.servlet.http.HttpServletResponse;
26+
import javax.xml.stream.XMLStreamException;
27+
1928
import org.dspace.xoai.dataprovider.builder.OAIRequestParametersBuilder;
2029
import org.dspace.xoai.dataprovider.exceptions.OAIException;
2130
import org.dspace.xoai.dataprovider.model.Context;
@@ -32,23 +41,21 @@
3241
import org.dspace.xoai.xml.XSISchema;
3342
import org.dspace.xoai.xml.XmlWriter;
3443

35-
import javax.inject.Inject;
36-
import javax.servlet.ServletConfig;
37-
import javax.servlet.ServletException;
38-
import javax.servlet.http.HttpServlet;
39-
import javax.servlet.http.HttpServletRequest;
40-
import javax.servlet.http.HttpServletResponse;
41-
import javax.xml.stream.XMLStreamException;
42-
import java.io.ByteArrayOutputStream;
43-
import java.io.IOException;
44-
import java.io.OutputStream;
45-
import java.util.Date;
46-
import java.util.Map;
47-
import java.util.logging.Logger;
44+
import com.lyncode.xml.exceptions.XmlWriteException;
4845

49-
import static org.dspace.xoai.model.oaipmh.OAIPMH.NAMESPACE_URI;
50-
import static org.dspace.xoai.model.oaipmh.OAIPMH.SCHEMA_LOCATION;
51-
import static org.dspace.xoai.xml.XmlWriter.defaultContext;
46+
import edu.harvard.iq.dataverse.DatasetDao;
47+
import edu.harvard.iq.dataverse.DataverseDao;
48+
import edu.harvard.iq.dataverse.export.ExportService;
49+
import edu.harvard.iq.dataverse.export.spi.Exporter;
50+
import edu.harvard.iq.dataverse.harvest.server.OAIRecordServiceBean;
51+
import edu.harvard.iq.dataverse.harvest.server.OAISetServiceBean;
52+
import edu.harvard.iq.dataverse.harvest.server.xoai.XdataProvider;
53+
import edu.harvard.iq.dataverse.harvest.server.xoai.XgetRecord;
54+
import edu.harvard.iq.dataverse.harvest.server.xoai.XitemRepository;
55+
import edu.harvard.iq.dataverse.harvest.server.xoai.XlistRecords;
56+
import edu.harvard.iq.dataverse.harvest.server.xoai.XsetRepository;
57+
import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
58+
import edu.harvard.iq.dataverse.util.SystemConfig;
5259

5360
/**
5461
* @author Leonid Andreev
@@ -129,7 +136,7 @@ public void init(ServletConfig config) throws ServletException {
129136
}
130137

131138
setRepository = new XsetRepository(setService);
132-
itemRepository = new XitemRepository(recordService, datasetDao);
139+
itemRepository = new XitemRepository(recordService, datasetDao, systemConfig);
133140

134141
repositoryConfiguration = createRepositoryConfiguration();
135142

@@ -175,7 +182,7 @@ protected OAIRequestParametersBuilder newXoaiRequest() {
175182
}
176183

177184
public boolean isHarvestingServerEnabled() {
178-
return settingsService.isTrueForKey(SettingsServiceBean.Key.OAIServerEnabled);
185+
return settingsService.isTrueForKey(OAIServerEnabled);
179186
}
180187

181188
// -------------------- PRIVATE --------------------
@@ -191,8 +198,9 @@ private void addSupportedMetadataFormats(Context context) {
191198
if (!exporter.isXMLFormat() || !exporter.isHarvestable()) {
192199
continue;
193200
}
194-
MetadataFormat metadataFormat = MetadataFormat.metadataFormat(exporter.getProviderName());
195-
if (exporter.getXMLNameSpace().isEmpty() && exporter.getXMLSchemaLocation().isEmpty()) {
201+
MetadataFormat metadataFormat = metadataFormat(exporter.getProviderName());
202+
if (exporter.getXMLNameSpace().isEmpty()
203+
&& exporter.getXMLSchemaLocation().isEmpty()) {
196204
continue;
197205
}
198206
metadataFormat.withNamespace(exporter.getXMLNameSpace());
@@ -219,32 +227,49 @@ private RepositoryConfiguration createRepositoryConfiguration() {
219227
// need to be configurable!
220228

221229
String dataverseName = dataverseDao.findRootDataverse().getName();
222-
String repositoryName = StringUtils.isEmpty(dataverseName) || "Root".equals(dataverseName) ? "Test Dataverse OAI Archive" : dataverseName + " Dataverse OAI Archive";
230+
String repositoryName = isEmpty(dataverseName) || "Root".equals(dataverseName)
231+
? "Test Dataverse OAI Archive"
232+
: dataverseName + " Dataverse OAI Archive";
223233
Date earliestDate = recordService.findEarliestDate();
224234

225235
RepositoryConfiguration repositoryConfiguration = new RepositoryConfiguration()
226236
.withRepositoryName(repositoryName)
227237
.withBaseUrl(systemConfig.getDataverseSiteUrl() + "/oai")
228238
.withCompression("gzip") // ?
229239
.withCompression("deflate") // ?
230-
.withAdminEmail(settingsService.getValueForKey(SettingsServiceBean.Key.SystemEmail))
240+
.withAdminEmail(settingsService.getValueForKey(SystemEmail))
231241
.withDeleteMethod(DeletedRecord.TRANSIENT)
232242
.withGranularity(Granularity.Second)
233243
.withMaxListIdentifiers(100)
234244
.withMaxListRecords(100)
235245
.withMaxListSets(100)
236-
.withEarliestDate(earliestDate != null ? earliestDate : new Date());
246+
.withEarliestDate(earliestDate != null ? earliestDate : new Date())
247+
.withDescription(createDescription());
237248

238249
return repositoryConfiguration;
239250
}
251+
252+
private String createDescription() {
253+
return "<description>"
254+
+ "<oai-identifier xmlns=\"http://www.openarchives.org/OAI/2.0/oai-identifier\">"
255+
+ "<scheme>oai</scheme>"
256+
+ "<repositoryIdentifier>"
257+
+ this.systemConfig.getDataverseServer()
258+
+ "</repositoryIdentifier>"
259+
+ "<delimiter>:</delimiter>"
260+
+ "<sampleIdentifier>oai:"
261+
+ this.systemConfig.getDataverseServer()
262+
+":doi%3A10.18150%2FAB1CD2</sampleIdentifier>"
263+
+ "</oai-identifier>"
264+
+ "</description>";
265+
}
240266

241267
private void processRequest(HttpServletRequest request, HttpServletResponse response)
242268
throws ServletException, IOException {
243269

244270
try {
245271
if (!isHarvestingServerEnabled()) {
246-
response.sendError(
247-
HttpServletResponse.SC_SERVICE_UNAVAILABLE,
272+
response.sendError(SC_SERVICE_UNAVAILABLE,
248273
"Sorry. OAI Service is disabled on this Dataverse node.");
249274
return;
250275
}
@@ -300,14 +325,15 @@ private void writeListRecords(HttpServletResponse response, OAIPMH handle)
300325
throw new IOException("An error or a valid response must be set");
301326
}
302327

303-
if (!verb.getType().equals(Verb.Type.ListRecords)) {
328+
if (!verb.getType().equals(ListRecords)) {
304329
throw new IOException("writeListRecords() called on a non-ListRecords verb");
305330
}
306331

307332
outputStream.write(("<" + verb.getType().displayName() + ">").getBytes());
308333
outputStream.flush();
309334

310-
((XlistRecords) verb).writeToStream(outputStream, exportService, systemConfig.getDataverseSiteUrl());
335+
((XlistRecords) verb).writeToStream(outputStream, exportService,
336+
systemConfig.getDataverseSiteUrl());
311337

312338
outputStream.write(("</" + verb.getType().displayName() + ">").getBytes());
313339
outputStream.write(("</" + OAI_PMH + ">\n").getBytes());
@@ -326,14 +352,15 @@ private void writeGetRecord(HttpServletResponse response, OAIPMH handle)
326352
throw new IOException("An error or a valid response must be set");
327353
}
328354

329-
if (!verb.getType().equals(Verb.Type.GetRecord)) {
355+
if (!verb.getType().equals(GetRecord)) {
330356
throw new IOException("writeListRecords() called on a non-GetRecord verb");
331357
}
332358

333359
outputStream.write(("<" + verb.getType().displayName() + ">").getBytes());
334360
outputStream.flush();
335361

336-
((XgetRecord) verb).writeToStream(outputStream, exportService, systemConfig.getDataverseSiteUrl());
362+
((XgetRecord) verb).writeToStream(outputStream, exportService,
363+
systemConfig.getDataverseSiteUrl());
337364

338365
outputStream.write(("</" + verb.getType().displayName() + ">").getBytes());
339366
outputStream.write(("</" + OAI_PMH + ">\n").getBytes());

‎dataverse-webapp/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XitemRepository.java‎

Lines changed: 38 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44
import edu.harvard.iq.dataverse.harvest.server.OAIRecordServiceBean;
55
import edu.harvard.iq.dataverse.persistence.dataset.Dataset;
66
import edu.harvard.iq.dataverse.persistence.harvest.OAIRecord;
7+
import edu.harvard.iq.dataverse.util.SystemConfig;
8+
79
import org.apache.commons.lang3.StringUtils;
810
import org.dspace.xoai.dataprovider.exceptions.IdDoesNotExistException;
911
import org.dspace.xoai.dataprovider.exceptions.OAIException;
@@ -13,6 +15,7 @@
1315
import org.dspace.xoai.dataprovider.model.Item;
1416
import org.dspace.xoai.dataprovider.model.ItemIdentifier;
1517
import org.dspace.xoai.dataprovider.repository.ItemRepository;
18+
import org.dspace.xoai.util.URLEncoder;
1619

1720
import java.util.ArrayList;
1821
import java.util.Date;
@@ -28,24 +31,27 @@
2831
* XOAI "items".
2932
*/
3033
public class XitemRepository implements ItemRepository {
31-
private static Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.harvest.server.xoai.XitemRepository");
34+
private static Logger logger = Logger.getLogger(XitemRepository.class.getCanonicalName());
3235

33-
private OAIRecordServiceBean recordService;
34-
private DatasetDao datasetDao;
36+
private final OAIRecordServiceBean recordService;
37+
private final DatasetDao datasetDao;
38+
private final SystemConfig systemConfig;
3539

3640
// -------------------- CONSTRUCTORS --------------------
3741

38-
public XitemRepository(OAIRecordServiceBean recordService, DatasetDao datasetDao) {
39-
super();
42+
public XitemRepository(OAIRecordServiceBean recordService, DatasetDao datasetDao,
43+
final SystemConfig systemConfig) {
4044
this.recordService = recordService;
4145
this.datasetDao = datasetDao;
46+
this.systemConfig = systemConfig;
4247
}
4348

4449
// -------------------- LOGIC --------------------
4550

4651
@Override
4752
public Item getItem(String identifier) throws IdDoesNotExistException, OAIException {
4853
logger.fine("getItem; calling findOaiRecordsByGlobalId, identifier " + identifier);
54+
identifier = parseIdetifier(identifier);
4955
List<OAIRecord> oaiRecords = recordService.findOaiRecordsByGlobalId(identifier);
5056
if (oaiRecords.isEmpty()) {
5157
throw new IdDoesNotExistException();
@@ -63,7 +69,7 @@ public Item getItem(String identifier) throws IdDoesNotExistException, OAIExcept
6369
.map(OAIRecord::getLastUpdateTime)
6470
.orElse(oaiRecords.get(0).getLastUpdateTime());
6571

66-
Xitem xoaiItem = new Xitem(identifier, lastUpdateTimestamp, removed)
72+
Xitem xoaiItem = new Xitem(prepareId(identifier), lastUpdateTimestamp, removed)
6773
.withDataset(dataset);
6874

6975
oaiRecords.forEach(record -> xoaiItem.addSet(record.getSetName()));
@@ -118,7 +124,7 @@ public ListItemIdentifiersResult getItemIdentifiers(List<ScopedFilter> filters,
118124
}
119125
for (int i = offset; i < offset + length && i < oaiRecords.size(); i++) {
120126
OAIRecord record = oaiRecords.get(i);
121-
Xitem xItem = new Xitem(record.getGlobalId(), record.getLastUpdateTime(), record.isRemoved());
127+
Xitem xItem = new Xitem(getIdOf(record), record.getLastUpdateTime(), record.isRemoved());
122128
xoaiItems.add(xItem);
123129
}
124130

@@ -185,7 +191,8 @@ public ListItemsResults getItems(List<ScopedFilter> filters, int offset, int len
185191
OAIRecord oaiRecord = oaiRecords.get(i);
186192
Dataset dataset = datasetDao.findByGlobalId(oaiRecord.getGlobalId());
187193
if (dataset != null) {
188-
Xitem xItem = new Xitem(oaiRecord.getGlobalId(), oaiRecord.getLastUpdateTime(), oaiRecord.isRemoved())
194+
Xitem xItem = new Xitem(getIdOf(oaiRecord), oaiRecord.getLastUpdateTime(),
195+
oaiRecord.isRemoved())
189196
.withDataset(dataset);
190197
xoaiItems.add(xItem);
191198
}
@@ -198,6 +205,29 @@ public ListItemsResults getItems(List<ScopedFilter> filters, int offset, int len
198205
}
199206

200207
// -------------------- PRIATE --------------------
208+
209+
private String getIdOf(final OAIRecord record) {
210+
return prepareId(record.getGlobalId());
211+
}
212+
213+
private String prepareId(final String id) {
214+
if (this.systemConfig.useOAIStrictIdentifierScheme()) {
215+
return "oai:" + this.systemConfig.getDataverseServer() + ':'
216+
+ URLEncoder.encode(id);
217+
} else {
218+
return id;
219+
}
220+
}
221+
222+
private String parseIdetifier(String id) {
223+
if (id.startsWith("oai:")) {
224+
id = id.substring(4);
225+
final int index = id.indexOf(':');
226+
return id.substring(index + 1);
227+
} else {
228+
return id;
229+
}
230+
}
201231

202232
private void addExtraSets(List<? extends ItemIdentifier> xoaiItemsList) {
203233
Map<String, Xitem> xoaiItemsMap = new HashMap<>();

‎dataverse-webapp/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -766,8 +766,17 @@ public enum Key {
766766
* Default value: empty.
767767
*/
768768
CustomSearchLocationsSolrFields,
769-
770-
MaxResultsCountSavedToFile;
769+
/**
770+
* Specifies maximum number of search results axported to CSV file
771+
*/
772+
MaxResultsCountSavedToFile,
773+
/**
774+
* If true, OAI-PMH exporters will use identifier scheme specified in
775+
* https://www.openarchives.org/OAI/openarchivesprotocol.html#UniqueIdentifier
776+
* https://www.openarchives.org/OAI/2.0/guidelines-oai-identifier.htm.
777+
* eg (oai:www.wbc.poznan.pl:568432).
778+
*/
779+
UseOAIStrictIdentifierScheme;
771780

772781

773782

‎dataverse-webapp/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,9 @@
1515
import javax.ejb.Stateless;
1616
import javax.inject.Inject;
1717
import javax.inject.Named;
18+
19+
import static edu.harvard.iq.dataverse.settings.SettingsServiceBean.Key.UseOAIStrictIdentifierScheme;
20+
1821
import java.net.MalformedURLException;
1922
import java.net.URL;
2023
import java.util.Arrays;
@@ -119,6 +122,10 @@ public String getVersion() {
119122

120123
return appVersion;
121124
}
125+
126+
public boolean useOAIStrictIdentifierScheme() {
127+
return this.settingsService.isTrueForKey(UseOAIStrictIdentifierScheme);
128+
}
122129

123130
public int getMinutesUntilPasswordResetTokenExpires() {
124131
return settingsService.getValueForKeyAsInt(SettingsServiceBean.Key.MinutesUntilPasswordResetTokenExpires);

0 commit comments

Comments
 (0)