2783 changes in identifier for oai pmh records

dataverse-webapp/src/main/java/edu/harvard/iq/dataverse/harvest/server/web/servlet/OAIServlet.java

@@ -1,21 +1,30 @@
 package edu.harvard.iq.dataverse.harvest.server.web.servlet;
 
-import com.lyncode.xml.exceptions.XmlWriteException;
-import edu.harvard.iq.dataverse.DatasetDao;
-import edu.harvard.iq.dataverse.DataverseDao;
-import edu.harvard.iq.dataverse.export.ExportService;
-import edu.harvard.iq.dataverse.export.ExporterType;
-import edu.harvard.iq.dataverse.export.spi.Exporter;
-import edu.harvard.iq.dataverse.harvest.server.OAIRecordServiceBean;
-import edu.harvard.iq.dataverse.harvest.server.OAISetServiceBean;
-import edu.harvard.iq.dataverse.harvest.server.xoai.XdataProvider;
-import edu.harvard.iq.dataverse.harvest.server.xoai.XgetRecord;
-import edu.harvard.iq.dataverse.harvest.server.xoai.XitemRepository;
-import edu.harvard.iq.dataverse.harvest.server.xoai.XlistRecords;
-import edu.harvard.iq.dataverse.harvest.server.xoai.XsetRepository;
-import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
-import edu.harvard.iq.dataverse.util.SystemConfig;
-import org.apache.commons.lang.StringUtils;
+import static edu.harvard.iq.dataverse.settings.SettingsServiceBean.Key.OAIServerEnabled;
+import static edu.harvard.iq.dataverse.settings.SettingsServiceBean.Key.SystemEmail;
+import static javax.servlet.http.HttpServletResponse.SC_SERVICE_UNAVAILABLE;
+import static org.apache.commons.lang.StringUtils.isEmpty;
+import static org.dspace.xoai.dataprovider.model.MetadataFormat.metadataFormat;
+import static org.dspace.xoai.model.oaipmh.OAIPMH.NAMESPACE_URI;
+import static org.dspace.xoai.model.oaipmh.OAIPMH.SCHEMA_LOCATION;
+import static org.dspace.xoai.model.oaipmh.Verb.Type.GetRecord;
+import static org.dspace.xoai.model.oaipmh.Verb.Type.ListRecords;
+import static org.dspace.xoai.xml.XmlWriter.defaultContext;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Date;
+import java.util.logging.Logger;
+
+import javax.inject.Inject;
+import javax.servlet.ServletConfig;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import javax.xml.stream.XMLStreamException;
+
 import org.dspace.xoai.dataprovider.builder.OAIRequestParametersBuilder;
 import org.dspace.xoai.dataprovider.exceptions.OAIException;
 import org.dspace.xoai.dataprovider.model.Context;
@@ -32,23 +41,21 @@
 import org.dspace.xoai.xml.XSISchema;
 import org.dspace.xoai.xml.XmlWriter;
 
-import javax.inject.Inject;
-import javax.servlet.ServletConfig;
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServlet;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-import javax.xml.stream.XMLStreamException;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.util.Date;
-import java.util.Map;
-import java.util.logging.Logger;
+import com.lyncode.xml.exceptions.XmlWriteException;
 
-import static org.dspace.xoai.model.oaipmh.OAIPMH.NAMESPACE_URI;
-import static org.dspace.xoai.model.oaipmh.OAIPMH.SCHEMA_LOCATION;
-import static org.dspace.xoai.xml.XmlWriter.defaultContext;
+import edu.harvard.iq.dataverse.DatasetDao;
+import edu.harvard.iq.dataverse.DataverseDao;
+import edu.harvard.iq.dataverse.export.ExportService;
+import edu.harvard.iq.dataverse.export.spi.Exporter;
+import edu.harvard.iq.dataverse.harvest.server.OAIRecordServiceBean;
+import edu.harvard.iq.dataverse.harvest.server.OAISetServiceBean;
+import edu.harvard.iq.dataverse.harvest.server.xoai.XdataProvider;
+import edu.harvard.iq.dataverse.harvest.server.xoai.XgetRecord;
+import edu.harvard.iq.dataverse.harvest.server.xoai.XitemRepository;
+import edu.harvard.iq.dataverse.harvest.server.xoai.XlistRecords;
+import edu.harvard.iq.dataverse.harvest.server.xoai.XsetRepository;
+import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
+import edu.harvard.iq.dataverse.util.SystemConfig;
 
 /**
  * @author Leonid Andreev
@@ -129,7 +136,7 @@ public void init(ServletConfig config) throws ServletException {
         }
 
         setRepository = new XsetRepository(setService);
-        itemRepository = new XitemRepository(recordService, datasetDao);
+        itemRepository = new XitemRepository(recordService, datasetDao, systemConfig);
 
         repositoryConfiguration = createRepositoryConfiguration();
 
@@ -175,7 +182,7 @@ protected OAIRequestParametersBuilder newXoaiRequest() {
     }
 
     public boolean isHarvestingServerEnabled() {
-        return settingsService.isTrueForKey(SettingsServiceBean.Key.OAIServerEnabled);
+        return settingsService.isTrueForKey(OAIServerEnabled);
     }
 
     // -------------------- PRIVATE --------------------
@@ -191,8 +198,9 @@ private void addSupportedMetadataFormats(Context context) {
             if (!exporter.isXMLFormat() || !exporter.isHarvestable()) {
                 continue;
             }
-            MetadataFormat metadataFormat = MetadataFormat.metadataFormat(exporter.getProviderName());
-            if (exporter.getXMLNameSpace().isEmpty() && exporter.getXMLSchemaLocation().isEmpty()) {
+            MetadataFormat metadataFormat = metadataFormat(exporter.getProviderName());
+            if (exporter.getXMLNameSpace().isEmpty() 
+                    && exporter.getXMLSchemaLocation().isEmpty()) {
                 continue;
             }
             metadataFormat.withNamespace(exporter.getXMLNameSpace());
@@ -219,32 +227,49 @@ private RepositoryConfiguration createRepositoryConfiguration() {
         // need to be configurable!
 
         String dataverseName = dataverseDao.findRootDataverse().getName();
-        String repositoryName = StringUtils.isEmpty(dataverseName) || "Root".equals(dataverseName) ? "Test Dataverse OAI Archive" : dataverseName + " Dataverse OAI Archive";
+        String repositoryName = isEmpty(dataverseName) || "Root".equals(dataverseName) 
+                ? "Test Dataverse OAI Archive" 
+                : dataverseName + " Dataverse OAI Archive";
         Date earliestDate = recordService.findEarliestDate();
 
         RepositoryConfiguration repositoryConfiguration = new RepositoryConfiguration()
                 .withRepositoryName(repositoryName)
                 .withBaseUrl(systemConfig.getDataverseSiteUrl() + "/oai")
                 .withCompression("gzip")        // ?
                 .withCompression("deflate")     // ?
-                .withAdminEmail(settingsService.getValueForKey(SettingsServiceBean.Key.SystemEmail))
+                .withAdminEmail(settingsService.getValueForKey(SystemEmail))
                 .withDeleteMethod(DeletedRecord.TRANSIENT)
                 .withGranularity(Granularity.Second)
                 .withMaxListIdentifiers(100)
                 .withMaxListRecords(100)
                 .withMaxListSets(100)
-                .withEarliestDate(earliestDate != null ? earliestDate : new Date());
+                .withEarliestDate(earliestDate != null ? earliestDate : new Date())
+                .withDescription(createDescription());
 
         return repositoryConfiguration;
     }
+
+    private String createDescription() {
+        return "<description>" 
+                + "<oai-identifier xmlns=\"http://www.openarchives.org/OAI/2.0/oai-identifier\">"
+                + "<scheme>oai</scheme>"
+                + "<repositoryIdentifier>"
+                + this.systemConfig.getDataverseServer()
+                + "</repositoryIdentifier>" 
+                + "<delimiter>:</delimiter>" 
+                + "<sampleIdentifier>oai:"
+                + this.systemConfig.getDataverseServer() 
+                +":doi%3A10.18150%2FAB1CD2</sampleIdentifier>"
+                + "</oai-identifier>"
+                + "</description>";
+    }
 
     private void processRequest(HttpServletRequest request, HttpServletResponse response)
             throws ServletException, IOException {
 
         try {
             if (!isHarvestingServerEnabled()) {
-                response.sendError(
-                        HttpServletResponse.SC_SERVICE_UNAVAILABLE,
+                response.sendError(SC_SERVICE_UNAVAILABLE,
                         "Sorry. OAI Service is disabled on this Dataverse node.");
                 return;
             }
@@ -300,14 +325,15 @@ private void writeListRecords(HttpServletResponse response, OAIPMH handle)
             throw new IOException("An error or a valid response must be set");
         }
 
-        if (!verb.getType().equals(Verb.Type.ListRecords)) {
+        if (!verb.getType().equals(ListRecords)) {
             throw new IOException("writeListRecords() called on a non-ListRecords verb");
         }
 
         outputStream.write(("<" + verb.getType().displayName() + ">").getBytes());
         outputStream.flush();
 
-        ((XlistRecords) verb).writeToStream(outputStream, exportService, systemConfig.getDataverseSiteUrl());
+        ((XlistRecords) verb).writeToStream(outputStream, exportService, 
+                systemConfig.getDataverseSiteUrl());
 
         outputStream.write(("</" + verb.getType().displayName() + ">").getBytes());
         outputStream.write(("</" + OAI_PMH + ">\n").getBytes());
@@ -326,14 +352,15 @@ private void writeGetRecord(HttpServletResponse response, OAIPMH handle)
             throw new IOException("An error or a valid response must be set");
         }
 
-        if (!verb.getType().equals(Verb.Type.GetRecord)) {
+        if (!verb.getType().equals(GetRecord)) {
             throw new IOException("writeListRecords() called on a non-GetRecord verb");
         }
 
         outputStream.write(("<" + verb.getType().displayName() + ">").getBytes());
         outputStream.flush();
 
-        ((XgetRecord) verb).writeToStream(outputStream, exportService, systemConfig.getDataverseSiteUrl());
+        ((XgetRecord) verb).writeToStream(outputStream, exportService, 
+                systemConfig.getDataverseSiteUrl());
 
         outputStream.write(("</" + verb.getType().displayName() + ">").getBytes());
         outputStream.write(("</" + OAI_PMH + ">\n").getBytes());

dataverse-webapp/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XitemRepository.java

@@ -4,6 +4,8 @@
 import edu.harvard.iq.dataverse.harvest.server.OAIRecordServiceBean;
 import edu.harvard.iq.dataverse.persistence.dataset.Dataset;
 import edu.harvard.iq.dataverse.persistence.harvest.OAIRecord;
+import edu.harvard.iq.dataverse.util.SystemConfig;
+
 import org.apache.commons.lang3.StringUtils;
 import org.dspace.xoai.dataprovider.exceptions.IdDoesNotExistException;
 import org.dspace.xoai.dataprovider.exceptions.OAIException;
@@ -13,6 +15,7 @@
 import org.dspace.xoai.dataprovider.model.Item;
 import org.dspace.xoai.dataprovider.model.ItemIdentifier;
 import org.dspace.xoai.dataprovider.repository.ItemRepository;
+import org.dspace.xoai.util.URLEncoder;
 
 import java.util.ArrayList;
 import java.util.Date;
@@ -28,24 +31,27 @@
  * XOAI "items".
  */
 public class XitemRepository implements ItemRepository {
-    private static Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.harvest.server.xoai.XitemRepository");
+    private static Logger logger = Logger.getLogger(XitemRepository.class.getCanonicalName());
 
-    private OAIRecordServiceBean recordService;
-    private DatasetDao datasetDao;
+    private final OAIRecordServiceBean recordService;
+    private final DatasetDao datasetDao;
+    private final SystemConfig systemConfig;
 
     // -------------------- CONSTRUCTORS --------------------
 
-    public XitemRepository(OAIRecordServiceBean recordService, DatasetDao datasetDao) {
-        super();
+    public XitemRepository(OAIRecordServiceBean recordService, DatasetDao datasetDao, 
+            final SystemConfig systemConfig) {
         this.recordService = recordService;
         this.datasetDao = datasetDao;
+        this.systemConfig = systemConfig;
     }
 
     // -------------------- LOGIC --------------------
 
     @Override
     public Item getItem(String identifier) throws IdDoesNotExistException, OAIException {
         logger.fine("getItem; calling findOaiRecordsByGlobalId, identifier " + identifier);
+        identifier = parseIdetifier(identifier);
         List<OAIRecord> oaiRecords = recordService.findOaiRecordsByGlobalId(identifier);
         if (oaiRecords.isEmpty()) {
             throw new IdDoesNotExistException();
@@ -63,7 +69,7 @@ public Item getItem(String identifier) throws IdDoesNotExistException, OAIExcept
                 .map(OAIRecord::getLastUpdateTime)
                 .orElse(oaiRecords.get(0).getLastUpdateTime());
 
-        Xitem xoaiItem = new Xitem(identifier, lastUpdateTimestamp, removed)
+        Xitem xoaiItem = new Xitem(prepareId(identifier), lastUpdateTimestamp, removed)
                 .withDataset(dataset);
 
         oaiRecords.forEach(record -> xoaiItem.addSet(record.getSetName()));
@@ -118,7 +124,7 @@ public ListItemIdentifiersResult getItemIdentifiers(List<ScopedFilter> filters,
         }
         for (int i = offset; i < offset + length && i < oaiRecords.size(); i++) {
             OAIRecord record = oaiRecords.get(i);
-            Xitem xItem = new Xitem(record.getGlobalId(), record.getLastUpdateTime(), record.isRemoved());
+            Xitem xItem = new Xitem(getIdOf(record), record.getLastUpdateTime(), record.isRemoved());
             xoaiItems.add(xItem);
         }
 
@@ -185,7 +191,8 @@ public ListItemsResults getItems(List<ScopedFilter> filters, int offset, int len
             OAIRecord oaiRecord = oaiRecords.get(i);
             Dataset dataset = datasetDao.findByGlobalId(oaiRecord.getGlobalId());
             if (dataset != null) {
-                Xitem xItem = new Xitem(oaiRecord.getGlobalId(), oaiRecord.getLastUpdateTime(), oaiRecord.isRemoved())
+                Xitem xItem = new Xitem(getIdOf(oaiRecord), oaiRecord.getLastUpdateTime(), 
+                        oaiRecord.isRemoved())
                         .withDataset(dataset);
                 xoaiItems.add(xItem);
             }
@@ -198,6 +205,29 @@ public ListItemsResults getItems(List<ScopedFilter> filters, int offset, int len
     }
 
     // -------------------- PRIATE --------------------
+
+    private String getIdOf(final OAIRecord record) {
+        return prepareId(record.getGlobalId());
+    }
+
+    private String prepareId(final String id) {
+        if (this.systemConfig.useOAIStrictIdentifierScheme()) {
+            return "oai:" + this.systemConfig.getDataverseServer() + ':'
+                    + URLEncoder.encode(id);
+        } else {
+            return id;
+        }
+    }
+
+    private String parseIdetifier(String id) {
+        if (id.startsWith("oai:")) {
+            id = id.substring(4);
+            final int index = id.indexOf(':');
+            return id.substring(index + 1);
+        } else {
+            return id;
+        }
+    }
 
     private void addExtraSets(List<? extends ItemIdentifier> xoaiItemsList) {
         Map<String, Xitem> xoaiItemsMap = new HashMap<>();

dataverse-webapp/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java

@@ -766,8 +766,17 @@ public enum Key {
          * Default value: empty.
          */
         CustomSearchLocationsSolrFields,
-
-        MaxResultsCountSavedToFile;
+        /**
+         * Specifies maximum number of search results axported to CSV file
+         */
+        MaxResultsCountSavedToFile,
+        /**
+         * If true, OAI-PMH exporters will use identifier scheme specified in 
+         * https://www.openarchives.org/OAI/openarchivesprotocol.html#UniqueIdentifier
+         * https://www.openarchives.org/OAI/2.0/guidelines-oai-identifier.htm.
+         * eg (oai:www.wbc.poznan.pl:568432).
+         */
+        UseOAIStrictIdentifierScheme;
 
 
 

dataverse-webapp/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java

@@ -15,6 +15,9 @@
 import javax.ejb.Stateless;
 import javax.inject.Inject;
 import javax.inject.Named;
+
+import static edu.harvard.iq.dataverse.settings.SettingsServiceBean.Key.UseOAIStrictIdentifierScheme;
+
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.util.Arrays;
@@ -119,6 +122,10 @@ public String getVersion() {
 
         return appVersion;
     }
+
+    public boolean useOAIStrictIdentifierScheme() {
+        return this.settingsService.isTrueForKey(UseOAIStrictIdentifierScheme);
+    }
 
     public int getMinutesUntilPasswordResetTokenExpires() {
         return settingsService.getValueForKeyAsInt(SettingsServiceBean.Key.MinutesUntilPasswordResetTokenExpires);