[TLC-1172] Paginate CrisMetric db rows when updating index

Author: kshepherd

dspace-api/src/main/java/org/dspace/metrics/UpdateCrisMetricsInSolrDocService.java

@@ -21,6 +21,7 @@
 import org.dspace.discovery.IndexingService;
 import org.dspace.discovery.SearchServiceException;
 import org.dspace.scripts.handler.DSpaceRunnableHandler;
+import org.dspace.services.ConfigurationService;
 import org.dspace.utils.DSpace;
 
 /**
@@ -30,6 +31,8 @@ public class UpdateCrisMetricsInSolrDocService {
 
     private static final Logger log = LogManager.getLogger(UpdateCrisMetricsInSolrDocService.class);
 
+    private ConfigurationService configurationService = new DSpace().getConfigurationService();
+
     private CrisMetricsService crisMetricsService = new DSpace().getServiceManager().getServiceByName(
             CrisMetricsServiceImpl.class.getName(), CrisMetricsServiceImpl.class);
 
@@ -42,21 +45,28 @@ public void performUpdate(Context context, DSpaceRunnableHandler handler, boolea
 
     public void performUpdate(Context context, DSpaceRunnableHandler handler, boolean optimize, UUID resourceUuid) {
         try {
-            List<CrisMetrics> metrics = resourceUuid == null
-                    ? crisMetricsService.findAllLast(context,-1,-1)
-                    : crisMetricsService.findLastMetricsByResourceId(context, resourceUuid, -1, -1);
+            int offset = 0;
+            int limit = configurationService.getIntProperty("metrics.indexer.page", 1000);
             handler.logInfo("Metric update start");
-            for (CrisMetrics metric : metrics) {
-                try {
-                    crisIndexingService.updateMetrics(context, metric);
-                } catch (RemoteSolrException rse) {
-                    if (StringUtils.containsIgnoreCase(rse.getMessage(), "Did not find child ID Item-")) {
-                        log.error(rse.getMessage());
-                    } else {
-                        throw rse;
+            List<CrisMetrics> metrics;
+            while (!(metrics = (resourceUuid == null
+                            ? crisMetricsService.findAllLast(context, limit, offset)
+                            : crisMetricsService.findLastMetricsByResourceId(context, resourceUuid, limit, offset)
+                            )).isEmpty()) {
+                for (CrisMetrics metric : metrics) {
+                    try {
+                        crisIndexingService.updateMetrics(context, metric);
+                    } catch (RemoteSolrException rse) {
+                        if (StringUtils.containsIgnoreCase(rse.getMessage(), "Did not find child ID Item-")) {
+                            log.error(rse.getMessage());
+                        } else {
+                            throw rse;
+                        }
                     }
                 }
+                offset += limit;
             }
+
             handler.logInfo("Metric update end");
             if (optimize) {
                 handler.logInfo("Starting solr optimization");
@@ -68,4 +78,4 @@ public void performUpdate(Context context, DSpaceRunnableHandler handler, boolea
             throw new RuntimeException(e.getMessage(), e);
         }
     }
-}
+}

dspace/config/modules/metrics.cfg

@@ -25,6 +25,10 @@ metrics.scopus.person.instToken = ${scopus.instToken}
 # The default limit for the items to be updated by the update-metrics script, used if no limit is provided as parameter
 metrics.update-metrics-script.limit = 1750
 
+# The page size of CrisMetrics rows to process when updating the solr index
+# Default: 1000
+metrics.indexer.page = 1000
+
 #---------------------------------------------------------------#
 # Configure altmetric.com badges.                               #
 # See http://api.altmetric.com/embeds.html                      #
@@ -131,4 +135,4 @@ plumx.list-data-hide-mentions = false
 plumx.list-data-hide-socialmedia = false
 plumx.list-data-hide-citations = false
 plumx.list-data-pass-hidden-categories = false
-plumx.list-data-detail-same-page = false
+plumx.list-data-detail-same-page = false