arg to do sorting on the server

Author: cgoina

colormipsearch-tools/src/main/java/org/janelia/colormipsearch/cmd/ExportData4NBCmd.java

@@ -178,6 +178,9 @@ static class ExportMatchesCmdArgs extends AbstractCmdArgs {
         @Parameter(names = {"--max-matches-per-mip"}, description = "Maximum number of matches per mip")
         int maxMatchesPerMIP = 0;
 
+        @Parameter(names = {"--use-local-sort"}, description = "Don't use (DB) server side sorting - retrieve all data and sort it locally", arity = 0)
+        boolean useLocalSort = false;
+
         ExportMatchesCmdArgs(CommonArgs commonArgs) {
             super(commonArgs);
         }
@@ -315,7 +318,8 @@ private DataExporter getDataExporter() {
                         args.processingPartitionSize,
                         args.maxMatchesPerMIP,
                         args.maxMatchedNamesPerMIP,
-                        args.dbReadPageSize
+                        args.dbReadPageSize,
+                        !args.useLocalSort
                 );
             case LM_CD_MATCHES:
                 return new LMCDMatchesExporter(
@@ -342,7 +346,8 @@ private DataExporter getDataExporter() {
                         args.processingPartitionSize,
                         args.maxMatchesPerMIP,
                         args.maxMatchedNamesPerMIP,
-                        args.dbReadPageSize
+                        args.dbReadPageSize,
+                        !args.useLocalSort
                 );
             case EM_PPP_MATCHES:
                 return new EMPPPMatchesExporter(

colormipsearch-tools/src/main/java/org/janelia/colormipsearch/cmd/dataexport/AbstractCDMatchesExporter.java

@@ -51,6 +51,7 @@ public abstract class AbstractCDMatchesExporter extends AbstractDataExporter {
     final int maxMatchedTargets;
     final int maxMatchesWithSameNamePerMIP;
     final int readPageSize;
+    final boolean useServerSorting;
 
     protected AbstractCDMatchesExporter(CachedDataHelper jacsDataHelper,
                                         DataSourceParam dataSourceParam,
@@ -71,7 +72,8 @@ protected AbstractCDMatchesExporter(CachedDataHelper jacsDataHelper,
                                         int processingPartitionSize,
                                         int maxMatchedTargets,
                                         int maxMatchesWithSameNamePerMIP,
-                                        int readPageSize) {
+                                        int readPageSize,
+                                        boolean useServerSorting) {
         super(jacsDataHelper, dataSourceParam, urlTransformer, imageStoreMapping, outputDir, executor);
         this.targetLibraries = targetLibraries;
         this.targetTags = targetTags;
@@ -87,6 +89,7 @@ protected AbstractCDMatchesExporter(CachedDataHelper jacsDataHelper,
         this.maxMatchedTargets = maxMatchedTargets;
         this.maxMatchesWithSameNamePerMIP = maxMatchesWithSameNamePerMIP;
         this.readPageSize = readPageSize;
+        this.useServerSorting = useServerSorting;
     }
 
     void retrieveAllCDMIPs(List<CDMatchEntity<AbstractNeuronEntity, AbstractNeuronEntity>> matches) {

colormipsearch-tools/src/main/java/org/janelia/colormipsearch/cmd/dataexport/EMCDMatchesExporter.java

@@ -22,6 +22,8 @@
 import org.janelia.colormipsearch.datarequests.PagedRequest;
 import org.janelia.colormipsearch.datarequests.PagedResult;
 import org.janelia.colormipsearch.datarequests.ScoresFilter;
+import org.janelia.colormipsearch.datarequests.SortCriteria;
+import org.janelia.colormipsearch.datarequests.SortDirection;
 import org.janelia.colormipsearch.dto.AbstractNeuronMetadata;
 import org.janelia.colormipsearch.dto.CDMatchedTarget;
 import org.janelia.colormipsearch.dto.EMNeuronMetadata;
@@ -59,7 +61,8 @@ public EMCDMatchesExporter(CachedDataHelper jacsDataHelper,
                                int processingPartitionSize,
                                int maxMatchedTargets,
                                int maxMatchesWithSameNamePerMIP,
-                               int readPageSize) {
+                               int readPageSize,
+                               boolean useServerSorting) {
         super(jacsDataHelper,
                 dataSourceParam,
                 targetLibraries,
@@ -79,7 +82,8 @@ public EMCDMatchesExporter(CachedDataHelper jacsDataHelper,
                 processingPartitionSize,
                 maxMatchedTargets,
                 maxMatchesWithSameNamePerMIP,
-                readPageSize);
+                readPageSize,
+                useServerSorting);
     }
 
     @Override
@@ -119,9 +123,9 @@ private void runExportForMaskIds(int jobId, List<String> maskMipIds) {
                     /* matchTags */null,
                     /* matchExcludedTags */matchesExcludedTags,
                     scoresFilter,
-                    null/*no sorting because it uses too much memory on the server*/,
+                    useServerSorting ? Collections.singletonList(new SortCriteria("normalizedScore", SortDirection.DESC)) : null,
                     /* from */0,
-                    /* nRecords */-1,
+                    /* nRecords */useServerSorting && maxMatchedTargets > 0 ? maxMatchedTargets : -1,
                     /* readPageSize */readPageSize);
             List<CDMatchEntity<AbstractNeuronEntity, AbstractNeuronEntity>> selectedMatchesForMask;
             LOG.info("Found {} color depth matches for mip {}", allMatchesForMask.size(), maskMipId);

colormipsearch-tools/src/main/java/org/janelia/colormipsearch/cmd/dataexport/LMCDMatchesExporter.java

@@ -22,6 +22,8 @@
 import org.janelia.colormipsearch.datarequests.PagedRequest;
 import org.janelia.colormipsearch.datarequests.PagedResult;
 import org.janelia.colormipsearch.datarequests.ScoresFilter;
+import org.janelia.colormipsearch.datarequests.SortCriteria;
+import org.janelia.colormipsearch.datarequests.SortDirection;
 import org.janelia.colormipsearch.dto.AbstractNeuronMetadata;
 import org.janelia.colormipsearch.dto.CDMatchedTarget;
 import org.janelia.colormipsearch.dto.EMNeuronMetadata;
@@ -59,7 +61,8 @@ public LMCDMatchesExporter(CachedDataHelper jacsDataHelper,
                                int processingPartitionSize,
                                int maxMatchedTargets,
                                int maxMatchesWithSameNamePerMIP,
-                               int readPageSize) {
+                               int readPageSize,
+                               boolean useServerSorting) {
         super(jacsDataHelper,
                 dataSourceParam,
                 maskLibraries,
@@ -79,7 +82,8 @@ public LMCDMatchesExporter(CachedDataHelper jacsDataHelper,
                 processingPartitionSize,
                 maxMatchedTargets,
                 maxMatchesWithSameNamePerMIP,
-                readPageSize);
+                readPageSize,
+                useServerSorting);
     }
 
     @Override
@@ -120,9 +124,9 @@ private void runExportForTargetIds(int jobId, List<String> targetMipIds) {
                     /* matchTags */null,
                     /* matchExcludedTags */matchesExcludedTags,
                     /* matchesScoresFilter */scoresFilter,
-                    /* no sorting yet because it uses too much memory on the server */null,
+                    useServerSorting ? Collections.singletonList(new SortCriteria("normalizedScore", SortDirection.DESC)) : null,
                     /* from */0,
-                    /* nRecords */-1,
+                    /* nRecords */useServerSorting && maxMatchedTargets > 0 ? maxMatchedTargets : -1,
                     /* readPageSize */readPageSize);
             LOG.info("Found {} color depth matches for mip {}", allMatchesForTarget.size(), targetMipId);
             List<CDMatchEntity<AbstractNeuronEntity, AbstractNeuronEntity>> selectedMatchesForTarget;