From b37f45c0e05fab4bdd6e1d92cea272d7869de2ce Mon Sep 17 00:00:00 2001 From: Christine Poerschke Date: Tue, 28 Sep 2021 14:03:00 +0100 Subject: [PATCH 01/13] git apply SOLR-13681.patch --- .../apache/solr/update/SolrIndexConfig.java | 22 +++++++++++++++++-- .../solrconfig-sortingmergepolicyfactory.xml | 1 + .../solr/update/SolrIndexConfigTest.java | 1 + .../src/indexconfig-in-solrconfig.adoc | 11 ++++++++++ 4 files changed, 33 insertions(+), 2 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java b/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java index 0bdfe84f073..08ab2b650cd 100644 --- a/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java +++ b/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java @@ -44,6 +44,8 @@ import org.apache.solr.index.MergePolicyFactoryArgs; import org.apache.solr.index.SortingMergePolicy; import org.apache.solr.schema.IndexSchema; +import org.apache.solr.search.SortSpec; +import org.apache.solr.search.SortSpecParsing; import org.apache.solr.util.SolrPluginUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -71,6 +73,7 @@ public class SolrIndexConfig implements MapSerializable { public final int writeLockTimeout; public final String lockType; + public final String indexSort; public final PluginInfo mergePolicyFactoryInfo; public final PluginInfo mergeSchedulerInfo; public final PluginInfo metricsInfo; @@ -89,6 +92,7 @@ private SolrIndexConfig(SolrConfig solrConfig) { ramPerThreadHardLimitMB = -1; writeLockTimeout = -1; lockType = DirectoryFactory.LOCK_TYPE_NATIVE; + indexSort = null; mergePolicyFactoryInfo = null; mergeSchedulerInfo = null; mergedSegmentWarmerInfo = null; @@ -137,6 +141,7 @@ public SolrIndexConfig(SolrConfig solrConfig, String prefix, SolrIndexConfig def writeLockTimeout=solrConfig.getInt(prefix+"/writeLockTimeout", def.writeLockTimeout); lockType=solrConfig.get(prefix+"/lockType", def.lockType); + indexSort=solrConfig.get(prefix+"/indexSort", def.indexSort); List infos = solrConfig.readPluginInfos(prefix + "/metrics", false, false); if (infos.isEmpty()) { @@ -192,6 +197,9 @@ public Map toMap(Map map) { if (metricsInfo != null) { m.put("metrics", metricsInfo); } + if (indexSort != null) { + m.put("indexSort", indexSort); + } if (mergePolicyFactoryInfo != null) { m.put("mergePolicyFactory", mergePolicyFactoryInfo); } @@ -238,9 +246,19 @@ public IndexWriterConfig toIndexWriterConfig(SolrCore core) throws IOException { iwc.setMergeScheduler(mergeScheduler); iwc.setInfoStream(infoStream); + if (indexSort != null) { + SortSpec indexSortSpec = SortSpecParsing.parseSortSpec(indexSort, schema); + iwc.setIndexSort(indexSortSpec.getSort()); + } + if (mergePolicy instanceof SortingMergePolicy) { - Sort indexSort = ((SortingMergePolicy) mergePolicy).getSort(); - iwc.setIndexSort(indexSort); + Sort mergeSort = ((SortingMergePolicy) mergePolicy).getSort(); + Sort indexSort = iwc.getIndexSort(); + if (indexSort != null && !indexSort.equals(mergeSort)) { + log.warn("indexSort={} differs from mergePolicySort={}", indexSort, mergeSort); + } else { + iwc.setIndexSort(mergeSort); + } } iwc.setUseCompoundFile(useCompoundFile); diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-sortingmergepolicyfactory.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-sortingmergepolicyfactory.xml index 17e6f4c7a56..fada9d8a816 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-sortingmergepolicyfactory.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-sortingmergepolicyfactory.xml @@ -29,6 +29,7 @@ timestamp_i_dvo desc ${solr.tests.lockType:single} + timestamp_i_dvo desc diff --git a/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java b/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java index de5944edb45..af40fce87f1 100644 --- a/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java +++ b/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java @@ -140,6 +140,7 @@ public void testSortingMPSolrIndexConfigCreation() throws Exception { final Sort expected = new Sort(new SortField(expectedFieldName, expectedFieldType, expectedFieldSortDescending)); final Sort actual = sortingMergePolicy.getSort(); assertEquals("SortingMergePolicy.getSort", expected, actual); + assertEquals("indexSort", expected, iwc.getIndexSort()); } public void testMergedSegmentWarmerIndexConfigCreation() throws Exception { diff --git a/solr/solr-ref-guide/src/indexconfig-in-solrconfig.adoc b/solr/solr-ref-guide/src/indexconfig-in-solrconfig.adoc index 39e87b942e1..ff35b565fb1 100644 --- a/solr/solr-ref-guide/src/indexconfig-in-solrconfig.adoc +++ b/solr/solr-ref-guide/src/indexconfig-in-solrconfig.adoc @@ -67,6 +67,17 @@ Sets the maximum memory (defined in megabytes) consumption per thread triggering NOTE: This is an expert level parameter as it triggers forced flush even if <> has not been exceeded. +=== indexSort + +TODO + +[source,xml] +---- +timestamp desc +---- + +TODO + == Merging Index Segments === mergePolicyFactory From bc397796e893ef283fd492017a1d23b2742ce597 Mon Sep 17 00:00:00 2001 From: Christine Poerschke Date: Tue, 28 Sep 2021 14:25:42 +0100 Subject: [PATCH 02/13] make it compile --- solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java b/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java index 52a55dd7463..9c335ed5890 100644 --- a/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java +++ b/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java @@ -215,7 +215,7 @@ public Map toMap(Map map) { map.put("metrics", metricsInfo); } if (indexSort != null) { - m.put("indexSort", indexSort); + map.put("indexSort", indexSort); } if (mergePolicyFactoryInfo != null) { map.put("mergePolicyFactory", mergePolicyFactoryInfo); From e8ecd260c6ef1fc664dd5d569d7ab7d4e61420f8 Mon Sep 17 00:00:00 2001 From: Christine Poerschke Date: Wed, 29 Sep 2021 09:08:36 +0100 Subject: [PATCH 03/13] code review feedback: logging tweak --- solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java b/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java index 9c335ed5890..245930dc04b 100644 --- a/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java +++ b/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java @@ -278,7 +278,7 @@ public IndexWriterConfig toIndexWriterConfig(SolrCore core) throws IOException { Sort mergeSort = ((SortingMergePolicy) mergePolicy).getSort(); Sort indexSort = iwc.getIndexSort(); if (indexSort != null && !indexSort.equals(mergeSort)) { - log.warn("indexSort={} differs from mergePolicySort={}", indexSort, mergeSort); + log.warn("indexSort={} differs from mergePolicySort={} (using indexSort)", indexSort, mergeSort); } else { iwc.setIndexSort(mergeSort); } From 7514d348b3d1b04ae5e1b101d97c86e364e82189 Mon Sep 17 00:00:00 2001 From: Christine Poerschke Date: Wed, 29 Sep 2021 09:09:37 +0100 Subject: [PATCH 04/13] add (skeleton) SolrIndexConfigIndexSortTest class --- .../cloud/SolrIndexConfigIndexSortTest.java | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 solr/core/src/test/org/apache/solr/cloud/SolrIndexConfigIndexSortTest.java diff --git a/solr/core/src/test/org/apache/solr/cloud/SolrIndexConfigIndexSortTest.java b/solr/core/src/test/org/apache/solr/cloud/SolrIndexConfigIndexSortTest.java new file mode 100644 index 00000000000..d87ce653511 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/cloud/SolrIndexConfigIndexSortTest.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.cloud; + +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.junit.Before; +import org.junit.Test; + +public class SolrIndexConfigIndexSortTest extends SolrCloudTestCase { + + private static String COLLECTION; + + @Before + public void setUp() throws Exception { + super.setUp(); + + // decide collection name ... + COLLECTION = "collection"+(1+random().nextInt(100)) ; + + // create and configure cluster + configureCluster(1) + .addConfig("conf", configset("cloud-minimal")) + .configure(); + + // create an empty collection + CollectionAdminRequest + .createCollection(COLLECTION, "conf", 1, 1) + .processAndWait(cluster.getSolrClient(), DEFAULT_TIMEOUT); + cluster.waitForActiveCollection(COLLECTION, 1, 1); + } + + @Test + public void testStuff() throws Exception { + // TODO + } + +} From 8171b27c9e03f330953b61f433a7f7d9f0979697 Mon Sep 17 00:00:00 2001 From: Christine Poerschke Date: Wed, 29 Sep 2021 09:10:11 +0100 Subject: [PATCH 05/13] add indexConfig fragments to TestSolrConfigHandler --- .../org/apache/solr/core/TestSolrConfigHandler.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java b/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java index e760c205bf6..c2e483f5e94 100644 --- a/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java +++ b/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java @@ -191,6 +191,16 @@ public void testProperty() throws Exception { m = getRespMap("/config/overlay", harness); assertNull(m._get("overlay/props/updateHandler/autoCommit/maxDocs",null)); assertEquals("10", m._getStr("overlay/props/updateHandler/autoCommit/maxTime",null)); + + m = getRespMap("/config", harness); + assertNotNull(m._get("config/indexConfig/useCompoundFile",null)); + assertNotNull(m._get("config/indexConfig/maxBufferedDocs",null)); + assertNotNull(m._get("config/indexConfig/ramBufferSizeMB",null)); + assertNotNull(m._get("config/indexConfig/ramPerThreadHardLimitMB",null)); + assertNotNull(m._get("config/indexConfig/maxCommitMergeWaitTime",null)); + assertNotNull(m._get("config/indexConfig/writeLockTimeout",null)); + assertNotNull(m._get("config/indexConfig/lockType",null)); + assertNotNull(m._get("config/indexConfig/infoStreamEnabled",null)); } public void testUserProp() throws Exception { From 33a722e14bc4d9a61b0828bca8e066248576facc Mon Sep 17 00:00:00 2001 From: Christine Poerschke Date: Wed, 29 Sep 2021 09:11:41 +0100 Subject: [PATCH 06/13] Revert "add indexConfig fragments to TestSolrConfigHandler" This reverts commit 8171b27c9e03f330953b61f433a7f7d9f0979697. --- .../org/apache/solr/core/TestSolrConfigHandler.java | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java b/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java index c2e483f5e94..e760c205bf6 100644 --- a/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java +++ b/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java @@ -191,16 +191,6 @@ public void testProperty() throws Exception { m = getRespMap("/config/overlay", harness); assertNull(m._get("overlay/props/updateHandler/autoCommit/maxDocs",null)); assertEquals("10", m._getStr("overlay/props/updateHandler/autoCommit/maxTime",null)); - - m = getRespMap("/config", harness); - assertNotNull(m._get("config/indexConfig/useCompoundFile",null)); - assertNotNull(m._get("config/indexConfig/maxBufferedDocs",null)); - assertNotNull(m._get("config/indexConfig/ramBufferSizeMB",null)); - assertNotNull(m._get("config/indexConfig/ramPerThreadHardLimitMB",null)); - assertNotNull(m._get("config/indexConfig/maxCommitMergeWaitTime",null)); - assertNotNull(m._get("config/indexConfig/writeLockTimeout",null)); - assertNotNull(m._get("config/indexConfig/lockType",null)); - assertNotNull(m._get("config/indexConfig/infoStreamEnabled",null)); } public void testUserProp() throws Exception { From 2247d798a7b4a0484de4a91f43362809a8ca50ce Mon Sep 17 00:00:00 2001 From: Christine Poerschke Date: Wed, 29 Sep 2021 13:23:59 +0100 Subject: [PATCH 07/13] SolrIndexConfigTest: add testIndexSortSolrIndexConfigCreation as counter-part to testSortingMPSolrIndexConfigCreation --- .../collection1/conf/solrconfig-indexSort.xml | 52 +++++++++++++++++++ .../solr/update/SolrIndexConfigTest.java | 18 +++++++ 2 files changed, 70 insertions(+) create mode 100644 solr/core/src/test-files/solr/collection1/conf/solrconfig-indexSort.xml diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-indexSort.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-indexSort.xml new file mode 100644 index 00000000000..0c7c5027b7b --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-indexSort.xml @@ -0,0 +1,52 @@ + + + + + + ${tests.luceneMatchVersion:LATEST} + + + + + ${solr.tests.lockType:single} + timestamp_i_dvo desc + + + + + + + ${solr.ulog.dir:} + + + + ${solr.autoCommit.maxTime:-1} + false + + + + ${solr.autoSoftCommit.maxTime:-1} + + + + + text + + + + diff --git a/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java b/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java index ed7ec427379..d5165379bfd 100644 --- a/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java +++ b/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java @@ -52,6 +52,7 @@ public class SolrIndexConfigTest extends SolrTestCaseJ4 { private static final String solrConfigFileNameTieredMergePolicyFactory = "solrconfig-tieredmergepolicyfactory.xml"; private static final String solrConfigFileNameConnMSPolicyFactory = "solrconfig-concurrentmergescheduler.xml"; private static final String solrConfigFileNameSortingMergePolicyFactory = "solrconfig-sortingmergepolicyfactory.xml"; + private static final String solrConfigFileNameIndexSort = "solrconfig-indexSort.xml"; private static final String schemaFileName = "schema.xml"; @BeforeClass @@ -149,6 +150,23 @@ public void testSortingMPSolrIndexConfigCreation() throws Exception { assertEquals("indexSort", expected, iwc.getIndexSort()); } + public void testIndexSortSolrIndexConfigCreation() throws Exception { + final String expectedFieldName = "timestamp_i_dvo"; + final SortField.Type expectedFieldType = SortField.Type.INT; + final boolean expectedFieldSortDescending = true; + + SolrConfig solrConfig = new SolrConfig(instanceDir, solrConfigFileNameIndexSort); + SolrIndexConfig solrIndexConfig = new SolrIndexConfig(solrConfig, null, null); + assertNotNull(solrIndexConfig); + IndexSchema indexSchema = IndexSchemaFactory.buildIndexSchema(schemaFileName, solrConfig); + + h.getCore().setLatestSchema(indexSchema); + IndexWriterConfig iwc = solrIndexConfig.toIndexWriterConfig(h.getCore()); + + final Sort expected = new Sort(new SortField(expectedFieldName, expectedFieldType, expectedFieldSortDescending)); + assertEquals("indexSort", expected, iwc.getIndexSort()); + } + public void testMergedSegmentWarmerIndexConfigCreation() throws Exception { SolrConfig solrConfig = new SolrConfig(instanceDir, solrConfigFileNameWarmerRandomMergePolicyFactory); SolrIndexConfig solrIndexConfig = new SolrIndexConfig(solrConfig, null, null); From 1478be7bbd44d05168b09dc8ece9d473e9ba3e36 Mon Sep 17 00:00:00 2001 From: Christine Poerschke Date: Wed, 29 Sep 2021 13:25:18 +0100 Subject: [PATCH 08/13] TestSegmentSorting: solrconfig-sortingmergepolicyfactory.xml/solrconfig-indexSort.xml dual test coverage (latter still failing) --- .../src/java/org/apache/solr/search/SolrIndexSearcher.java | 2 +- .../src/test/org/apache/solr/cloud/TestSegmentSorting.java | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java index 2e6e3368353..e7b8f33a2c1 100644 --- a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java +++ b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java @@ -178,7 +178,7 @@ private Collector buildAndRunCollectorChain(QueryResult qr, Query query, Collect if (cmd.getSegmentTerminateEarly()) { final Sort cmdSort = cmd.getSort(); final int cmdLen = cmd.getLen(); - final Sort mergeSort = core.getSolrCoreState().getMergePolicySort(); + final Sort mergeSort = core.getSolrCoreState().getMergePolicySort(); // TODO: need this account for indexSort also? if (cmdSort == null || cmdLen <= 0 || mergeSort == null || !EarlyTerminatingSortingCollector.canEarlyTerminate(cmdSort, mergeSort)) { diff --git a/solr/core/src/test/org/apache/solr/cloud/TestSegmentSorting.java b/solr/core/src/test/org/apache/solr/cloud/TestSegmentSorting.java index 9e83b55288a..dcb115568ec 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestSegmentSorting.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestSegmentSorting.java @@ -71,8 +71,10 @@ public void createCollection() throws Exception { final String collectionName = testName.getMethodName(); final CloudSolrClient cloudSolrClient = cluster.getSolrClient(); + final String solrConfigFileName = random().nextBoolean() ? "solrconfig-sortingmergepolicyfactory.xml" : "solrconfig-indexSort.xml"; + final Map collectionProperties = new HashMap<>(); - collectionProperties.put(CoreDescriptor.CORE_CONFIG, "solrconfig-sortingmergepolicyfactory.xml"); + collectionProperties.put(CoreDescriptor.CORE_CONFIG, solrConfigFileName); CollectionAdminRequest.Create cmd = CollectionAdminRequest.createCollection(collectionName, configName, From 14b8c4927d55d6b5c3e00d68fba8a8e98059b8cc Mon Sep 17 00:00:00 2001 From: Christine Poerschke Date: Wed, 29 Sep 2021 13:29:04 +0100 Subject: [PATCH 09/13] Revert "add (skeleton) SolrIndexConfigIndexSortTest class" This reverts commit 7514d348b3d1b04ae5e1b101d97c86e364e82189. --- .../cloud/SolrIndexConfigIndexSortTest.java | 52 ------------------- 1 file changed, 52 deletions(-) delete mode 100644 solr/core/src/test/org/apache/solr/cloud/SolrIndexConfigIndexSortTest.java diff --git a/solr/core/src/test/org/apache/solr/cloud/SolrIndexConfigIndexSortTest.java b/solr/core/src/test/org/apache/solr/cloud/SolrIndexConfigIndexSortTest.java deleted file mode 100644 index d87ce653511..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/SolrIndexConfigIndexSortTest.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud; - -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.junit.Before; -import org.junit.Test; - -public class SolrIndexConfigIndexSortTest extends SolrCloudTestCase { - - private static String COLLECTION; - - @Before - public void setUp() throws Exception { - super.setUp(); - - // decide collection name ... - COLLECTION = "collection"+(1+random().nextInt(100)) ; - - // create and configure cluster - configureCluster(1) - .addConfig("conf", configset("cloud-minimal")) - .configure(); - - // create an empty collection - CollectionAdminRequest - .createCollection(COLLECTION, "conf", 1, 1) - .processAndWait(cluster.getSolrClient(), DEFAULT_TIMEOUT); - cluster.waitForActiveCollection(COLLECTION, 1, 1); - } - - @Test - public void testStuff() throws Exception { - // TODO - } - -} From 7feca02a3597e34be3a0cf88a564c2b7810a9bc6 Mon Sep 17 00:00:00 2001 From: Christine Poerschke Date: Wed, 29 Sep 2021 13:44:23 +0100 Subject: [PATCH 10/13] SolrIndexConfigTest: assert that indexSort is not present if not configured --- .../src/test/org/apache/solr/update/SolrIndexConfigTest.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java b/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java index d5165379bfd..82a8b4fa57e 100644 --- a/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java +++ b/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java @@ -103,6 +103,7 @@ public void testTieredMPSolrIndexConfigCreation() throws Exception { assertEquals("ms.maxThreadCount", 42, ms.getMaxThreadCount()); assertEquals("ms.isAutoIOThrottle", true, ms.getAutoIOThrottle()); + assertNull("indexSort", iwc.getIndexSort()); } @Test @@ -125,6 +126,7 @@ public void testConcurrentMergeSchedularSolrIndexConfigCreation() throws Excepti assertEquals("ms.maxThreadCount", 42, ms.getMaxThreadCount()); assertEquals("ms.isAutoIOThrottle", false, ms.getAutoIOThrottle()); + assertNull("indexSort", iwc.getIndexSort()); } public void testSortingMPSolrIndexConfigCreation() throws Exception { @@ -178,6 +180,8 @@ public void testMergedSegmentWarmerIndexConfigCreation() throws Exception { h.getCore().setLatestSchema(indexSchema); IndexWriterConfig iwc = solrIndexConfig.toIndexWriterConfig(h.getCore()); assertEquals(SimpleMergedSegmentWarmer.class, iwc.getMergedSegmentWarmer().getClass()); + + assertNull("indexSort", iwc.getIndexSort()); } public void testToMap() throws Exception { From 736a75b5466954e8522beda5a6006a7e34e66baa Mon Sep 17 00:00:00 2001 From: Christine Poerschke Date: Thu, 30 Sep 2021 12:50:11 +0100 Subject: [PATCH 11/13] address TODO in SolrIndexSearcher.buildAndRunCollectorChain to make TestSegmentSorting pass --- .../apache/solr/search/SolrIndexSearcher.java | 21 +++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java index e7b8f33a2c1..c92e741135a 100644 --- a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java +++ b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java @@ -178,11 +178,24 @@ private Collector buildAndRunCollectorChain(QueryResult qr, Query query, Collect if (cmd.getSegmentTerminateEarly()) { final Sort cmdSort = cmd.getSort(); final int cmdLen = cmd.getLen(); - final Sort mergeSort = core.getSolrCoreState().getMergePolicySort(); // TODO: need this account for indexSort also? - if (cmdSort == null || cmdLen <= 0 || mergeSort == null || - !EarlyTerminatingSortingCollector.canEarlyTerminate(cmdSort, mergeSort)) { - log.warn("unsupported combination: segmentTerminateEarly=true cmdSort={} cmdLen={} mergeSort={}", cmdSort, cmdLen, mergeSort); + final Sort indexOrMergeSort; + final Sort mergeSort = core.getSolrCoreState().getMergePolicySort(); + final Sort indexSort; + { + final String indexSortStr = core.getSolrConfig().indexConfig.indexSort; + if (indexSortStr != null) { + indexSort = SortSpecParsing.parseSortSpec(indexSortStr, core.getLatestSchema()).getSort(); + indexOrMergeSort = indexSort; + } else { + indexSort = null; + indexOrMergeSort = mergeSort; + } + } + + if (cmdSort == null || cmdLen <= 0 || indexOrMergeSort == null || + !EarlyTerminatingSortingCollector.canEarlyTerminate(cmdSort, indexOrMergeSort)) { + log.warn("unsupported combination: segmentTerminateEarly=true cmdSort={} cmdLen={} indexOrMergeSort={}", cmdSort, cmdLen, indexOrMergeSort); } else { collector = earlyTerminatingSortingCollector = new EarlyTerminatingSortingCollector(collector, cmdSort, cmd.getLen()); } From f0889c0a5cf73eaa5cdc860ef437ace4deab954d Mon Sep 17 00:00:00 2001 From: Christine Poerschke Date: Wed, 22 Dec 2021 13:40:36 +0000 Subject: [PATCH 12/13] adjust SolrIndexConfigTest.testIndexSortSolrIndexConfigCreation() after origin/main merge --- .../src/test/org/apache/solr/update/SolrIndexConfigTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java b/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java index 7b9928fd8af..c9b7287e816 100644 --- a/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java +++ b/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java @@ -159,7 +159,7 @@ public void testIndexSortSolrIndexConfigCreation() throws Exception { final boolean expectedFieldSortDescending = true; SolrConfig solrConfig = new SolrConfig(instanceDir, solrConfigFileNameIndexSort); - SolrIndexConfig solrIndexConfig = new SolrIndexConfig(solrConfig, null, null); + SolrIndexConfig solrIndexConfig = new SolrIndexConfig(solrConfig, null); assertNotNull(solrIndexConfig); IndexSchema indexSchema = IndexSchemaFactory.buildIndexSchema(schemaFileName, solrConfig); From eae8527b3bd6469348e18d742021d47c9e57b12d Mon Sep 17 00:00:00 2001 From: Christine Poerschke Date: Fri, 27 May 2022 19:53:34 +0100 Subject: [PATCH 13/13] ./gradlew spotlessApply --- .../src/java/org/apache/solr/update/SolrIndexConfig.java | 3 ++- .../test/org/apache/solr/cloud/TestSegmentSorting.java | 8 +++++--- .../test/org/apache/solr/update/SolrIndexConfigTest.java | 3 ++- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java b/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java index e9b3dfe28b3..d22f5b07cf0 100644 --- a/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java +++ b/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java @@ -280,7 +280,8 @@ public IndexWriterConfig toIndexWriterConfig(SolrCore core) throws IOException { Sort mergeSort = ((SortingMergePolicy) mergePolicy).getSort(); Sort indexSort = iwc.getIndexSort(); if (indexSort != null && !indexSort.equals(mergeSort)) { - log.warn("indexSort={} differs from mergePolicySort={} (using indexSort)", indexSort, mergeSort); + log.warn( + "indexSort={} differs from mergePolicySort={} (using indexSort)", indexSort, mergeSort); } else { iwc.setIndexSort(mergeSort); } diff --git a/solr/core/src/test/org/apache/solr/cloud/TestSegmentSorting.java b/solr/core/src/test/org/apache/solr/cloud/TestSegmentSorting.java index cee5f6d78b7..fdc389f20e4 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestSegmentSorting.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestSegmentSorting.java @@ -65,11 +65,13 @@ public void createCollection() throws Exception { final String collectionName = testName.getMethodName(); final CloudSolrClient cloudSolrClient = cluster.getSolrClient(); - final String solrConfigFileName = random().nextBoolean() ? "solrconfig-sortingmergepolicyfactory.xml" : "solrconfig-indexSort.xml"; + final String solrConfigFileName = + random().nextBoolean() + ? "solrconfig-sortingmergepolicyfactory.xml" + : "solrconfig-indexSort.xml"; final Map collectionProperties = new HashMap<>(); - collectionProperties.put( - CoreDescriptor.CORE_CONFIG, solrConfigFileName); + collectionProperties.put(CoreDescriptor.CORE_CONFIG, solrConfigFileName); CollectionAdminRequest.Create cmd = CollectionAdminRequest.createCollection( diff --git a/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java b/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java index 285d9cf56eb..f9aa8315f83 100644 --- a/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java +++ b/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java @@ -176,7 +176,8 @@ public void testIndexSortSolrIndexConfigCreation() throws Exception { h.getCore().setLatestSchema(indexSchema); IndexWriterConfig iwc = solrIndexConfig.toIndexWriterConfig(h.getCore()); - final Sort expected = new Sort(new SortField(expectedFieldName, expectedFieldType, expectedFieldSortDescending)); + final Sort expected = + new Sort(new SortField(expectedFieldName, expectedFieldType, expectedFieldSortDescending)); assertEquals("indexSort", expected, iwc.getIndexSort()); }