Fix flaky ThreadPoolMergeSchedulerStressTestIT (#139819) (#140343)

* Increase wait time for flaky test

Increasing maxWaitTime for merges to start and complete to 10 minutes.
The previous time out of 1 minute leave room for scheduling variability
to tip the test over the wait threshold and fail the test. If test
stills fails on 10 minute timeout we can be quite sure that it hangs.

* Bind node processors setting in test

Some CI environments only have 4 cores and will fail this test if rnd
gods are grumpy and generate a 5 or higher. NODE_PROCESSORS_SETTING is
limited to the number of available processors.

Author: burqen

server/src/internalClusterTest/java/org/elasticsearch/index/engine/ThreadPoolMergeSchedulerStressTestIT.java

@@ -62,7 +62,10 @@ protected Settings nodeSettings() {
             .put(ThreadPoolMergeScheduler.USE_THREAD_POOL_MERGE_SCHEDULER_SETTING.getKey(), true)
             // when there are more threads than scheduler(s)' concurrency capacity, excess merges will be backlogged
             // alternatively, when scheduler(s)' concurrency capacity exceeds the executor's thread count, excess merges will be enqueued
-            .put(EsExecutors.NODE_PROCESSORS_SETTING.getKey(), MERGE_SCHEDULER_MAX_CONCURRENCY + randomFrom(-2, -1, 0, 1, 2))
+            .put(
+                EsExecutors.NODE_PROCESSORS_SETTING.getKey(),
+                Math.min(MERGE_SCHEDULER_MAX_CONCURRENCY + randomFrom(-2, -1, 0, 1, 2), Runtime.getRuntime().availableProcessors())
+            )
             .build();
     }
 
@@ -87,6 +90,7 @@ class TestInternalEngine extends org.elasticsearch.index.engine.InternalEngine {
                 super(engineConfig);
             }
 
+            @Override
             protected ElasticsearchMergeScheduler createMergeScheduler(
                 ShardId shardId,
                 IndexSettings indexSettings,
@@ -255,7 +259,7 @@ public void testMergingFallsBehindAndThenCatchesUp() throws Exception {
         assertBusy(() -> {
             // wait for merges to enqueue or backlog
             assertThat(testEnginePlugin.enqueuedMergesSet.size(), greaterThanOrEqualTo(testEnginePlugin.waitMergesEnqueuedCount));
-        }, 1, TimeUnit.MINUTES);
+        }, 10, TimeUnit.MINUTES);
         // finish up indexing
         indexingDone.set(true);
         for (Thread indexingThread : indexingThreads) {
@@ -271,7 +275,7 @@ public void testMergingFallsBehindAndThenCatchesUp() throws Exception {
             assertThat(testEnginePlugin.runningMergesSet.size(), is(0));
             assertThat(testEnginePlugin.enqueuedMergesSet.size(), is(0));
             testEnginePlugin.mergeExecutorServiceReference.get().allDone();
-        }, 1, TimeUnit.MINUTES);
+        }, 10, TimeUnit.MINUTES);
         // indices stats says that no merge is currently running (meaning merging did catch up)
         IndicesStatsResponse indicesStatsResponse = client().admin().indices().prepareStats("index").setMerge(true).get();
         long currentMergeCount = indicesStatsResponse.getIndices().get("index").getPrimaries().merge.getCurrent();