From c5dec5bb1b279f914cfdca68fa47de4ca587478e Mon Sep 17 00:00:00 2001 From: Nikola Grcevski Date: Mon, 16 Dec 2019 16:46:40 -0500 Subject: [PATCH 1/3] Fix socket leak in benchmark client. (cherry picked from commit 9ac4cca0ccc561f8e1bdc58ea31f7a39924612df) --- .../org/apache/pinotdruidbenchmark/PinotThroughput.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/contrib/pinot-druid-benchmark/src/main/java/org/apache/pinotdruidbenchmark/PinotThroughput.java b/contrib/pinot-druid-benchmark/src/main/java/org/apache/pinotdruidbenchmark/PinotThroughput.java index 1ac1bf255ba3..3d7c3ee8c038 100644 --- a/contrib/pinot-druid-benchmark/src/main/java/org/apache/pinotdruidbenchmark/PinotThroughput.java +++ b/contrib/pinot-druid-benchmark/src/main/java/org/apache/pinotdruidbenchmark/PinotThroughput.java @@ -33,7 +33,7 @@ import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; - +import org.apache.http.util.EntityUtils; /** * Test throughput for Pinot. @@ -90,8 +90,9 @@ public void run() { try (CloseableHttpClient httpClient = HttpClients.createDefault()) { while (System.currentTimeMillis() < endTime) { long startTime = System.currentTimeMillis(); - CloseableHttpResponse httpResponse = httpClient.execute(httpPosts[RANDOM.nextInt(numQueries)]); - httpResponse.close(); + try (CloseableHttpResponse httpResponse = httpClient.execute(httpPosts[RANDOM.nextInt(numQueries)])) { + EntityUtils.consume(httpResponse.getEntity()); + } long responseTime = System.currentTimeMillis() - startTime; counter.getAndIncrement(); totalResponseTime.getAndAdd(responseTime); From d79d7e88a739e131b16bbcefb8b2a643128620ff Mon Sep 17 00:00:00 2001 From: Nikola Grcevski Date: Mon, 16 Dec 2019 16:47:02 -0500 Subject: [PATCH 2/3] Fix issue with benchmark server boot (cherry picked from commit 9c55f710f8547fa09505c3458b47fc27a47f051a) --- .../pinot/controller/helix/core/PinotHelixResourceManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java index 5bff3ee83cac..04c969a544b5 100644 --- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java +++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java @@ -264,7 +264,7 @@ public ZkHelixPropertyStore getPropertyStore() { private void addInstanceGroupTagIfNeeded() { InstanceConfig instanceConfig = getHelixInstanceConfig(_instanceId); assert instanceConfig != null; - if (!instanceConfig.containsTag(Helix.CONTROLLER_INSTANCE)) { + if (instanceConfig != null && !instanceConfig.containsTag(Helix.CONTROLLER_INSTANCE)) { LOGGER.info("Controller: {} doesn't contain group tag: {}. Adding one.", _instanceId, Helix.CONTROLLER_INSTANCE); instanceConfig.addTag(Helix.CONTROLLER_INSTANCE); HelixDataAccessor accessor = _helixZkManager.getHelixDataAccessor(); From 2eb1e6bb1affc2bfe29962fcce138495d21cee81 Mon Sep 17 00:00:00 2001 From: Nikola Grcevski Date: Tue, 28 Jan 2020 12:22:18 -0500 Subject: [PATCH 3/3] Add comment about instanceConfig and fix benchmark driver server name configuration. (cherry picked from commit b0dcfbfe5c9bdc5628fbf854fc8dade4c526eb0e) --- .../pinot/controller/helix/core/PinotHelixResourceManager.java | 2 +- .../java/org/apache/pinot/tools/perf/PerfBenchmarkDriver.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java index 04c969a544b5..584c68eedbd3 100644 --- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java +++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java @@ -263,7 +263,7 @@ public ZkHelixPropertyStore getPropertyStore() { */ private void addInstanceGroupTagIfNeeded() { InstanceConfig instanceConfig = getHelixInstanceConfig(_instanceId); - assert instanceConfig != null; + // The instanceConfig can be null when connecting as a participant while running from PerfBenchmarkRunner if (instanceConfig != null && !instanceConfig.containsTag(Helix.CONTROLLER_INSTANCE)) { LOGGER.info("Controller: {} doesn't contain group tag: {}. Adding one.", _instanceId, Helix.CONTROLLER_INSTANCE); instanceConfig.addTag(Helix.CONTROLLER_INSTANCE); diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/perf/PerfBenchmarkDriver.java b/pinot-tools/src/main/java/org/apache/pinot/tools/perf/PerfBenchmarkDriver.java index 4240e7cb0056..a1b493bdc831 100644 --- a/pinot-tools/src/main/java/org/apache/pinot/tools/perf/PerfBenchmarkDriver.java +++ b/pinot-tools/src/main/java/org/apache/pinot/tools/perf/PerfBenchmarkDriver.java @@ -238,7 +238,7 @@ private void startServer() if (_segmentFormatVersion != null) { serverConfiguration.setProperty(CommonConstants.Server.CONFIG_OF_SEGMENT_FORMAT_VERSION, _segmentFormatVersion); } - serverConfiguration.setProperty(CommonConstants.Helix.Instance.INSTANCE_ID_KEY, _serverInstanceName); + serverConfiguration.setProperty(CommonConstants.Server.CONFIG_OF_INSTANCE_ID, _serverInstanceName); LOGGER.info("Starting server instance: {}", _serverInstanceName); new HelixServerStarter(_clusterName, _zkAddress, serverConfiguration); }