diff --git a/solr/benchmark/src/java/org/apache/solr/bench/SolrBenchState.java b/solr/benchmark/src/java/org/apache/solr/bench/SolrBenchState.java index 1aade6389ddc..0a7e9b1f2bec 100755 --- a/solr/benchmark/src/java/org/apache/solr/bench/SolrBenchState.java +++ b/solr/benchmark/src/java/org/apache/solr/bench/SolrBenchState.java @@ -47,6 +47,7 @@ import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.IOUtils; import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.ObjectReleaseTracker; import org.apache.solr.common.util.SolrNamedThreadFactory; import org.apache.solr.common.util.SuppressForbidden; import org.apache.solr.embedded.JettySolrRunner; @@ -154,6 +155,11 @@ public void shutdownSolr(BenchmarkParams benchmarkParams, BaseBenchState baseBen IOUtils.closeQuietly(client); cluster.shutdown(); logClusterDirectorySize(); + + String orr = ObjectReleaseTracker.clearObjectTrackerAndCheckEmpty(); + if (orr != null) { + throw new AssertionError("ObjectReleaseTracker found unreleased objects:\n" + orr); + } } private void logClusterDirectorySize() throws IOException { diff --git a/solr/benchmark/src/java/org/apache/solr/bench/generators/IntegersDSL.java b/solr/benchmark/src/java/org/apache/solr/bench/generators/IntegersDSL.java index 6a41c10c317b..89e6d3ad24fb 100644 --- a/solr/benchmark/src/java/org/apache/solr/bench/generators/IntegersDSL.java +++ b/solr/benchmark/src/java/org/apache/solr/bench/generators/IntegersDSL.java @@ -116,7 +116,7 @@ private static class IntegerMaxCardinalitySolrGen extends SolrGen { private final Gen integers; /** The Cardinality start. */ - Integer cardinalityStart; + volatile Integer cardinalityStart; /** * Instantiates a new Integer max cardinality solr gen. @@ -132,13 +132,18 @@ public IntegerMaxCardinalitySolrGen(int maxCardinality, Gen integers) { @Override public Integer generate(SolrRandomnessSource in) { - if (cardinalityStart == null) { - cardinalityStart = - SolrGenerate.range(0, Integer.MAX_VALUE - maxCardinality - 1).generate(in); + Integer localStart = cardinalityStart; + if (localStart == null) { + synchronized (this) { + localStart = cardinalityStart; + if (localStart == null) { + localStart = SolrGenerate.range(0, Integer.MAX_VALUE - maxCardinality - 1).generate(in); + cardinalityStart = localStart; + } + } } - long seed = - SolrGenerate.range(cardinalityStart, cardinalityStart + maxCardinality - 1).generate(in); + long seed = SolrGenerate.range(localStart, localStart + maxCardinality - 1).generate(in); return integers.generate(new SplittableRandomSource(new SplittableRandom(seed))); } } diff --git a/solr/benchmark/src/java/org/apache/solr/bench/search/FilterCache.java b/solr/benchmark/src/java/org/apache/solr/bench/search/FilterCache.java index f736aa4bf802..cf90548f2960 100644 --- a/solr/benchmark/src/java/org/apache/solr/bench/search/FilterCache.java +++ b/solr/benchmark/src/java/org/apache/solr/bench/search/FilterCache.java @@ -19,9 +19,11 @@ import static org.apache.solr.bench.generators.SourceDSL.integers; import java.io.IOException; -import java.net.HttpURLConnection; import java.net.URI; -import java.nio.charset.StandardCharsets; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.util.stream.Collectors; import org.apache.solr.bench.BaseBenchState; import org.apache.solr.bench.Docs; import org.apache.solr.bench.SolrBenchState; @@ -115,19 +117,21 @@ public void setupIteration(SolrBenchState solrBenchState) public void dumpMetrics(SolrBenchState solrBenchState) { // TODO add a verbose flag - String url = - solrBenchState.nodes.get(0) - + "/admin/metrics?prefix=CACHE.searcher.filterCache&omitHeader=true"; - HttpURLConnection conn = null; - try { - conn = (HttpURLConnection) URI.create(url).toURL().openConnection(); - conn.connect(); - BaseBenchState.log( - new String(conn.getInputStream().readAllBytes(), StandardCharsets.UTF_8)); - } catch (IOException e) { - // ignored - } finally { - if (conn != null) conn.disconnect(); + String url = solrBenchState.nodes.getFirst() + "/admin/metrics?category=CACHE?wt=prometheus"; + try (HttpClient client = HttpClient.newHttpClient()) { + HttpRequest request = HttpRequest.newBuilder().uri(URI.create(url)).GET().build(); + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + + // Filter to only lines containing filterCache metrics + String filteredMetrics = + response + .body() + .lines() + .filter(line -> line.contains("filterCache")) + .collect(Collectors.joining("\n")); + BaseBenchState.log(filteredMetrics); + } catch (IOException | InterruptedException e) { + throw new RuntimeException(e); } } } diff --git a/solr/benchmark/src/java/org/apache/solr/bench/search/QueryResponseWriters.java b/solr/benchmark/src/java/org/apache/solr/bench/search/QueryResponseWriters.java index 6929d2810a57..6c804c60afc0 100644 --- a/solr/benchmark/src/java/org/apache/solr/bench/search/QueryResponseWriters.java +++ b/solr/benchmark/src/java/org/apache/solr/bench/search/QueryResponseWriters.java @@ -22,6 +22,8 @@ import static org.apache.solr.bench.generators.SourceDSL.strings; import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import org.apache.solr.bench.Docs; import org.apache.solr.bench.SolrBenchState; import org.apache.solr.client.solrj.SolrServerException; @@ -29,6 +31,7 @@ import org.apache.solr.client.solrj.response.InputStreamResponseParser; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.util.NamedList; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.Fork; @@ -56,7 +59,7 @@ public class QueryResponseWriters { @State(Scope.Benchmark) public static class BenchState { - @Param({CommonParams.JAVABIN, CommonParams.JSON, "cbor", "smile", "xml", "raw"}) + @Param({CommonParams.JAVABIN, CommonParams.JSON, "cbor", "smile", "xml"}) String wt; private int docs = 100; @@ -91,6 +94,12 @@ public void setup(SolrBenchState solrBenchState) throws Exception { @Benchmark public Object query(BenchState benchState, SolrBenchState solrBenchState) throws SolrServerException, IOException { - return solrBenchState.client.request(benchState.q, collection); + NamedList response = solrBenchState.client.request(benchState.q, collection); + // consume the stream completely + try (InputStream responseStream = + (InputStream) response.get(InputStreamResponseParser.STREAM_KEY)) { + responseStream.transferTo(OutputStream.nullOutputStream()); + } + return response; } } diff --git a/solr/benchmark/src/java/org/apache/solr/bench/search/StreamingSearch.java b/solr/benchmark/src/java/org/apache/solr/bench/search/StreamingSearch.java index c354377eaf2f..5797682c63e5 100644 --- a/solr/benchmark/src/java/org/apache/solr/bench/search/StreamingSearch.java +++ b/solr/benchmark/src/java/org/apache/solr/bench/search/StreamingSearch.java @@ -25,6 +25,7 @@ import java.util.List; import org.apache.solr.bench.Docs; import org.apache.solr.bench.SolrBenchState; +import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.io.SolrClientCache; import org.apache.solr.client.solrj.io.Tuple; @@ -34,6 +35,7 @@ import org.apache.solr.client.solrj.jetty.HttpJettySolrClient; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.util.IOUtils; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.Fork; @@ -67,7 +69,7 @@ public static class BenchState { private String zkHost; private ModifiableSolrParams params; private StreamContext streamContext; - private HttpJettySolrClient httpJettySolrClient; + private SolrClient httpSolrClient; @Setup(Level.Trial) public void setup(SolrBenchState solrBenchState) throws Exception { @@ -95,11 +97,9 @@ public void setup(SolrBenchState solrBenchState) throws Exception { @Setup(Level.Iteration) public void setupIteration(SolrBenchState solrBenchState) throws SolrServerException, IOException { - SolrClientCache solrClientCache; - // TODO tune params? - var client = new HttpJettySolrClient.Builder().useHttp1_1(useHttp1).build(); - solrClientCache = new SolrClientCache(client); - + var httpSolrClient = new HttpJettySolrClient.Builder().useHttp1_1(useHttp1).build(); + this.httpSolrClient = httpSolrClient; + SolrClientCache solrClientCache = new SolrClientCache(httpSolrClient); streamContext = new StreamContext(); streamContext.setSolrClientCache(solrClientCache); } @@ -107,9 +107,7 @@ public void setupIteration(SolrBenchState solrBenchState) @TearDown(Level.Iteration) public void teardownIt() { streamContext.getSolrClientCache().close(); - if (httpJettySolrClient != null) { - httpJettySolrClient.close(); - } + IOUtils.closeQuietly(httpSolrClient); } } diff --git a/solr/benchmark/src/resources/configs/cloud-minimal/conf/schema.xml b/solr/benchmark/src/resources/configs/cloud-minimal/conf/schema.xml index e517aea59307..b1851c3c2906 100644 --- a/solr/benchmark/src/resources/configs/cloud-minimal/conf/schema.xml +++ b/solr/benchmark/src/resources/configs/cloud-minimal/conf/schema.xml @@ -36,7 +36,7 @@ - +