diff --git a/hbase-diagnostics/src/main/java/org/apache/hadoop/hbase/PerformanceEvaluation.java b/hbase-diagnostics/src/main/java/org/apache/hadoop/hbase/PerformanceEvaluation.java index f2e3a9ac1b72..3af33f8a834e 100644 --- a/hbase-diagnostics/src/main/java/org/apache/hadoop/hbase/PerformanceEvaluation.java +++ b/hbase-diagnostics/src/main/java/org/apache/hadoop/hbase/PerformanceEvaluation.java @@ -22,6 +22,7 @@ import io.opentelemetry.api.trace.Span; import io.opentelemetry.context.Scope; import java.io.IOException; +import java.io.InputStream; import java.io.PrintStream; import java.lang.reflect.Constructor; import java.math.BigDecimal; @@ -31,6 +32,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Date; +import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Locale; @@ -46,6 +48,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.ThreadLocalRandom; +import java.util.function.Consumer; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; @@ -113,6 +116,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hbase.thirdparty.com.google.common.base.Splitter; import org.apache.hbase.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.hbase.thirdparty.com.google.gson.Gson; @@ -2843,9 +2847,79 @@ protected static void printUsage(final String shortName, final String message) { * arguments. */ static TestOptions parseOpts(Queue args) { - TestOptions opts = new TestOptions(); + final TestOptions opts = new TestOptions(); + + final Map> flagHandlers = new HashMap<>(); + flagHandlers.put("nomapred", v -> opts.nomapred = v); + flagHandlers.put("flushCommits", v -> opts.flushCommits = v); + flagHandlers.put("writeToWAL", v -> opts.writeToWAL = v); + flagHandlers.put("inmemory", v -> opts.inMemoryCF = v); + flagHandlers.put("autoFlush", v -> opts.autoFlush = v); + flagHandlers.put("oneCon", v -> opts.oneCon = v); + flagHandlers.put("latency", v -> opts.reportLatency = v); + flagHandlers.put("usetags", v -> opts.useTags = v); + flagHandlers.put("filterAll", v -> opts.filterAll = v); + flagHandlers.put("valueRandom", v -> opts.valueRandom = v); + flagHandlers.put("valueZipf", v -> opts.valueZipf = v); + flagHandlers.put("addColumns", v -> opts.addColumns = v); + flagHandlers.put("asyncPrefetch", v -> opts.asyncPrefetch = v); + flagHandlers.put("cacheBlocks", v -> opts.cacheBlocks = v); + + final Map> handlers = new HashMap<>(); + handlers.put("rows", v -> opts.perClientRunRows = Long.parseLong(v)); + handlers.put("cycles", v -> opts.cycles = Integer.parseInt(v)); + handlers.put("sampleRate", v -> opts.sampleRate = Float.parseFloat(v)); + handlers.put("table", v -> opts.tableName = v); + handlers.put("startRow", v -> opts.startRow = Long.parseLong(v)); + handlers.put("compress", v -> opts.compression = Compression.Algorithm.valueOf(v)); + handlers.put("encryption", v -> opts.encryption = v); + handlers.put("traceRate", v -> opts.traceRate = Double.parseDouble(v)); + handlers.put("blockEncoding", v -> opts.blockEncoding = DataBlockEncoding.valueOf(v)); + handlers.put("presplit", v -> opts.presplitRegions = Integer.parseInt(v)); + handlers.put("connCount", v -> opts.connCount = Integer.parseInt(v)); + handlers.put("latencyThreshold", v -> opts.latencyThreshold = Integer.parseInt(v)); + handlers.put("multiGet", v -> opts.multiGet = Integer.parseInt(v)); + handlers.put("multiPut", v -> opts.multiPut = Integer.parseInt(v)); + handlers.put("numoftags", v -> opts.noOfTags = Integer.parseInt(v)); + handlers.put("replicas", v -> opts.replicas = Integer.parseInt(v)); + handlers.put("size", v -> { + opts.size = Float.parseFloat(v); + if (opts.size <= 1.0f) { + throw new IllegalStateException("Size must be > 1; i.e. 1GB"); + } + }); + handlers.put("splitPolicy", v -> opts.splitPolicy = v); + handlers.put("randomSleep", v -> opts.randomSleep = Integer.parseInt(v)); + handlers.put("measureAfter", v -> opts.measureAfter = Integer.parseInt(v)); + handlers.put("bloomFilter", v -> opts.bloomType = BloomType.valueOf(v)); + handlers.put("blockSize", v -> opts.blockSize = Integer.parseInt(v)); + handlers.put("valueSize", v -> opts.valueSize = Integer.parseInt(v)); + handlers.put("period", v -> opts.period = Integer.parseInt(v)); + handlers.put("inmemoryCompaction", + v -> opts.inMemoryCompaction = MemoryCompactionPolicy.valueOf(v)); + handlers.put("columns", v -> opts.columns = Integer.parseInt(v)); + handlers.put("families", v -> opts.families = Integer.parseInt(v)); + handlers.put("caching", v -> opts.caching = Integer.parseInt(v)); + handlers.put("scanReadType", v -> opts.scanReadType = Scan.ReadType.valueOf(v.toUpperCase())); + handlers.put("bufferSize", v -> opts.bufferSize = Long.parseLong(v)); + handlers.put("commandPropertiesFile", fileName -> { + try { + final Properties properties = new Properties(); + final InputStream resourceStream = + PerformanceEvaluation.class.getClassLoader().getResourceAsStream(fileName); + if (resourceStream == null) { + throw new IllegalArgumentException("Resource file not found: " + fileName); + } + properties.load(resourceStream); + opts.commandProperties = properties; + } catch (IOException e) { + throw new IllegalStateException("Failed to load command properties from file: " + fileName, + e); + } + }); String cmd = null; + final Splitter splitter = Splitter.on("=").limit(2).trimResults(); while ((cmd = args.poll()) != null) { if (cmd.equals("-h") || cmd.startsWith("--h")) { // place item back onto queue so that caller knows parsing was incomplete @@ -2853,289 +2927,32 @@ static TestOptions parseOpts(Queue args) { break; } - final String nmr = "--nomapred"; - if (cmd.startsWith(nmr)) { - opts.nomapred = true; - continue; - } - - final String rows = "--rows="; - if (cmd.startsWith(rows)) { - opts.perClientRunRows = Long.parseLong(cmd.substring(rows.length())); - continue; - } - - final String cycles = "--cycles="; - if (cmd.startsWith(cycles)) { - opts.cycles = Integer.parseInt(cmd.substring(cycles.length())); - continue; - } - - final String sampleRate = "--sampleRate="; - if (cmd.startsWith(sampleRate)) { - opts.sampleRate = Float.parseFloat(cmd.substring(sampleRate.length())); - continue; - } - - final String table = "--table="; - if (cmd.startsWith(table)) { - opts.tableName = cmd.substring(table.length()); - continue; - } - - final String startRow = "--startRow="; - if (cmd.startsWith(startRow)) { - opts.startRow = Long.parseLong(cmd.substring(startRow.length())); - continue; - } - - final String compress = "--compress="; - if (cmd.startsWith(compress)) { - opts.compression = Compression.Algorithm.valueOf(cmd.substring(compress.length())); - continue; - } - - final String encryption = "--encryption="; - if (cmd.startsWith(encryption)) { - opts.encryption = cmd.substring(encryption.length()); - continue; - } - - final String traceRate = "--traceRate="; - if (cmd.startsWith(traceRate)) { - opts.traceRate = Double.parseDouble(cmd.substring(traceRate.length())); - continue; - } - - final String blockEncoding = "--blockEncoding="; - if (cmd.startsWith(blockEncoding)) { - opts.blockEncoding = DataBlockEncoding.valueOf(cmd.substring(blockEncoding.length())); - continue; - } - - final String flushCommits = "--flushCommits="; - if (cmd.startsWith(flushCommits)) { - opts.flushCommits = Boolean.parseBoolean(cmd.substring(flushCommits.length())); - continue; - } - - final String writeToWAL = "--writeToWAL="; - if (cmd.startsWith(writeToWAL)) { - opts.writeToWAL = Boolean.parseBoolean(cmd.substring(writeToWAL.length())); - continue; - } - - final String presplit = "--presplit="; - if (cmd.startsWith(presplit)) { - opts.presplitRegions = Integer.parseInt(cmd.substring(presplit.length())); - continue; - } - - final String inMemory = "--inmemory="; - if (cmd.startsWith(inMemory)) { - opts.inMemoryCF = Boolean.parseBoolean(cmd.substring(inMemory.length())); - continue; - } - - final String autoFlush = "--autoFlush="; - if (cmd.startsWith(autoFlush)) { - opts.autoFlush = Boolean.parseBoolean(cmd.substring(autoFlush.length())); - continue; - } - - final String onceCon = "--oneCon="; - if (cmd.startsWith(onceCon)) { - opts.oneCon = Boolean.parseBoolean(cmd.substring(onceCon.length())); - continue; - } - - final String connCount = "--connCount="; - if (cmd.startsWith(connCount)) { - opts.connCount = Integer.parseInt(cmd.substring(connCount.length())); - continue; - } - - final String latencyThreshold = "--latencyThreshold="; - if (cmd.startsWith(latencyThreshold)) { - opts.latencyThreshold = Integer.parseInt(cmd.substring(latencyThreshold.length())); - continue; - } - - final String latency = "--latency"; - if (cmd.startsWith(latency)) { - opts.reportLatency = true; - continue; - } - - final String multiGet = "--multiGet="; - if (cmd.startsWith(multiGet)) { - opts.multiGet = Integer.parseInt(cmd.substring(multiGet.length())); - continue; - } - - final String multiPut = "--multiPut="; - if (cmd.startsWith(multiPut)) { - opts.multiPut = Integer.parseInt(cmd.substring(multiPut.length())); - continue; - } - - final String useTags = "--usetags="; - if (cmd.startsWith(useTags)) { - opts.useTags = Boolean.parseBoolean(cmd.substring(useTags.length())); - continue; - } - - final String noOfTags = "--numoftags="; - if (cmd.startsWith(noOfTags)) { - opts.noOfTags = Integer.parseInt(cmd.substring(noOfTags.length())); - continue; - } - - final String replicas = "--replicas="; - if (cmd.startsWith(replicas)) { - opts.replicas = Integer.parseInt(cmd.substring(replicas.length())); - continue; - } - - final String filterOutAll = "--filterAll"; - if (cmd.startsWith(filterOutAll)) { - opts.filterAll = true; - continue; - } + if (cmd.startsWith("--")) { + final List parts = splitter.splitToList(cmd.substring(2)); + final String key = parts.get(0); - final String size = "--size="; - if (cmd.startsWith(size)) { - opts.size = Float.parseFloat(cmd.substring(size.length())); - if (opts.size <= 1.0f) throw new IllegalStateException("Size must be > 1; i.e. 1GB"); - continue; - } - - final String splitPolicy = "--splitPolicy="; - if (cmd.startsWith(splitPolicy)) { - opts.splitPolicy = cmd.substring(splitPolicy.length()); - continue; - } - - final String randomSleep = "--randomSleep="; - if (cmd.startsWith(randomSleep)) { - opts.randomSleep = Integer.parseInt(cmd.substring(randomSleep.length())); - continue; - } - - final String measureAfter = "--measureAfter="; - if (cmd.startsWith(measureAfter)) { - opts.measureAfter = Integer.parseInt(cmd.substring(measureAfter.length())); - continue; - } - - final String bloomFilter = "--bloomFilter="; - if (cmd.startsWith(bloomFilter)) { - opts.bloomType = BloomType.valueOf(cmd.substring(bloomFilter.length())); - continue; - } - - final String blockSize = "--blockSize="; - if (cmd.startsWith(blockSize)) { - opts.blockSize = Integer.parseInt(cmd.substring(blockSize.length())); - continue; - } - - final String valueSize = "--valueSize="; - if (cmd.startsWith(valueSize)) { - opts.valueSize = Integer.parseInt(cmd.substring(valueSize.length())); - continue; - } - - final String valueRandom = "--valueRandom"; - if (cmd.startsWith(valueRandom)) { - opts.valueRandom = true; - continue; - } - - final String valueZipf = "--valueZipf"; - if (cmd.startsWith(valueZipf)) { - opts.valueZipf = true; - continue; - } - - final String period = "--period="; - if (cmd.startsWith(period)) { - opts.period = Integer.parseInt(cmd.substring(period.length())); - continue; - } - - final String addColumns = "--addColumns="; - if (cmd.startsWith(addColumns)) { - opts.addColumns = Boolean.parseBoolean(cmd.substring(addColumns.length())); - continue; - } - - final String inMemoryCompaction = "--inmemoryCompaction="; - if (cmd.startsWith(inMemoryCompaction)) { - opts.inMemoryCompaction = - MemoryCompactionPolicy.valueOf(cmd.substring(inMemoryCompaction.length())); - continue; - } - - final String columns = "--columns="; - if (cmd.startsWith(columns)) { - opts.columns = Integer.parseInt(cmd.substring(columns.length())); - continue; - } - - final String families = "--families="; - if (cmd.startsWith(families)) { - opts.families = Integer.parseInt(cmd.substring(families.length())); - continue; - } - - final String caching = "--caching="; - if (cmd.startsWith(caching)) { - opts.caching = Integer.parseInt(cmd.substring(caching.length())); - continue; - } - - final String asyncPrefetch = "--asyncPrefetch"; - if (cmd.startsWith(asyncPrefetch)) { - opts.asyncPrefetch = true; - continue; - } - - final String cacheBlocks = "--cacheBlocks="; - if (cmd.startsWith(cacheBlocks)) { - opts.cacheBlocks = Boolean.parseBoolean(cmd.substring(cacheBlocks.length())); - continue; - } - - final String scanReadType = "--scanReadType="; - if (cmd.startsWith(scanReadType)) { - opts.scanReadType = - Scan.ReadType.valueOf(cmd.substring(scanReadType.length()).toUpperCase()); - continue; - } - - final String bufferSize = "--bufferSize="; - if (cmd.startsWith(bufferSize)) { - opts.bufferSize = Long.parseLong(cmd.substring(bufferSize.length())); - continue; - } - - final String commandPropertiesFile = "--commandPropertiesFile="; - if (cmd.startsWith(commandPropertiesFile)) { - String fileName = String.valueOf(cmd.substring(commandPropertiesFile.length())); - Properties properties = new Properties(); try { - properties - .load(PerformanceEvaluation.class.getClassLoader().getResourceAsStream(fileName)); - opts.commandProperties = properties; - } catch (IOException e) { - LOG.error("Failed to load metricIds from properties file", e); + // Boolean options can be specified as --flag or --flag=true/false + final Consumer flagHandler = flagHandlers.get(key); + if (flagHandler != null) { + flagHandler.accept(parts.size() > 1 ? parseBoolean(parts.get(1)) : true); + continue; + } + + // Options that require a value followed by an equals sign + final Consumer handler = handlers.get(key); + if (handler != null) { + if (parts.size() < 2) { + throw new IllegalArgumentException("--" + key + " requires a value"); + } + handler.accept(parts.get(1)); + continue; + } + } catch (RuntimeException e) { + throw new IllegalArgumentException("Invalid option: " + cmd, e); } - continue; } - validateParsedOpts(opts); - if (isCommandClass(cmd)) { opts.cmdName = cmd; try { @@ -3143,20 +2960,28 @@ static TestOptions parseOpts(Queue args) { } catch (NoSuchElementException | NumberFormatException e) { throw new IllegalArgumentException("Command " + cmd + " does not have threads number", e); } - opts = calculateRowsAndSize(opts); + calculateRowsAndSize(opts); break; - } else { - printUsageAndExit("ERROR: Unrecognized option/command: " + cmd, -1); } - // Not matching any option or command. - System.err.println("Error: Wrong option or command: " + cmd); - args.add(cmd); - break; + printUsageAndExit("ERROR: Unrecognized option/command: " + cmd, -1); } + + validateParsedOpts(opts); return opts; } + // Boolean.parseBoolean is not strict enough for our needs, so we implement our own + private static boolean parseBoolean(String value) { + if ("true".equalsIgnoreCase(value)) { + return true; + } + if ("false".equalsIgnoreCase(value)) { + return false; + } + throw new IllegalArgumentException("Invalid boolean value: " + value); + } + /** * Validates opts after all the opts are parsed, so that caller need not to maintain order of opts */ diff --git a/hbase-diagnostics/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java b/hbase-diagnostics/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java index 91722be61642..84b211d4e317 100644 --- a/hbase-diagnostics/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java +++ b/hbase-diagnostics/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java @@ -20,6 +20,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -403,4 +404,30 @@ boolean testRow(long i, long startTime) throws IOException, InterruptedException return false; } } + + @Test + public void testParseBooleanFlags() { + final Queue opts = new LinkedList<>(); + opts.offer("--valueRandom"); + opts.offer("--autoFlush"); // default: false + opts.offer("--inmemory=true"); // default: false + opts.offer("--writeToWAL=false"); // default: true + opts.offer(PerformanceEvaluation.RANDOM_READ); + opts.offer("1"); + + final PerformanceEvaluation.TestOptions options = PerformanceEvaluation.parseOpts(opts); + assertTrue(options.valueRandom); + assertTrue(options.autoFlush); + assertTrue(options.inMemoryCF); + assertFalse(options.writeToWAL); + assertEquals(PerformanceEvaluation.RANDOM_READ, options.getCmdName()); + assertEquals(1, options.getNumClientThreads()); + } + + @Test + public void testOptionMissingValue() { + final Queue opts = new LinkedList<>(); + opts.offer("--presplit"); + assertThrows(IllegalArgumentException.class, () -> PerformanceEvaluation.parseOpts(opts)); + } }