feat(benchmarking): add optional QUERIES custom variable

Author: pflanze

benchmarking/Makefile_api

@@ -12,6 +12,7 @@ API_QUERY_OPTS=--randomize
 else
 $(error "RANDOMIZED must be either 0 or 1, got '$(RANDOMIZED)'")
 endif
+export API_QUERY_OPTS
 
 # Passed to api-query
 ifeq ($(CONCURRENCY),)
@@ -23,6 +24,13 @@ ifeq ($(REPEAT),)
 $(error "REPEAT must be a natural number, missing")
 endif
 
+ifeq ($(QUERIES),)
+QUERIES_DIR=$(DATASET_DIR)/queries
+else
+QUERIES_DIR=$(DATASET_DIR)/$(QUERIES)
+endif
+export QUERIES_DIR
+
 # ---- Get dependencies --------------------------------------------------------
 
 API_QUERY_VERSION=e005495f09b76e8c418a43cc07bf2f398e4c48bd
@@ -32,8 +40,10 @@ API_QUERY_CLONE=$(API_QUERY_DIR)/Cargo.toml
 API_QUERY_CHECKOUT_STAMP_DIR=$(API_QUERY_DIR)/.git/.checkout
 API_QUERY_CHECKOUT=$(API_QUERY_CHECKOUT_STAMP_DIR)/$(API_QUERY_VERSION)
 API_QUERY=$(API_QUERY_DIR)/target/release/api-query
+export API_QUERY
 
 API_QUERY_LOG_CSV:=$(OUTPUT_DIR)/api-query-log-$(shell bin/date-rfc-3339).csv
+export API_QUERY_LOG_CSV
 
 $(API_QUERY_CLONE):
 	git clone https://github.com/GenSpectrum/api-query $(API_QUERY_DIR)
@@ -61,9 +71,9 @@ dependencies: $(API_QUERY)
 	bin/stop-silo
 	touch .silo.stopped
 
-bench: $(DATASET_DIR)/silo_queries.ndjson $(API_QUERY) .silo.stopped
+bench: $(QUERIES_DIR)/queries.ndjson $(API_QUERY) .silo.stopped
 	@echo "running benchmark"
 	make -f Makefile_api .silo.pid
-	$(API_QUERY) iter $(API_QUERY_OPTS) --repeat $(REPEAT) $(DATASET_DIR)/silo_queries.ndjson --drop --log-csv $(API_QUERY_LOG_CSV) --concurrency $(CONCURRENCY)
+	bin/api-query-iter
 	make -f Makefile_api .silo.stopped
-	$(API_QUERY)-log compare --queries $(DATASET_DIR)/silo_queries.ndjson --ignore-from $(DATASET_DIR)/ignore_queries_for_checksum_regex.txt $(DATASET_DIR)/good-api-query-log.csv $(API_QUERY_LOG_CSV)
+	bin/api-query-log-compare

benchmarking/bin/api-query-iter

@@ -0,0 +1,18 @@
+#!/bin/bash
+set -meuo pipefail
+IFS=' '
+
+if [ $# -ne 0 ]; then
+    echo "usage: all parameters must be passed as env variables"
+    exit 1
+fi
+
+set -x
+
+"$API_QUERY" iter \
+             $API_QUERY_OPTS \
+             --repeat "$REPEAT" \
+             --concurrency "$CONCURRENCY" \
+             "$QUERIES_DIR"/queries.ndjson \
+             --drop --log-csv "$API_QUERY_LOG_CSV"
+

benchmarking/bin/api-query-log-compare

@@ -0,0 +1,21 @@
+#!/bin/bash
+set -meuo pipefail
+IFS=
+
+if [ $# -ne 0 ]; then
+    echo "usage: all parameters must be passed as env variables"
+    exit 1
+fi
+
+if [ -e "$QUERIES_DIR"/good-api-query-log.csv ]; then
+    set -x
+    "$API_QUERY"-log compare \
+                --queries "$QUERIES_DIR"/queries.ndjson \
+                --ignore-from "$QUERIES_DIR"/ignore_queries_for_checksum_regex.txt \
+                "$QUERIES_DIR"/good-api-query-log.csv "$API_QUERY_LOG_CSV"
+else
+    echo -n "There's no good-api-query-log.csv for '$QUERIES'. "
+    echo "Automatically install the current file as presumably good:"
+    set -x
+    cp -a "$API_QUERY_LOG_CSV" "$QUERIES_DIR"/good-api-query-log.csv
+fi