Benchmarking: add QUERIES custom variable

benchmarking/Makefile_api

@@ -12,6 +12,7 @@ API_QUERY_OPTS=--randomize
 else
 $(error "RANDOMIZED must be either 0 or 1, got '$(RANDOMIZED)'")
 endif
+export API_QUERY_OPTS
 
 # Passed to api-query
 ifeq ($(CONCURRENCY),)
@@ -23,17 +24,26 @@ ifeq ($(REPEAT),)
 $(error "REPEAT must be a natural number, missing")
 endif
 
+ifeq ($(QUERIES),)
+QUERIES_DIR=$(DATASET_DIR)/queries
+else
+QUERIES_DIR=$(DATASET_DIR)/$(QUERIES)
+endif
+export QUERIES_DIR
+
 # ---- Get dependencies --------------------------------------------------------
 
-API_QUERY_VERSION=f28f644b54427575319a6e98f668fc023824b303
+API_QUERY_VERSION=e005495f09b76e8c418a43cc07bf2f398e4c48bd
 
 API_QUERY_DIR=api-query
 API_QUERY_CLONE=$(API_QUERY_DIR)/Cargo.toml
 API_QUERY_CHECKOUT_STAMP_DIR=$(API_QUERY_DIR)/.git/.checkout
 API_QUERY_CHECKOUT=$(API_QUERY_CHECKOUT_STAMP_DIR)/$(API_QUERY_VERSION)
 API_QUERY=$(API_QUERY_DIR)/target/release/api-query
+export API_QUERY
 
 API_QUERY_LOG_CSV:=$(OUTPUT_DIR)/api-query-log-$(shell bin/date-rfc-3339).csv
+export API_QUERY_LOG_CSV
 
 $(API_QUERY_CLONE):
 	git clone https://github.com/GenSpectrum/api-query $(API_QUERY_DIR)
@@ -61,9 +71,9 @@ dependencies: $(API_QUERY)
 	bin/stop-silo
 	touch .silo.stopped
 
-bench: $(DATASET_DIR)/silo_queries.ndjson $(API_QUERY) .silo.stopped
+bench: $(QUERIES_DIR)/queries.ndjson $(API_QUERY) .silo.stopped
 	@echo "running benchmark"
 	make -f Makefile_api .silo.pid
-	$(API_QUERY) iter $(API_QUERY_OPTS) --repeat $(REPEAT) $(DATASET_DIR)/silo_queries.ndjson --drop --log-csv $(API_QUERY_LOG_CSV) --concurrency $(CONCURRENCY)
+	bin/api-query-iter
 	make -f Makefile_api .silo.stopped
-	$(API_QUERY)-log compare --queries $(DATASET_DIR)/silo_queries.ndjson --ignore-from $(DATASET_DIR)/ignore_queries_for_checksum_regex.txt $(DATASET_DIR)/good-api-query-log.csv $(API_QUERY_LOG_CSV)
+	bin/api-query-log-compare

benchmarking/README.md

@@ -39,13 +39,26 @@ a server on
  1. Make sure that you have dataset folders as configured in the evobench config file (`evobench-run.ron`); see [Offer dataset versioning #32](https://github.com/GenSpectrum/evobench/issues/32) for the required folder structure. Each versioned dataset folder needs these files (or symlinks to them):
 
         database_config.yaml
-        good-api-query-log.csv
-        ignore_queries_for_checksum_regex.txt
         input_file.ndjson.zst
         possibly: lineage_definitions.yaml
         preprocessing_config.yaml
+        queries/ignore_queries_for_checksum_regex.txt
+        queries/queries.ndjson
         reference_genomes.json
-        silo_queries.ndjson
+
+    Note that a file `queries/good-api-query-log.csv` is automatically
+    stored upon the first benchmarking run; it is assumed to contain
+    the proper checksums. Later runs compare against that file and
+    flag differences that are not ignored via the regex in
+    `ignore_queries_for_checksum_regex.txt` (leave empty to not ignore
+    anything) as errors. If that happens, you are to check which
+    version is legit, or ignore those queries via the regex. The regex
+    syntax is the one supported by [Rust's regex
+    crate](https://docs.rs/regex/1.12.2/regex/).
+
+    If you use the `QUERIES` evobench custom variable (i.e. env
+    variable), which names a subfolder, you need to provide a
+    corresponding alternative folder to the `queries/` folder.
 
  1. Run an instance of a daemon, `evobench-run --verbose run daemon`
     (the `--verbose` allows you to see what's going on, feel free to

benchmarking/bin/api-query-iter

@@ -0,0 +1,18 @@
+#!/bin/bash
+set -meuo pipefail
+IFS=' '
+
+if [ $# -ne 0 ]; then
+    echo "usage: all parameters must be passed as env variables"
+    exit 1
+fi
+
+set -x
+
+"$API_QUERY" iter \
+             $API_QUERY_OPTS \
+             --repeat "$REPEAT" \
+             --concurrency "$CONCURRENCY" \
+             "$QUERIES_DIR"/queries.ndjson \
+             --drop --log-csv "$API_QUERY_LOG_CSV"
+

benchmarking/bin/api-query-log-compare

@@ -0,0 +1,21 @@
+#!/bin/bash
+set -meuo pipefail
+IFS=
+
+if [ $# -ne 0 ]; then
+    echo "usage: all parameters must be passed as env variables"
+    exit 1
+fi
+
+if [ -e "$QUERIES_DIR"/good-api-query-log.csv ]; then
+    set -x
+    "$API_QUERY"-log compare \
+                --queries "$QUERIES_DIR"/queries.ndjson \
+                --ignore-from "$QUERIES_DIR"/ignore_queries_for_checksum_regex.txt \
+                "$QUERIES_DIR"/good-api-query-log.csv "$API_QUERY_LOG_CSV"
+else
+    echo -n "There's no good-api-query-log.csv for '$QUERIES'. "
+    echo "Automatically install the current file as presumably good:"
+    set -x
+    cp -a "$API_QUERY_LOG_CSV" "$QUERIES_DIR"/good-api-query-log.csv
+fi