s-b: add the error-limit flag

Implements the "error-limit" flag which was recently added in
scylladb/scylla-bench#110. The flag allows to limit the total number
of errors allowed before a run is stopped. Support for that flag
is implemented via the `max_retries_in_total` parameter added
in the previous commit.

Author: piodul

src/bin/cql-stress-scylla-bench/args.rs

@@ -43,6 +43,7 @@ pub(crate) struct ScyllaBenchArgs {
     pub mode: Mode,
     pub latency_type: LatencyType,
     pub max_retries_per_op: u64,
+    pub max_retries_in_total: u64,
     pub concurrency: u64,
     pub maximum_rate: u64,
 
@@ -178,6 +179,12 @@ where
         After exceeding it, the workflow will terminate with an error. \
         Set to 0 if you want to have unlimited retries",
     );
+    let max_errors = flag.u64_var(
+        "error-limit",
+        0,
+        "the number of total errors after which the workflow should stop and fail; \
+        set it to 0 (the default) to disable this limit",
+    );
     let concurrency = flag.u64_var("concurrency", 16, "number of used tasks");
     let maximum_rate = flag.u64_var(
         "max-rate",
@@ -332,6 +339,9 @@ where
         // therefore just subtract with wraparound and treat u64::MAX as infinity
         let max_retries_per_op = max_errors_at_row.get().wrapping_sub(1);
 
+        // Similar to above
+        let max_retries_in_total = max_errors.get().wrapping_sub(1);
+
         let hdr_latency_resolution = match hdr_latency_units.get().as_str() {
             "ns" => 1,
             "us" => 1000,
@@ -377,6 +387,7 @@ where
             concurrency,
             latency_type,
             max_retries_per_op,
+            max_retries_in_total,
             maximum_rate,
             test_duration: test_duration.get(),
             partition_count,

src/bin/cql-stress-scylla-bench/main.rs

@@ -146,7 +146,7 @@ async fn prepare(args: Arc<ScyllaBenchArgs>, stats: Arc<ShardedStats>) -> Result
         rate_limit_per_second,
         operation_factory,
         max_retries_per_op: args.max_retries_per_op as usize,
-        max_retries_in_total: usize::MAX,
+        max_retries_in_total: args.max_retries_in_total as usize,
     })
 }