fio: add latency steady state detection

Add fio latency steady state support. The implementation calculates
weighted average latency across all I/O directions and supports both
maximum mean deviation and slope-based detection methods.

Tested successfully against NVMe device with debug output confirming
proper latency calculation and steady state evaluation.

Quick demo with the null engine:

fio --name=test --ioengine=null --size=1G --rw=randread --bs=4k --iodepth=64 --steadystate=lat:100us --ss_dur=10s --runtime=120s --time_based=1
test: (g=0): rw=randread, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=null, iodepth=64
fio-3.41-20-gf2b2e-dirty
Starting 1 process
Jobs: 1 (f=1): [f(1)][100.0%][r=15.6GiB/s][r=4078k IOPS][eta 00m:00s]
test: (groupid=0, jobs=1): err= 0: pid=1207074: Tue Oct  7 11:47:27 2025
  read: IOPS=4054k, BW=15.5GiB/s (16.6GB/s)(153GiB/9899msec)
    slat (nsec): min=18, max=33347, avg=22.20, stdev=64.49
    clat (nsec): min=135, max=42283, avg=152.74, stdev=173.53
     lat (nsec): min=156, max=42306, avg=174.94, stdev=185.32
    clat percentiles (nsec):
     |  1.00th=[  139],  5.00th=[  141], 10.00th=[  141], 20.00th=[  143],
     | 30.00th=[  143], 40.00th=[  145], 50.00th=[  147], 60.00th=[  153],
     | 70.00th=[  155], 80.00th=[  157], 90.00th=[  163], 95.00th=[  171],
     | 99.00th=[  217], 99.50th=[  221], 99.90th=[  239], 99.95th=[  270],
     | 99.99th=[ 2096]
   bw (  MiB/s): min=15344, max=15944, per=100.00%, avg=15835.92, stdev=132.94, samples=19
   iops        : min=3928298, max=4081694, avg=4053996.63, stdev=34033.65, samples=19
  lat (nsec)   : 250=99.93%, 500=0.06%, 750=0.01%, 1000=0.01%
  lat (usec)   : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
  cpu          : usr=99.93%, sys=0.03%, ctx=88, majf=0, minf=6
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued rwts: total=40129848,0,0,0 short=0,0,0,0 dropped=0,0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=64
  steadystate  : attained=yes, bw=15.2GiB/s (15.9GB/s), iops=3976k, lat=152nsec, lat mean dev=2.700

Run status group 0 (all jobs):
   READ: bw=15.5GiB/s (16.6GB/s), 15.5GiB/s-15.5GiB/s (16.6GB/s-16.6GB/s), io=153GiB (164GB), run=9899-9899msec

Generated-by: Claude AI
Signed-off-by: Luis Chamberlain <[email protected]>

Author: mcgrof

HOWTO.rst

@@ -4216,6 +4216,18 @@ Steady state
 			Collect bandwidth data and calculate the least squares regression
 			slope. Stop the job if the slope falls below the specified limit.
 
+		**lat**
+			Collect completion latency data and calculate the maximum mean
+			deviation. Stop the job if the deviation falls below the specified
+			limit. The latency values are weighted by the number of I/O samples
+			in each measurement interval.
+
+		**lat_slope**
+			Collect completion latency data and calculate the least squares
+			regression slope. Stop the job if the slope falls below the
+			specified limit. The latency values are weighted by the number
+			of I/O samples in each measurement interval.
+
 .. option:: steadystate_duration=time, ss_dur=time
 
         A rolling window of this duration will be used to judge whether steady

client.c

@@ -1079,6 +1079,7 @@ static void convert_ts(struct thread_stat *dst, struct thread_stat *src)
 		for (i = 0; i < dst->ss_dur; i++ ) {
 			dst->ss_iops_data[i] = le64_to_cpu(src->ss_iops_data[i]);
 			dst->ss_bw_data[i] = le64_to_cpu(src->ss_bw_data[i]);
+			dst->ss_lat_data[i] = le64_to_cpu(src->ss_lat_data[i]);
 		}
 	}
 

example_latency_steadystate.fio

@@ -0,0 +1,47 @@
+# Example FIO job file demonstrating latency steady state detection
+# This example shows how to use FIO's latency steady state detection
+# to automatically terminate workloads when latency stabilizes
+#
+# Based on SNIA SSD Performance Test Specification requirements:
+# - Steady state is achieved when latency measurements don't change more than
+#   20% for 5 measurement windows and remain within 5% of a line with 10% slope
+# - This example uses more conservative 5% deviation threshold for demonstration
+
+[global]
+# Basic I/O parameters
+ioengine=libaio
+iodepth=32
+bs=4k
+direct=1
+rw=randread
+numjobs=1
+time_based=1
+runtime=3600  # Max runtime: 1 hour (will terminate early if steady state reached)
+
+# Steady state detection parameters
+steadystate=lat:5%           # Stop when latency mean deviation < 5% of average
+steadystate_duration=300     # Use 5-minute rolling window for measurements
+steadystate_ramp_time=60     # Wait 1 minute before starting measurements
+steadystate_check_interval=10 # Take measurements every 10 seconds
+
+# Output options
+write_lat_log=lat_steadystate
+log_avg_msec=10000           # Log average latency every 10 seconds
+
+[latency_steady_test]
+filename=/dev/nvme3n1
+size=10G
+
+# Alternative steady state configurations (uncomment to try):
+
+# Use slope-based detection instead of deviation:
+# steadystate=lat_slope:0.1%
+
+# More aggressive detection (faster convergence):
+# steadystate=lat:2%
+# steadystate_duration=120    # 2-minute window
+# steadystate_check_interval=5 # Check every 5 seconds
+
+# More conservative detection (slower convergence):
+# steadystate=lat:10%
+# steadystate_duration=600    # 10-minute window

options.c

@@ -1364,7 +1364,8 @@ static int str_random_distribution_cb(void *data, const char *str)
 static bool is_valid_steadystate(unsigned int state)
 {
 	return (state == FIO_SS_IOPS || state == FIO_SS_IOPS_SLOPE ||
-		state == FIO_SS_BW || state == FIO_SS_BW_SLOPE);
+		state == FIO_SS_BW || state == FIO_SS_BW_SLOPE ||
+		state == FIO_SS_LAT || state == FIO_SS_LAT_SLOPE);
 }
 
 static int str_steadystate_cb(void *data, const char *str)
@@ -1419,6 +1420,21 @@ static int str_steadystate_cb(void *data, const char *str)
 			return 0;
 
 		td->o.ss_limit.u.f = val;
+        } else if (td->o.ss_state & FIO_SS_LAT) {
+                long long tns;
+                if (check_str_time(nr, &tns, 0)) {
+                        log_err("fio: steadystate latency threshold parsing failed\n");
+                        free(nr);
+                        return 1;
+                }
+
+                dprint(FD_PARSE, "set steady state latency threshold to %lld nsec\n", tns);
+                free(nr);
+                if (parse_dryrun())
+                        return 0;
+
+                td->o.ss_limit.u.f = (double) tns;
+
 	} else {	/* bandwidth criterion */
 		if (str_to_decimal(nr, &ll, 1, td, 0, 0)) {
 			log_err("fio: steadystate BW threshold postfix parsing failed\n");
@@ -5536,6 +5552,14 @@ struct fio_option fio_options[FIO_MAX_OPTS] = {
 			    .oval = FIO_SS_BW_SLOPE,
 			    .help = "slope calculated from bandwidth measurements",
 			  },
+                          { .ival = "lat",
+                            .oval = FIO_SS_LAT,
+                            .help = "maximum mean deviation of latency measurements",
+                          },
+                          { .ival = "lat_slope",
+                            .oval = FIO_SS_LAT_SLOPE,
+                            .help = "slope calculated from latency measurements",
+                          },
 		},
 		.category = FIO_OPT_C_GENERAL,
 		.group  = FIO_OPT_G_RUNTIME,

stat.c

@@ -935,31 +935,52 @@ static void show_block_infos(int nr_block_infos, uint32_t *block_infos,
 
 static void show_ss_normal(const struct thread_stat *ts, struct buf_output *out)
 {
-	char *p1, *p1alt, *p2;
-	unsigned long long bw_mean, iops_mean;
+	char *p1, *p1alt, *p2, *p3 = NULL;
+	unsigned long long bw_mean, iops_mean, lat_mean;
 	const int i2p = is_power_of_2(ts->kb_base);
 
 	if (!ts->ss_dur)
 		return;
 
 	bw_mean = steadystate_bw_mean(ts);
 	iops_mean = steadystate_iops_mean(ts);
+	lat_mean = steadystate_lat_mean(ts);
 
 	p1 = num2str(bw_mean / ts->kb_base, ts->sig_figs, ts->kb_base, i2p, ts->unit_base);
 	p1alt = num2str(bw_mean / ts->kb_base, ts->sig_figs, ts->kb_base, !i2p, ts->unit_base);
 	p2 = num2str(iops_mean, ts->sig_figs, 1, 0, N2S_NONE);
+	if (ts->ss_state & FIO_SS_LAT) {
+		const char *lat_unit = "nsec";
+		unsigned long long lat_val = lat_mean;
+		double lat_mean_d = lat_mean, lat_dev_d = 0.0;
+		char *lat_num;
 
-	log_buf(out, "  steadystate  : attained=%s, bw=%s (%s), iops=%s, %s%s=%.3f%s\n",
+		if (nsec_to_msec(&lat_val, &lat_val, &lat_mean_d, &lat_dev_d))
+			lat_unit = "msec";
+		else if (nsec_to_usec(&lat_val, &lat_val, &lat_mean_d, &lat_dev_d))
+			lat_unit = "usec";
+
+		lat_num = num2str((unsigned long long)lat_mean_d, ts->sig_figs, 1, 0, N2S_NONE);
+		if (asprintf(&p3, "%s%s", lat_num, lat_unit) < 0)
+			p3 = NULL;
+		free(lat_num);
+	}
+
+	log_buf(out, "  steadystate  : attained=%s, bw=%s (%s), iops=%s%s%s, %s%s=%.3f%s\n",
 		ts->ss_state & FIO_SS_ATTAINED ? "yes" : "no",
 		p1, p1alt, p2,
-		ts->ss_state & FIO_SS_IOPS ? "iops" : "bw",
+		p3 ? ", lat=" : "",
+		p3 ? p3 : "",
+		ts->ss_state & FIO_SS_IOPS ? "iops" : (ts->ss_state & FIO_SS_LAT ? "lat" : "bw"),
 		ts->ss_state & FIO_SS_SLOPE ? " slope": " mean dev",
 		ts->ss_criterion.u.f,
 		ts->ss_state & FIO_SS_PCT ? "%" : "");
 
 	free(p1);
 	free(p1alt);
 	free(p2);
+	if (p3)
+		free(p3);
 }
 
 static void show_agg_stats(const struct disk_util_agg *agg, int terse,
@@ -1903,7 +1924,7 @@ static struct json_object *show_thread_status_json(struct thread_stat *ts,
 		int intervals = ts->ss_dur / (ss_check_interval / 1000L);
 
 		snprintf(ss_buf, sizeof(ss_buf), "%s%s:%f%s",
-			ts->ss_state & FIO_SS_IOPS ? "iops" : "bw",
+			ts->ss_state & FIO_SS_IOPS ? "iops" : (ts->ss_state & FIO_SS_LAT ? "lat" : "bw"),
 			ts->ss_state & FIO_SS_SLOPE ? "_slope" : "",
 			(float) ts->ss_limit.u.f,
 			ts->ss_state & FIO_SS_PCT ? "%" : "");
@@ -1942,6 +1963,16 @@ static struct json_object *show_thread_status_json(struct thread_stat *ts,
 		}
 		json_object_add_value_int(data, "bw_mean", steadystate_bw_mean(ts));
 		json_object_add_value_int(data, "iops_mean", steadystate_iops_mean(ts));
+		if (ts->ss_state & FIO_SS_LAT) {
+			struct json_array *lat;
+			lat = json_create_array();
+			for (l = 0; l < intervals; l++) {
+				k = (j + l) % intervals;
+				json_array_add_value_int(lat, ts->ss_lat_data[k]);
+			}
+			json_object_add_value_int(data, "lat_mean", steadystate_lat_mean(ts));
+			json_object_add_value_array(data, "lat", lat);
+		}
 		json_object_add_value_array(data, "iops", iops);
 		json_object_add_value_array(data, "bw", bw);
 	}
@@ -2600,6 +2631,7 @@ void __show_run_stats(void)
 			ts->ss_head = td->ss.head;
 			ts->ss_bw_data = td->ss.bw_data;
 			ts->ss_iops_data = td->ss.iops_data;
+			ts->ss_lat_data = td->ss.lat_data;
 			ts->ss_limit.u.f = td->ss.limit;
 			ts->ss_slope.u.f = td->ss.slope;
 			ts->ss_deviation.u.f = td->ss.deviation;

stat.h

@@ -283,6 +283,16 @@ struct thread_stat {
 		uint64_t pad5;
 	};
 
+	union {
+		uint64_t *ss_lat_data;
+		/*
+		 * For FIO_NET_CMD_TS, the pointed to data will temporarily
+		 * be stored at this offset from the start of the payload.
+		 */
+		uint64_t ss_lat_data_offset;
+		uint64_t pad5b;
+	};
+
 	union {
 		struct clat_prio_stat *clat_prio[DDIR_RWDIR_CNT];
 		/*