0173-cpuidle-psd-add-power-sleep-demotion-prevention-for-.patch

From f024054b3a97507df9b838e4840100c01d5f1d7d Mon Sep 17 00:00:00 2001
From: Colin Ian King <colin.i.king@intel.com>
Date: Tue, 3 Jun 2025 15:42:52 +0100
Subject: [PATCH] cpuidle: psd: power sleep demotion prevention for PCIe NVME I/O devices

Modern processors can drop into deep sleep states relatively quickly
to save power. However, coming out of deep sleep states takes a small
amount of time and this is detrimental to performance for I/O devices
such as fast PCIe NVME drives when servicing a completed I/O
transactions.

Testing with fio with read/write RAID0 PCIe NVME devices on various
modern SMP based systems (such as 96 thead Granite Rapids Xeon 6741P)
has shown that on 85-90% of read/write transactions issued on a CPU
are completed by the same CPU, so it makes some sense to prevent the
CPU from dropping into a deep sleep state to help reduce I/O handling
latency.

This commit introduces a simple, very lightweight and fast power sleep
demotion mechanism that provides the PCIe NVME driver a way to inform
the menu governor to prevent a CPU from going into a deep sleep when an
I/O operation is requested. While it is true that some I/Os may not
be serviced on the same CPU that issued the I/O request, the mechanism
does work well in the vast majority of I/O operations and there is very
small overhead with the sleep demotion prevention.

Test results on a 96 thread Xeon 6741P with a 6 way RAID0 PCIe NVME md
array. Tested with the NHM_C1_AUTO_DEMOTE bit set in the
MSR_PKG_CST_CONFIG_CONTROL (set in the BIOS).

Benchmarked with 4 I/O test cases from the Phoronix Test Suite v10.8.5,
each result is the average of 3 runs per test:

dbench 1.0.2:
  client count  original   patched   improvement
     1           1180.99   1320.52   11.8%
     6           3442.35   3606.76    4.7%
    12           5201.84   5441.22    4.6%
    48          11923.20  12638.30    6.0%
   128          14713.30  15314.10    4.1%
   256          15628.90  16208.00    3.7%

sqlite 2.2.0
  threads      original    patched  improvement
    16             7.93       7.19   -9.3%
    32            13.51      15.48   14.6%
    64            28.87      34.17   18.4%
    96            45.47      48.92    7.6%

iozone 1.9.6, 8GB file size:
I/O Size  Test original   patched   improvement
   4KB    Read  6838.45   6939.40     1.5%
   2MB    Read  7360.80   7464.94     1.4%
   4KB   Write    31.75     47.89    50.8%
   2MB   Write  1653.08   1817.86    10.6%

fio 2.1.0, Engine: Sync, Block Size 4K, I/O rate MB/sec
Test          Direct  Jobs  original  patched  improvement
Random Read       No   1        48.3     54.5   12.8%
Random Read       No   2        48.1     54.3   12.9%
Random Read       No   4        48.0     54.4   13.3%
Random Read       No   8        48.1     54.4   13.1%
Random Read       No  16        48.2     54.4   12.9%
Random Read       No  32        48.2     54.5   13.1%
Random Read       No  64        48.3     54.5   12.8%
Random Read       No  96        48.1     54.3   12.9%
Geometric Mean:                 48.2     54.4   13.0%

Random Read      Yes   1        50.7     56.0   10.5%
Random Read      Yes   2        50.7     55.9   10.3%
Random Read      Yes   4        50.7     55.9   10.3%
Random Read      Yes   8        50.7     56.0   10.5%
Random Read      Yes  16        50.7     55.9   10.3%
Random Read      Yes  32        50.5     55.9   10.7%
Random Read      Yes  64        50.6     55.8   10.3%
Random Read      Yes  96        50.7     55.8   10.1%
Geometric Mean:                 50.7     55.9   10.3%

Random Write      No   1        1488     1487   -0.1%
Random Write      No   2        1492     1483   -0.6%
Random Write      No   4        1485     1480   -0.3%
Random Write      No   8        1484     1479   -0.3%
Random Write      No  16        1490     1483   -0.5%
Random Write      No  32        1483     1490    0.5%
Random Write      No  64        1486     1485   -0.1%
Random Write      No  96        1476     1481    0.3%
Geometric Mean:                 1476     1477    0.1%

Random Write     Yes   1         298      335   12.4%
Random Write     Yes   2         297      335   12.8%
Random Write     Yes   4         298      334   12.1%
Random Write     Yes   8         299      337   12.7%
Random Write     Yes  16         296      332   12.2%
Random Write     Yes  32         298      331   11.1%
Random Write     Yes  64         297      335   12.8%
Random Write     Yes  96         298      335   12.4%
Geometric Mean:                  298      334   12.3%

Sequential Read   No   1        2622    2688     2.5%
Sequential Read   No   2        2647    2692     1.7%
Sequential Read   No   4        2634    2692     2.2%
Sequential Read   No   8        2661    2673     0.5%
Sequential Read   No  16        2673    2689     0.6%
Sequential Read   No  32        2667    2681     0.5%
Sequential Read   No  64        2640    2673     1.3%
Sequential Read   No  96        2635    2672     1.4%
Geometric Mean:                 2647    2683     1.1%

Sequential Read  Yes   1        87.4   105.0    20.1%
Sequential Read  Yes   2        87.5   104.0    18.9%
Sequential Read  Yes   4        87.3   105.0    20.3%
Sequential Read  Yes   8        87.2   105.0    20.4%
Sequential Read  Yes  16        87.4   105.0    20.1%
Sequential Read  Yes  32        86.5   105.0    21.4%
Sequential Read  Yes  64        87.0   104.0    19.5%
Sequential Read  Yes  96        87.2   105.0    20.4%
Geometric Mean:                 87.2   104.7    20.1%

Sequential Write  No   1        1868    1870     0.1%
Sequential Write  No   2        1879    1881     0.1%
Sequential Write  No   4        1877    1878     0.1%
Sequential Write  No   8        1887    1874    -0.7%
Sequential Write  No  16        1878    1885     0.4%
Sequential Write  No  32        1875    1878     0.2%
Sequential Write  No  64        1888    1879    -0.5%
Sequential Write  No  96        1892    1888    -0.5%
Geometric Mean:                 1881    1879    -0.2%

Sequential Write Yes   1         307     339    10.4%
Sequential Write Yes   2         306     341    11.4%
Sequential Write Yes   4         306     338    10.5%
Sequential Write Yes   8         307     339    10.4%
Sequential Write Yes  16         307     342    11.4%
Sequential Write Yes  32         307     337     9.8%
Sequential Write Yes  64         308     341    10.7%
Sequential Write Yes  96         306     341    11.4%
Geometric Mean:                  307     340    10.7%

For kernel builds, where all CPUs are fully loaded with a
small performance improvement based on the results of 5 kernel:
build test runs:

Kernel build time (geomean of 5 kernel builds, real time)
                  original   patched   improvement
Geometric Mean:      37.63s    37.39   0.8%
% Std.Dev.:           0.37%    0.31%

By default, CPU power sleep demotion blocking is set to run
for 1 ms on PCIe NVME I/O requests.

Signed-off-by: Colin Ian King <colin.i.king@intel.com>
---
 drivers/cpuidle/Kconfig          |  10 +++
 drivers/cpuidle/Makefile         |   1 +
 drivers/cpuidle/governors/menu.c |   4 ++
 drivers/cpuidle/psd.c            | 104 +++++++++++++++++++++++++++++++
 drivers/nvme/host/pci.c          |   3 +
 include/linux/cpuidle_psd.h      |  32 ++++++++++
 6 files changed, 154 insertions(+)
 create mode 100644 drivers/cpuidle/psd.c
 create mode 100644 include/linux/cpuidle_psd.h

diff --git a/drivers/cpuidle/Kconfig b/drivers/cpuidle/Kconfig
index cac5997dca50..b47b8e9e77b6 100644
--- a/drivers/cpuidle/Kconfig
+++ b/drivers/cpuidle/Kconfig
@@ -81,6 +81,16 @@ config HALTPOLL_CPUIDLE
 	 before halting in the guest (more efficient than polling in the
 	 host via halt_poll_ns for some scenarios).
 
+config CPU_IDLE_PSD
+	bool "prevent sleep demotion (PSD) for fast I/O devices"
+        default y
+        help
+         This option enables deferring of deep sleep states when a future
+         I/O based servicing event very probably going to happen in the very
+         near future, such as handling fast NVME device I/O. This reduces
+         uncessary transistions to deep idle sleep and reduces latency. This
+         provides the latency benefits of disabling deep sleep with the
+         power saving benefits of deep sleep when I/O is idle.
 endif
 
 config ARCH_NEEDS_CPU_IDLE_COUPLED
diff --git a/drivers/cpuidle/Makefile b/drivers/cpuidle/Makefile
index 1de9e92c5b0f..b219d488fe41 100644
--- a/drivers/cpuidle/Makefile
+++ b/drivers/cpuidle/Makefile
@@ -12,6 +12,7 @@ obj-$(CONFIG_DT_IDLE_STATES)		  += dt_idle_states.o
 obj-$(CONFIG_DT_IDLE_GENPD)		  += dt_idle_genpd.o
 obj-$(CONFIG_ARCH_HAS_CPU_RELAX)	  += poll_state.o
 obj-$(CONFIG_HALTPOLL_CPUIDLE)		  += cpuidle-haltpoll.o
+obj-$(CONFIG_CPU_IDLE_PSD)	  	  += psd.o
 
 ##################################################################################
 # ARM SoC drivers
diff --git a/drivers/cpuidle/governors/menu.c b/drivers/cpuidle/governors/menu.c
index 52d5d26fc7c6..eb5f04def188 100644
--- a/drivers/cpuidle/governors/menu.c
+++ b/drivers/cpuidle/governors/menu.c
@@ -16,6 +16,7 @@
 #include <linux/tick.h>
 #include <linux/sched/stat.h>
 #include <linux/math64.h>
+#include <linux/cpuidle_psd.h>
 
 #include "gov.h"
 
@@ -224,6 +225,9 @@ static int menu_select(struct cpuidle_driver *drv, struct cpuidle_device *dev,
 		data->needs_update = 0;
 	}
 
+	if (have_prevent_sleep_demotion())
+		latency_req = 0;
+
 	/* Find the shortest expected idle interval. */
 	predicted_ns = get_typical_interval(data) * NSEC_PER_USEC;
 	if (predicted_ns > RESIDENCY_THRESHOLD_NS) {
diff --git a/drivers/cpuidle/psd.c b/drivers/cpuidle/psd.c
new file mode 100644
index 000000000000..92251178a0b9
--- /dev/null
+++ b/drivers/cpuidle/psd.c
@@ -0,0 +1,104 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ *  Copyright (C) 2025 Intel Corporation
+ *  Author: Colin Ian King <colin.king@intel.com>
+ *
+ *  Kernel Prevent Sleep Demotion (PSD)
+ */
+#include <linux/kernel.h>
+#include <linux/cpu.h>
+#include <linux/device.h>
+#include <linux/percpu.h>
+#include <linux/jiffies.h>
+#include <linux/cpuidle_psd.h>
+
+/* jiffies at which the lease times out */
+static DEFINE_PER_CPU(unsigned long, psd_timeout);
+static int psd_cpu_lat_timeout_jiffies;
+
+/*
+ * A note about the use of the current cpu versus preemption.
+ *
+ * The use of have_prevent_sleep_demotion() is inside local
+ * power management code, and are pinned to that cpu already.
+ *
+ * On the "set" side, interrupt level code is obviously also fully
+ * migration-race free.
+ *
+ * All other cases are exposed to a migration-race.
+ *
+ * The goal of prevent sleep demotion is statistical rather than
+ * deterministic, e.g. on average the CPU that hits event X will go
+ * towards Y more often than not, and the impact of being wrong is a
+ * bit of extra power potentially for some short durations.
+ * Weighted against the costs in performance and complexity of dealing
+ * with the race, the race condition is acceptable.
+ *
+ * The second known race is where interrupt context might set a
+ * psd time in the middle of process context setting a different but
+ * psd smaller time, with the result that process context will win
+ * incorrectly, and the actual psd time will be less than expected,
+ * but still non-zero. Here also the cost of dealing with the race
+ * is outweight with the limited impact.
+ *
+ * The use of timings in jiffies is intentional, it is lightweight
+ * read and very fast. While it mau seem that using finer resolution
+ * timings is preferable, the expense is too high on I/O fast paths
+ * when preventing sleep demotions via prevent_sleep_demotion.
+ *
+ */
+int have_prevent_sleep_demotion(void)
+{
+	if (likely(psd_cpu_lat_timeout_jiffies)) {
+		int cpu = raw_smp_processor_id();
+
+		if (time_before(jiffies, per_cpu(psd_timeout, cpu)))
+			return 1;
+
+		/* keep the stored time value close to current */
+		per_cpu(psd_timeout, cpu) = jiffies;
+	}
+	return 0;
+}
+EXPORT_SYMBOL_GPL(have_prevent_sleep_demotion);
+
+void prevent_sleep_demotion(void)
+{
+	if (likely(psd_cpu_lat_timeout_jiffies)) {
+		const unsigned long next_jiffies = jiffies + psd_cpu_lat_timeout_jiffies;
+		const int cpu = raw_smp_processor_id();
+
+		/*  need to round up an extra jiffie */
+		if (time_before(per_cpu(psd_timeout, cpu), next_jiffies))
+			per_cpu(psd_timeout, cpu) = next_jiffies;
+	}
+}
+EXPORT_SYMBOL_GPL(prevent_sleep_demotion);
+
+static int psd_msecs_to_jiffies(const int msec)
+{
+	int ret = msecs_to_jiffies(msec);
+
+	return msec > 0 && ret == 0 ? 1 : ret;
+}
+
+static __init int prevent_sleep_demotion_init(void)
+{
+	struct device *dev_root = bus_get_dev_root(&cpu_subsys);
+	unsigned int cpu;
+
+	if (!dev_root)
+		return -1;
+
+	psd_cpu_lat_timeout_jiffies = psd_msecs_to_jiffies(PSD_NVME_DISK_MSEC);
+
+	pr_info("cpuidle-psd: using %d msec (%d jiffies) for idle timing\n",
+		PSD_NVME_DISK_MSEC, psd_cpu_lat_timeout_jiffies);
+
+	for_each_possible_cpu(cpu)
+		per_cpu(psd_timeout, cpu) = jiffies;
+
+	return 0;
+}
+
+late_initcall(prevent_sleep_demotion_init);
diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
index e0bfe04a2bc2..2bfee1a5bf78 100644
--- a/drivers/nvme/host/pci.c
+++ b/drivers/nvme/host/pci.c
@@ -28,6 +28,7 @@
 #include <linux/io-64-nonatomic-hi-lo.h>
 #include <linux/sed-opal.h>
 #include <linux/pci-p2pdma.h>
+#include <linux/cpuidle_psd.h>
 
 #include "trace.h"
 #include "nvme.h"
@@ -1044,6 +1045,7 @@ static blk_status_t nvme_queue_rq(struct blk_mq_hw_ctx *hctx,
 	ret = nvme_prep_rq(dev, req);
 	if (unlikely(ret))
 		return ret;
+	prevent_sleep_demotion();
 	spin_lock(&nvmeq->sq_lock);
 	nvme_sq_copy_cmd(nvmeq, &iod->cmd);
 	nvme_write_sq_db(nvmeq, bd->last);
@@ -1089,6 +1091,7 @@ static void nvme_queue_rqs(struct rq_list *rqlist)
 	struct nvme_queue *nvmeq = NULL;
 	struct request *req;
 
+	prevent_sleep_demotion();
 	while ((req = rq_list_pop(rqlist))) {
 		if (nvmeq && nvmeq != req->mq_hctx->driver_data)
 			nvme_submit_cmds(nvmeq, &submit_list);
diff --git a/include/linux/cpuidle_psd.h b/include/linux/cpuidle_psd.h
new file mode 100644
index 000000000000..5bcca5d53675
--- /dev/null
+++ b/include/linux/cpuidle_psd.h
@@ -0,0 +1,32 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ *  Copyright (C) 2025 Intel Corporation
+ *  Author: Colin Ian King <colin.king@intel.com>
+ *
+ *  Kernel prevent sleep demotion infrastructructure
+ */
+#ifndef _LINUX_CPUIDLE_PSD_H
+#define _LINUX_CPUIDLE_PSD_H
+
+/* duration of sleep demotion for PCIe NVME disks in msec */
+#define PSD_NVME_DISK_MSEC		(1)
+
+/* API prototypes */
+#ifdef CONFIG_CPU_IDLE_PSD
+
+extern void prevent_sleep_demotion(void);
+extern int have_prevent_sleep_demotion(void);
+
+#else
+
+static inline void prevent_sleep_demotion(void)
+{
+}
+
+static inline int have_prevent_sleep_demotion(void)
+{
+	return 0;
+}
+#endif
+
+#endif
-- 
2.49.0