NVMe write warning

Signed-off-by: tunji-ruwase_snow <tunji.ruwase@snowflake.com>

Author: sfc-gh-truwase

.github/workflows/aws-torch-latest-full.yml

@@ -359,7 +359,7 @@ jobs:
           rm -rf /mnt/aio/pytest
           pytest --instafail --timeout 600 --forked -n 8 --basetemp=/mnt/aio/pytest unit/ \
             --ignore=unit/runtime/zero/test_nvme_checkpointing.py \
-            --ignore=unit/ops/aio/test_gds.py \
+            --ignore=unit/v1/nvme/test_gds.py \
             --ignore=unit/launcher/test_user_args.py \
             --ignore=unit/runtime/zenflow \
             --ignore=unit/ops/adam/test_zf_torch_adam.py \
@@ -372,7 +372,7 @@ jobs:
           rm -rf /mnt/aio/pytest
           pytest --instafail --timeout 600 -m 'sequential' --basetemp=/mnt/aio/pytest unit/ \
             --ignore=unit/runtime/zero/test_nvme_checkpointing.py \
-            --ignore=unit/ops/aio/test_gds.py \
+            --ignore=unit/v1/nvme/test_gds.py \
             --ignore=unit/launcher/test_user_args.py \
             --ignore=unit/runtime/zenflow \
             --ignore=unit/ops/adam/test_zf_torch_adam.py \

csrc/aio/py_lib/deepspeed_aio_op_desc.cpp

@@ -5,8 +5,30 @@
 
 #include "deepspeed_aio_op_desc.h"
 
+#include <mutex>
+
 using namespace std;
 
+namespace {
+// Warn home users only once per process: repeatedly offloading data (e.g. hidden
+// states) to a consumer SSD can exhaust its rated write endurance (TBW).
+std::once_flag write_warning_flag;
+
+void warn_consumer_ssd_writes()
+{
+    const char* msg =
+        "Offloading to NVMe can generate heavy write traffic. On consumer/home SSDs, repeatedly "
+        "offloading data (e.g. optimizer states, hidden states) can blow through the drive's "
+        "rated write endurance (TBW) and shorten its lifespan. Prefer enterprise/datacenter SSDs "
+        "for sustained offloading workloads.";
+
+    // Route through DeepSpeed's Python logger. Acquire the GIL since this may be
+    // invoked from a non-Python thread.
+    pybind11::gil_scoped_acquire acquire;
+    pybind11::module_::import("deepspeed.utils").attr("logger").attr("warning")(msg);
+}
+}  // namespace
+
 io_op_desc_t::io_op_desc_t(const bool read_op,
                            const torch::Tensor& buffer,
                            const int fd,
@@ -24,6 +46,7 @@ io_op_desc_t::io_op_desc_t(const bool read_op,
       _validate(validate)
 {
     if (validate) { assert(nullptr != filename); }
+    if (!read_op) { std::call_once(write_warning_flag, warn_consumer_ssd_writes); }
 }
 
 char* io_op_desc_t::data_ptr() const { return (char*)_contiguous_buffer.data_ptr(); }