NVMe write warning

.github/workflows/aws-torch-latest-full.yml

@@ -359,7 +359,7 @@ jobs:
           rm -rf /mnt/aio/pytest
           pytest --instafail --timeout 600 --forked -n 8 --basetemp=/mnt/aio/pytest unit/ \
             --ignore=unit/runtime/zero/test_nvme_checkpointing.py \
-            --ignore=unit/ops/aio/test_gds.py \
+            --ignore=unit/v1/nvme/test_gds.py \
             --ignore=unit/launcher/test_user_args.py \
             --ignore=unit/runtime/zenflow \
             --ignore=unit/ops/adam/test_zf_torch_adam.py \
@@ -372,7 +372,7 @@ jobs:
           rm -rf /mnt/aio/pytest
           pytest --instafail --timeout 600 -m 'sequential' --basetemp=/mnt/aio/pytest unit/ \
             --ignore=unit/runtime/zero/test_nvme_checkpointing.py \
-            --ignore=unit/ops/aio/test_gds.py \
+            --ignore=unit/v1/nvme/test_gds.py \
             --ignore=unit/launcher/test_user_args.py \
             --ignore=unit/runtime/zenflow \
             --ignore=unit/ops/adam/test_zf_torch_adam.py \

.github/workflows/aws-torch-latest.yml

@@ -106,8 +106,10 @@ jobs:
 
       - name: Run unit tests
         run: |
-          pytest -n 4 --forked --verbose tests/unit/v1/ --torch_ver=${{ env.TORCH_VER }} --cuda_ver=${{ env.CUDA_VER }}
+          # GDS tests require GPUDirect Storage support unavailable on these runners
+          pytest -n 4 --forked --verbose tests/unit/v1/ --ignore=tests/unit/v1/nvme/test_gds.py --torch_ver=${{ env.TORCH_VER }} --cuda_ver=${{ env.CUDA_VER }}
 
       - name: Run sequential unit tests
         run: |
-          pytest --verbose -m 'sequential' tests/unit/v1/ --torch_ver=${{ env.TORCH_VER }} --cuda_ver=${{ env.CUDA_VER }}
+          # GDS tests require GPUDirect Storage support unavailable on these runners
+          pytest --verbose -m 'sequential' tests/unit/v1/ --ignore=tests/unit/v1/nvme/test_gds.py --torch_ver=${{ env.TORCH_VER }} --cuda_ver=${{ env.CUDA_VER }}

ci/torch_latest.py

@@ -211,6 +211,8 @@ def pytest():
             "-n",
             "4",
             "--verbose",
+            # GDS tests require GPUDirect Storage support unavailable on these runners
+            "--ignore=tests/unit/v1/nvme/test_gds.py",
             *selected_tests,
             f"--torch_ver={MODAL_TORCH_TEST_VERSION}",
             f"--cuda_ver={MODAL_CUDA_TEST_VERSION}",

csrc/aio/py_lib/deepspeed_aio_op_desc.cpp

@@ -5,8 +5,30 @@
 
 #include "deepspeed_aio_op_desc.h"
 
+#include <mutex>
+
 using namespace std;
 
+namespace {
+// Warn home users only once per process: repeatedly offloading data (e.g. hidden
+// states) to a consumer SSD can exhaust its rated write endurance (TBW).
+std::once_flag write_warning_flag;
+
+void warn_consumer_ssd_writes()
+{
+    const char* msg =
+        "Offloading to NVMe can generate heavy write traffic. On consumer/home SSDs, repeatedly "
+        "offloading data (e.g. optimizer states, hidden states) can blow through the drive's "
+        "rated write endurance (TBW) and shorten its lifespan. Prefer enterprise/datacenter SSDs "
+        "for sustained offloading workloads.";
+
+    // Route through DeepSpeed's Python logger. Acquire the GIL since this may be
+    // invoked from a non-Python thread.
+    pybind11::gil_scoped_acquire acquire;
+    pybind11::module_::import("deepspeed.utils").attr("logger").attr("warning")(msg);
+}
+}  // namespace
+
 io_op_desc_t::io_op_desc_t(const bool read_op,
                            const torch::Tensor& buffer,
                            const int fd,
@@ -24,6 +46,7 @@ io_op_desc_t::io_op_desc_t(const bool read_op,
       _validate(validate)
 {
     if (validate) { assert(nullptr != filename); }
+    if (!read_op) { std::call_once(write_warning_flag, warn_consumer_ssd_writes); }
 }
 
 char* io_op_desc_t::data_ptr() const { return (char*)_contiguous_buffer.data_ptr(); }