MdeModulePkg: Add alternative queue sizes in nvme driver to support specific devices

Add a new PCD that toggles between the old driver behavior and new behavior that uses a different hardcoded queue size to support specific devices.  Default behavior should not change.

MdeModulePkg/NvmExpressDxe: Improve NVMe controller init robustness

It has been observed that some faulty NVMe devices may return
invalid values. The code in NvmExpressDxe recognizes the controller
is not responding correctly and issues an ASSERT() often in DEBUG
builds or a reset in RELEASE builds.

The following changes are made to NvmeControllerInit() to prevent a
faulty NVMe device from halting the overall boot:

1. Check the Vendor ID and Device ID to verify they are read properly
   and if not, return EFI_DEVICE_ERROR
2. Replace the ASSERT() when Memory Page Size Minimum (Cap.Mpsmin)
   with an error message and return EFI_DEVICE_ERROR

Signed-off-by: Michael Kubacki <[email protected]>

MdeModulePkg/NvmExpressDxe: Correct function parameter modifer

Updates the `Cap` parameter for `ReadNvmeControllerCapabilities()`
to be `OUT` since the buffer pointed to is written within the
function.

Signed-off-by: Michael Kubacki <[email protected]>

Author: chris-oo

MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpress.c

@@ -655,6 +655,9 @@ ProcessAsyncTaskList (
   EFI_BLOCK_IO2_TOKEN           *Token;
   BOOLEAN                       HasNewItem;
   EFI_STATUS                    Status;
+  // MU_CHANGE - Support alternative hardware queue sizes in NVME driver
+  UINT16  QueueSize = PcdGetBool (PcdSupportAlternativeQueueSize) ?
+                      NVME_ALTERNATIVE_MAX_QUEUE_SIZE : NVME_ASYNC_CCQ_SIZE;
 
   Private    = (NVME_CONTROLLER_PRIVATE_DATA *)Context;
   QueueId    = 2;
@@ -797,7 +800,8 @@ ProcessAsyncTaskList (
     }
 
     Private->CqHdbl[QueueId].Cqh++;
-    if (Private->CqHdbl[QueueId].Cqh > MIN (NVME_ASYNC_CCQ_SIZE, Private->Cap.Mqes)) {
+    // MU_CHANGE - Support alternative hardware queue sizes in NVME driver
+    if (Private->CqHdbl[QueueId].Cqh > MIN (QueueSize, Private->Cap.Mqes)) {
       Private->CqHdbl[QueueId].Cqh = 0;
       Private->Pt[QueueId]        ^= 1;
     }

MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpress.h

@@ -136,6 +136,20 @@ extern EFI_DRIVER_SUPPORTED_EFI_VERSION_PROTOCOL  gNvmExpressDriverSupportedEfiV
 
 #define NVME_ALL_NAMESPACES  0xFFFFFFFF
 
+// MU_CHANGE [BEGIN] - Support alternative hardware queue sizes in NVME driver
+
+//
+// NVMe DXE to accommodate hardware which requires queue size 255.
+// Driver supports queue size up to 255 (4 page SQ buffer).
+// DXE driver creates queue size MIN(Cap.Mqes, NVME_MAX_QUEUE_SIZE) for all queues.
+// Driver allocates queue buffer to support 255 max queue size.
+// Each submission queue buffer is allocated as 64B * 256 = 4 * 4kB = 4 pages.
+// Each completion queue buffer is allocated as 16B * 256 = 4kB = 1 page.
+//
+#define NVME_ALTERNATIVE_MAX_QUEUE_SIZE  255
+
+// MU_CHANGE [END] - Support alternative hardware queue sizes in NVME driver
+
 #define NVME_CONTROLLER_ID  0
 
 //

MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressDxe.inf

@@ -77,6 +77,11 @@
   gMediaSanitizeProtocolGuid                  ## PRODUCES
   gEfiResetNotificationProtocolGuid           ## CONSUMES
 
+[Pcd]
+  ## MU_CHANGE [BEGIN] - Support alternative hardware queue sizes in NVME driver
+  gEfiMdeModulePkgTokenSpaceGuid.PcdSupportAlternativeQueueSize ## CONSUMES
+  ## MU_CHANGE [END] - Support alternative hardware queue sizes in NVME driver
+
 # [Event]
 # EVENT_TYPE_RELATIVE_TIMER ## SOMETIMES_CONSUMES
 #

MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressHci.c

@@ -769,7 +769,11 @@ NvmeCreateIoCompletionQueue (
     CommandPacket.QueueType      = NVME_ADMIN_QUEUE;
 
     // MU_CHANGE [BEGIN] - Use the Mqes value from the Cap register
-    if (Index == 1) {
+    // MU_CHANGE [BEGIN] - Support alternative hardware queue sizes in NVME driver
+    if (PcdGetBool (PcdSupportAlternativeQueueSize)) {
+      QueueSize = MIN (NVME_ALTERNATIVE_MAX_QUEUE_SIZE, Private->Cap.Mqes);
+    } else if (Index == 1) {
+      // MU_CHANGE [END] - Support alternative hardware queue sizes in NVME driver
       QueueSize = MIN (NVME_CCQ_SIZE, Private->Cap.Mqes);
     } else {
       QueueSize = MIN (NVME_ASYNC_CCQ_SIZE, Private->Cap.Mqes);
@@ -846,7 +850,11 @@ NvmeCreateIoSubmissionQueue (
     CommandPacket.QueueType      = NVME_ADMIN_QUEUE;
 
     // MU_CHANGE [BEGIN] - Use the Mqes value from the Cap register
-    if (Index == 1) {
+    // MU_CHANGE [BEGIN] - Support alternative hardware queue sizes in NVME driver
+    if (PcdGetBool (PcdSupportAlternativeQueueSize)) {
+      QueueSize = MIN (NVME_ALTERNATIVE_MAX_QUEUE_SIZE, Private->Cap.Mqes);
+    } else if (Index == 1) {
+      // MU_CHANGE [END] - Support alternative hardware queue sizes in NVME driver
       QueueSize = MIN (NVME_CSQ_SIZE, Private->Cap.Mqes);
     } else {
       QueueSize = MIN (NVME_ASYNC_CSQ_SIZE, Private->Cap.Mqes);
@@ -1036,7 +1044,7 @@ NvmeControllerInit (
   EFI_STATUS           Status;
   EFI_PCI_IO_PROTOCOL  *PciIo;
   UINT64               Supports;
-  // NVME_AQA             Aqa;
+  UINT16               VidDid[2];  // MU_CHANGE - Improve NVMe controller init robustness
   // NVME_ASQ             Asq;
   // NVME_ACQ             Acq;
   UINT8                 Sn[21];
@@ -1050,10 +1058,33 @@ NvmeControllerInit (
 
   // MU_CHANGE [END] - Allocate IO Queue Buffer
 
+  // MU_CHANGE [BEGIN] - Improve NVMe controller init robustness
+  PciIo = Private->PciIo;
+
+  //
+  // Verify the controller is still accessible
+  //
+  Status = PciIo->Pci.Read (
+                        PciIo,
+                        EfiPciIoWidthUint16,
+                        PCI_VENDOR_ID_OFFSET,
+                        ARRAY_SIZE (VidDid),
+                        VidDid
+                        );
+  if (EFI_ERROR (Status)) {
+    ASSERT_EFI_ERROR (Status);
+    return EFI_DEVICE_ERROR;
+  }
+
+  if ((VidDid[0] == NVME_INVALID_VID_DID) || (VidDid[1] == NVME_INVALID_VID_DID)) {
+    return EFI_DEVICE_ERROR;
+  }
+
+  // MU_CHANGE [END] - Improve NVMe controller init robustness
+
   //
   // Enable this controller.
   //
-  PciIo  = Private->PciIo;
   Status = PciIo->Attributes (
                     PciIo,
                     EfiPciIoAttributeOperationSupported,
@@ -1092,7 +1123,16 @@ NvmeControllerInit (
   //
   // Currently the driver only supports 4k page size.
   //
-  ASSERT ((Private->Cap.Mpsmin + 12) <= EFI_PAGE_SHIFT);
+  // MU_CHANGE [BEGIN] - Improve NVMe controller init robustness
+
+  // Currently, this means Cap.Mpsmin must be zero for an EFI_PAGE_SHIFT size of 12.
+  // ASSERT ((Private->Cap.Mpsmin + 12) <= EFI_PAGE_SHIFT);
+  if ((Private->Cap.Mpsmin + 12) > EFI_PAGE_SHIFT) {
+    DEBUG ((DEBUG_ERROR, "NvmeControllerInit: Mpsmin is larger than expected (0x%02x).\n", Private->Cap.Mpsmin));
+    return EFI_DEVICE_ERROR;
+  }
+
+  // MU_CHANGE [END] - Improve NVMe controller init robustness
 
   // MU_CHANGE [BEGIN] - Allocate IO Queue Buffer
 
@@ -1122,10 +1162,12 @@ NvmeControllerInit (
   //
   // set number of entries admin submission & completion queues.
   //
-  Aqa.Asqs  = MIN (NVME_ASQ_SIZE, Private->Cap.Mqes);
+  // MU_CHANGE [BEGIN] - Support alternative hardware queue sizes in NVME driver
+  Aqa.Asqs  = PcdGetBool (PcdSupportAlternativeQueueSize) ? MIN (NVME_ALTERNATIVE_MAX_QUEUE_SIZE, Private->Cap.Mqes) : MIN (NVME_ASQ_SIZE, Private->Cap.Mqes);
   Aqa.Rsvd1 = 0;
-  Aqa.Acqs  = MIN (NVME_ACQ_SIZE, Private->Cap.Mqes);
+  Aqa.Acqs  = PcdGetBool (PcdSupportAlternativeQueueSize) ? MIN (NVME_ALTERNATIVE_MAX_QUEUE_SIZE, Private->Cap.Mqes) : MIN (NVME_ACQ_SIZE, Private->Cap.Mqes);
   Aqa.Rsvd2 = 0;
+  // MU_CHANGE [END] - Support alternative hardware queue sizes in NVME driver
 
   //
   // Set admin queue entry size to default

MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressPassthru.c

@@ -553,7 +553,15 @@ NvmExpressPassThru (
   Prp         = NULL;
   TimerEvent  = NULL;
   Status      = EFI_SUCCESS;
-  QueueSize   = MIN (NVME_ASYNC_CSQ_SIZE, Private->Cap.Mqes) + 1;
+
+  // MU_CHANGE [BEGIN] - Support alternative hardware queue sizes in NVME driver
+  if (PcdGetBool (PcdSupportAlternativeQueueSize)) {
+    QueueSize = MIN (NVME_ALTERNATIVE_MAX_QUEUE_SIZE, Private->Cap.Mqes) + 1;
+  } else {
+    QueueSize = MIN (NVME_ASYNC_CSQ_SIZE, Private->Cap.Mqes) + 1;
+  }
+
+  // MU_CHANGE [END] - Support alternative hardware queue sizes in NVME driver
 
   if (Packet->QueueType == NVME_ADMIN_QUEUE) {
     QueueId = 0;
@@ -581,6 +589,14 @@ NvmExpressPassThru (
     return EFI_INVALID_PARAMETER;
   }
 
+  // MU_CHANGE - Support alternative hardware queue sizes in NVME driver
+  //
+  // Nvme DXE driver polls phase bit for CQe completion.
+  // Explicitly assign phase bit with the bit before completion.
+  // A flipped phase bit will be assigned by device upon CQe completes.
+  //
+  Cq->Pt = Private->Pt[QueueId];
+
   ZeroMem (Sq, sizeof (NVME_SQ));
   Sq->Opc  = (UINT8)Packet->NvmeCmd->Cdw0.Opcode;
   Sq->Fuse = (UINT8)Packet->NvmeCmd->Cdw0.FusedOperation;
@@ -723,16 +739,23 @@ NvmExpressPassThru (
     Sq->Payload.Raw.Cdw15 = Packet->NvmeCmd->Cdw15;
   }
 
-  //
-  // Ring the submission queue doorbell.
-  //
-  if ((Event != NULL) && (QueueId != 0)) {
-    Private->SqTdbl[QueueId].Sqt =
-      (Private->SqTdbl[QueueId].Sqt + 1) % QueueSize;
+  // MU_CHANGE [BEGIN] - Support alternative hardware queue sizes in NVME driver
+  if (PcdGetBool (PcdSupportAlternativeQueueSize)) {
+    Private->SqTdbl[QueueId].Sqt = (Private->SqTdbl[QueueId].Sqt + 1) % QueueSize;
   } else {
-    Private->SqTdbl[QueueId].Sqt ^= 1;
+    //
+    // Ring the submission queue doorbell.
+    //
+    if ((Event != NULL) && (QueueId != 0)) {
+      Private->SqTdbl[QueueId].Sqt =
+        (Private->SqTdbl[QueueId].Sqt + 1) % QueueSize;
+    } else {
+      Private->SqTdbl[QueueId].Sqt ^= 1;
+    }
   }
 
+  // MU_CHANGE [END] - Support alternative hardware queue sizes in NVME driver
+
   Data   = ReadUnaligned32 ((UINT32 *)&Private->SqTdbl[QueueId]);
   Status = PciIo->Mem.Write (
                         PciIo,
@@ -865,10 +888,20 @@ NvmExpressPassThru (
     goto EXIT;
   }
 
-  if ((Private->CqHdbl[QueueId].Cqh ^= 1) == 0) {
-    Private->Pt[QueueId] ^= 1;
+  // MU_CHANGE [BEGIN] - Support alternative hardware queue sizes in NVME driver
+  if (PcdGetBool (PcdSupportAlternativeQueueSize)) {
+    Private->CqHdbl[QueueId].Cqh = (Private->CqHdbl[QueueId].Cqh + 1) % QueueSize;
+    if (Private->CqHdbl[QueueId].Cqh == 0) {
+      Private->Pt[QueueId] ^= 1;
+    }
+  } else {
+    if ((Private->CqHdbl[QueueId].Cqh ^= 1) == 0) {
+      Private->Pt[QueueId] ^= 1;
+    }
   }
 
+  // MU_CHANGE [END] - Support alternative hardware queue sizes in NVME driver
+
   Data           = ReadUnaligned32 ((UINT32 *)&Private->CqHdbl[QueueId]);
   PreviousStatus = Status;
   Status         = PciIo->Mem.Write (

MdeModulePkg/MdeModulePkg.dec

@@ -1385,6 +1385,10 @@
   # Default is 2000, it represent delay is 2 ms.
   # @Prompt Delay access XHCI register after it issues HCRST (us)
   gEfiMdeModulePkgTokenSpaceGuid.PcdDelayXhciHCReset|2000|UINT16|0x30001060
+  
+  # MU_CHANGE [BEGIN] - Support alternative hardware queue sizes in NVME driver
+  gEfiMdeModulePkgTokenSpaceGuid.PcdSupportAlternativeQueueSize|FALSE|BOOLEAN|0x40000151
+  # MU_CHANGE [END]
 
   ## Specifies the page count allocated for the MM communication buffer.
   # @Prompt Defines the page allocation for the MM communication buffer; default is 128 pages (512KB).