- Update README

- Add MSI-X support
- Disarm the RX/TX interrupts in HW

Author: filipnavara

README.md

@@ -10,12 +10,20 @@ The driver contains code from the Intel FreeBSD igb driver version 2.5.30.
 
 Thanks to *CoolStar* for writing open source [RealTek driver](https://github.com/coolstar/if_re-win), and to *alotipac* for the [Raspberry Pi GENET Ethernet driver](https://github.com/raspberrypi/windows-drivers/). Both of them were invaluable resources that helped me bootstrap the prototype.
 
+## Technical details
+
+The driver supports one 1 TX queue and up to 4 RX queues. This was chosen to balance the supported send offloading methods, receive side scaling, and the number of interrupt vectors supported by the card. For transmission the NIC supports offloading of sending large IPv4, IPv6, TCP, and UDP packets. Since the NIC doesn't support receive side coalescing, we rely mostly on receive side scaling to distribute the load of RX queues.
+
+### Interrupt mapping
+
+In MSI/Legacy mode the RX queues are mapped into the first 4 bits of the Extended Interrupt Cause register. The TX queue uses the legacy bits.
+
+In MSI-X mode we map the RX queues to the first 4 interrupts. The 5th interrupt is used for link status changes and the TX queue.
+
 ## TODO
 
-- [ ] Efficient MSI/MSI-X interrupt handling
-- [x] Offloading
-  - [x] Checksums
-  - [x] Large send offload
 - [ ] Energy Efficient Ethernet
-- [ ] RSS
+- [ ] Wake on LAN
+- [ ] Manual Duplex/Speed setting
+- [ ] VLAN filtering
 - [ ] Enable support for more NIC models

adapter.cpp

@@ -145,12 +145,17 @@ IgbAdapterSetDatapathCapabilities(
 	NET_ADAPTER_DMA_CAPABILITIES rxDmaCapabilities;
 	NET_ADAPTER_DMA_CAPABILITIES_INIT(&rxDmaCapabilities, adapter->DmaEnabler);
 
+	SIZE_T maxRxQueues = IGB_MAX_RX_QUEUES;
+
+	if (adapter->MsiInterrupts > 1)
+		maxRxQueues = min(maxRxQueues, adapter->MsiInterrupts - 1);
+
 	NET_ADAPTER_RX_CAPABILITIES rxCapabilities;
 	NET_ADAPTER_RX_CAPABILITIES_INIT_SYSTEM_MANAGED_DMA(
 		&rxCapabilities,
 		&rxDmaCapabilities,
 		IGB_BUF_SIZE,
-		IGB_MAX_RX_QUEUES);
+		maxRxQueues);
 
 	rxCapabilities.FragmentBufferAlignment = IGB_RX_BUFFER_ALIGN;
 	rxCapabilities.FragmentRingNumberOfElementsHint = IGB_RX_BUF_NUM;
@@ -297,12 +302,14 @@ EvtAdapterReceiveScalingSetIndirectionEntries(
 	{
 		const ULONG queueId = IgbGetRxQueueContext(indirectionEntries->Entries[i].PacketQueue)->QueueId;
 		const UINT32 index = indirectionEntries->Entries[i].Index;
-		u32 reta = E1000_READ_REG(&adapter->Hw, E1000_RETA(index >> 2));
-		u32 shift = (index & 2) << 3;
-		reta ^= reta & (0xff << shift);
-		reta |= queueId << shift;
-		E1000_WRITE_REG(&adapter->Hw, E1000_RETA(index >> 2), reta);
-		indirectionEntries->Entries[i].Status = STATUS_SUCCESS;
+		if (index < 128)
+		{
+			u32 reta = E1000_READ_REG(&adapter->Hw, E1000_RETA(index >> 2));
+			u32 shift = (index & 2) << 3;
+			reta ^= reta & (0xff << shift);
+			reta |= queueId << shift;
+			E1000_WRITE_REG(&adapter->Hw, E1000_RETA(index >> 2), reta);
+		}
 	}
 
 	return STATUS_SUCCESS;

adapter.h

@@ -7,7 +7,7 @@ extern "C"
 
 #define IGB_MAX_TX_QUEUES 1
 #define IGB_MAX_RX_QUEUES 4
-//#define IGB_MAX_QUEUE_INTERRUPTS 4
+#define IGB_MAX_INTERRUPTS 5
 #define IGB_MAX_MCAST_LIST 32
 
 #define IGB_TX_BUF_NUM 1024
@@ -33,8 +33,9 @@ typedef struct _IGB_ADAPTER
 	SIZE_T MMIOSize;
 	BUS_INTERFACE_STANDARD PciConfig;
 
-	// Main interrupt
-	IGB_INTERRUPT* Interrupt;
+	PIGB_INTERRUPT MiscInterrupt;
+	PIGB_INTERRUPT Interrupts[IGB_MAX_INTERRUPTS];
+	UINT MsiInterrupts;
 
 	NET_PACKET_FILTER_FLAGS PacketFilterFlags;
 	UINT MCAddressLength;

device.cpp

@@ -47,18 +47,13 @@ IgbGetResources(
 		}
 		else if (rawDescriptor->Type == CmResourceTypeInterrupt)
 		{
-			/*if (translatedDescriptor->Flags & CM_RESOURCE_INTERRUPT_MESSAGE)
-			{
-				DBGPRINT("MSI/MSI-X interrupt %d\n", rawDescriptor->u.MessageInterrupt.Raw.MessageCount);
-			}
-			else
-			{
-				DBGPRINT("LBI interrupt\n");
-			}*/
-			if (intCnt == 0)
+			if (intCnt < IGB_MAX_INTERRUPTS)
 			{
 				GOTO_IF_NOT_NT_SUCCESS(Exit, status,
-					IgbInterruptCreate(adapter->WdfDevice, adapter, rawDescriptor, translatedDescriptor, &adapter->Interrupt));
+					IgbInterruptCreate(adapter->WdfDevice, adapter, rawDescriptor, translatedDescriptor, &adapter->Interrupts[intCnt]));
+
+				if (translatedDescriptor->Flags & CM_RESOURCE_INTERRUPT_MESSAGE)
+					adapter->MsiInterrupts++;
 			}
 			intCnt++;
 		}
@@ -89,7 +84,6 @@ NTSTATUS
 IgbRegisterScatterGatherDma(
 	_In_ IGB_ADAPTER* adapter)
 {
-	//TraceEntryRtAdapter(adapter);
 	DBGPRINT("IntelRegisterScatterGatherDma\n");
 
 	WDF_DMA_ENABLER_CONFIG dmaEnablerConfig;

forward.h

@@ -1,5 +1,6 @@
 #pragma once
 
-typedef struct _IGB_ADAPTER IGB_ADAPTER;
-typedef struct _IGB_DEVICE IGB_DEVICE;
-typedef struct _IGB_INTERRUPT IGB_INTERRUPT;
+typedef struct _IGB_ADAPTER IGB_ADAPTER, *PIGB_ADAPTER;
+typedef struct _IGB_DEVICE IGB_DEVICE, *PIGB_DEVICE;
+typedef struct _IGB_INTERRUPT IGB_INTERRUPT, *PIGB_INTERRUPT;
+typedef struct _IGB_QUEUE_INTERRUPT IGB_QUEUE_INTERRUPT, *PIGB_QUEUE_INTERRUPT;

interrupt.cpp

@@ -34,7 +34,6 @@ IgbInterruptCreate(
 	IGB_INTERRUPT* context = IgbGetInterruptContext(wdfInterrupt);
 	context->Adapter = adapter;
 	context->Handle = wdfInterrupt;
-	context->IsMsi = translatedDescriptor->Flags & CM_RESOURCE_INTERRUPT_MESSAGE ? TRUE : FALSE;
 	*interrupt = context;
 
 Exit:
@@ -43,14 +42,44 @@ IgbInterruptCreate(
 
 void
 IgbInterruptInitialize(
-	_In_ IGB_INTERRUPT* interrupt)
+	_In_ IGB_ADAPTER* adapter)
 {
-	// Map RX queues to EICR bits
 	// TODO: Handle 82575/82576 models
+
+	if (adapter->MsiInterrupts > 1)
+	{
+		UINT miscInterrupt = adapter->MsiInterrupts - 1;
+
+		// Enable MSI-X mode
+		E1000_WRITE_REG(&adapter->Hw, E1000_GPIE, E1000_GPIE_NSICR | E1000_GPIE_MSIX_MODE | E1000_GPIE_EIAME | E1000_GPIE_PBA);
+
+		// Map link & misc interrupts
+		E1000_WRITE_REG(&adapter->Hw, E1000_IVAR_MISC, ((miscInterrupt | E1000_IVAR_VALID) << 8));
+
+		// Map the TX queue to the misc interrupt
+		u32 ivar = E1000_READ_REG_ARRAY(&adapter->Hw, E1000_IVAR0, 0);
+		E1000_WRITE_REG_ARRAY(&adapter->Hw, E1000_IVAR0, 0, (ivar & 0xFFFF00FF) | ((miscInterrupt | E1000_IVAR_VALID) << 8));
+
+		// Automatically clear interrupt causes
+		E1000_WRITE_REG(&adapter->Hw, E1000_EIAC, (1 << adapter->MsiInterrupts) - 1);
+		// Automatically mask RX queues
+		E1000_WRITE_REG(&adapter->Hw, E1000_EIAM, (1 << (adapter->MsiInterrupts - 1)) - 1);
+		// Enable misc interrupt
+		E1000_WRITE_REG(&adapter->Hw, E1000_EIMS, (1 << miscInterrupt));
+
+		adapter->MiscInterrupt = adapter->Interrupts[miscInterrupt];
+	}
+	else
+	{
+		E1000_WRITE_REG(&adapter->Hw, E1000_GPIE, adapter->MsiInterrupts == 1 ? E1000_GPIE_NSICR : 0);
+		adapter->MiscInterrupt = adapter->Interrupts[0];
+	}
+
+	// Map RX queues to EICR bits
 	for (int i = 0; i < IGB_MAX_RX_QUEUES; i++)
 	{
 		u32 index = i >> 1;
-		u32 ivar = E1000_READ_REG_ARRAY(&interrupt->Adapter->Hw, E1000_IVAR0, index);
+		u32 ivar = E1000_READ_REG_ARRAY(&adapter->Hw, E1000_IVAR0, index);
 		if (i & 1)
 		{
 			ivar &= 0xFF00FFFF;
@@ -61,98 +90,130 @@ IgbInterruptInitialize(
 			ivar &= 0xFFFFFF00;
 			ivar |= i | E1000_IVAR_VALID;
 		}
-		E1000_WRITE_REG_ARRAY(&interrupt->Adapter->Hw, E1000_IVAR0, index, ivar);
+		E1000_WRITE_REG_ARRAY(&adapter->Hw, E1000_IVAR0, index, ivar);
 	}
 
-	// TODO: MSI-X
-	if (interrupt->IsMsi)
-	{
-		E1000_WRITE_REG(&interrupt->Adapter->Hw, E1000_GPIE, E1000_GPIE_NSICR);
-	}
-	else
-	{
-		E1000_WRITE_REG(&interrupt->Adapter->Hw, E1000_GPIE, 0);
-	}
+	E1000_WRITE_FLUSH(&adapter->Hw);
 }
 
 _Use_decl_annotations_
 BOOLEAN
 EvtInterruptIsr(
 	_In_ WDFINTERRUPT wdfInterrupt,
-	ULONG MessageID)
+	_In_ ULONG messageId)
 {
-	UNREFERENCED_PARAMETER((MessageID));
-
 	IGB_INTERRUPT* interrupt = IgbGetInterruptContext(wdfInterrupt);
 	IGB_ADAPTER* adapter = interrupt->Adapter;
 	struct e1000_hw* hw = &adapter->Hw;
-	u32 icr, eicr;
-	BOOLEAN queueDPC = FALSE;
-
-	eicr = E1000_READ_REG(hw, E1000_EICR);
-
-	// Hot eject
-	if (eicr == 0xffffffff)
-	{
-		return TRUE;
-	}
 
-	if (eicr != 0)
+	if (adapter->MsiInterrupts > 1)
 	{
-		InterlockedOr(&interrupt->EICR, eicr);
-
-		if ((eicr & E1000_EICR_OTHER) != 0)
+		if (interrupt == adapter->MiscInterrupt)
 		{
-			icr = E1000_READ_REG(hw, E1000_ICR);
+			// We use MSI-X for RX queues and we are forbidden to read EICR
+			u32 icr = E1000_READ_REG(hw, E1000_ICR);
+			
+			// Disarm the TX interrupts for queue where a transfer finished
+			if ((icr & (E1000_ICR_TXDW | E1000_ICR_TXQE)) != 0)
+			{
+				E1000_WRITE_REG(&interrupt->Adapter->Hw, E1000_IMC, E1000_IMS_TXDW | E1000_ICR_TXQE);
+				E1000_WRITE_FLUSH(&interrupt->Adapter->Hw);
+			}
+
 			InterlockedOr(&interrupt->ICR, icr);
+			InterlockedOr(&interrupt->EICR, E1000_EICR_OTHER);
+		}
+		else
+		{
+			// Mark the RX queue
+			InterlockedOr(&interrupt->EICR, 1 << messageId);
 		}
 
 		WdfInterruptQueueDpcForIsr(wdfInterrupt);
 		return TRUE;
 	}
+	else
+	{
+		u32 eicr = E1000_READ_REG(hw, E1000_EICR);
+
+		// Hot eject
+		if (eicr == 0xffffffff)
+		{
+			return TRUE;
+		}
+
+		if (eicr != 0)
+		{
+			// Disarm the RX interrupts for queues where we received packets
+			u32 eicrQueueMask = ((1 << IGB_MAX_RX_QUEUES) - 1);
+			if ((eicr & eicrQueueMask) != 0)
+			{
+				E1000_WRITE_REG(&interrupt->Adapter->Hw, E1000_EIMC, eicr & eicrQueueMask);
+				E1000_WRITE_FLUSH(&interrupt->Adapter->Hw);
+			}
+
+			if ((eicr & E1000_EICR_OTHER) != 0)
+			{
+				u32 icr = E1000_READ_REG(hw, E1000_ICR);
+
+				// Disarm the TX interrupts for queue where a transfer finished
+				if ((icr & (E1000_ICR_TXDW | E1000_ICR_TXQE)) != 0)
+				{
+					E1000_WRITE_REG(&interrupt->Adapter->Hw, E1000_IMC, E1000_IMS_TXDW | E1000_ICR_TXQE);
+					E1000_WRITE_FLUSH(&interrupt->Adapter->Hw);
+				}
+
+				InterlockedOr(&interrupt->ICR, icr);
+			}
+
+			InterlockedOr(&interrupt->EICR, eicr);
+
+			WdfInterruptQueueDpcForIsr(wdfInterrupt);
+			return TRUE;
+		}
 
-	return interrupt->IsMsi;
+		return FALSE;
+	}
 }
 
 _Use_decl_annotations_
 VOID
 EvtInterruptDpc(
-	_In_ WDFINTERRUPT Interrupt,
-	_In_ WDFOBJECT AssociatedObject)
+	_In_ WDFINTERRUPT wdfInterrupt,
+	_In_ WDFOBJECT associatedObject)
 {
-	UNREFERENCED_PARAMETER(AssociatedObject);
+	UNREFERENCED_PARAMETER(associatedObject);
 
-	IGB_INTERRUPT* interrupt = IgbGetInterruptContext(Interrupt);
+	IGB_INTERRUPT* interrupt = IgbGetInterruptContext(wdfInterrupt);
 	IGB_ADAPTER* adapter = interrupt->Adapter;
-	struct e1000_hw* hw = &adapter->Hw;
-	u32 icr = 0, eicr;
 
-	eicr = InterlockedExchange(&interrupt->EICR, 0);
+	LONG eicr = InterlockedExchange(&interrupt->EICR, 0);
 
-	for (int i = 0; i < IGB_MAX_RX_QUEUES; i++)
+	if ((eicr & ((1 << IGB_MAX_RX_QUEUES) - 1)) != 0)
 	{
-		if ((eicr & (1 << i)) != 0)
+		for (int i = 0; i < IGB_MAX_RX_QUEUES; i++)
 		{
-			if (InterlockedExchange(&interrupt->RxNotifyArmed[i], FALSE))
+			if ((eicr & (1 << i)) != 0)
+			{
 				NetRxQueueNotifyMoreReceivedPacketsAvailable(adapter->RxQueues[i]);
+			}
 		}
 	}
 
 	if ((eicr & E1000_EICR_OTHER) != 0)
 	{
-		icr = InterlockedExchange(&interrupt->ICR, 0);
+		LONG icr = InterlockedExchange(&interrupt->ICR, 0);
 
-		if ((icr & (E1000_ICR_TXDW | E1000_ICR_TXQE)) != 0 &&
-			InterlockedExchange(&interrupt->TxNotifyArmed[0], FALSE))
+		if ((icr & (E1000_ICR_TXDW | E1000_ICR_TXQE)) != 0)
 		{
 			NetTxQueueNotifyMoreCompletedPacketsAvailable(adapter->TxQueues[0]);
 		}
 
 		if ((icr & E1000_ICR_LSC) != 0)
 		{
 			DBGPRINT("Link Interrupt!\n");
-			hw->mac.get_link_status = 1;
+			adapter->Hw.mac.get_link_status = 1;
 			IgbCheckLinkStatus(adapter);
 		}
 	}
-}
+}

interrupt.h

@@ -4,15 +4,8 @@ typedef struct _IGB_INTERRUPT
 {
 	IGB_ADAPTER* Adapter;
 	WDFINTERRUPT Handle;
-
-	// Armed Notifications
-	LONG RxNotifyArmed[IGB_MAX_RX_QUEUES];
-	LONG TxNotifyArmed[IGB_MAX_TX_QUEUES];
-
 	LONG EICR;
 	LONG ICR;
-
-	BOOLEAN IsMsi;
 } IGB_INTERRUPT, *PIGB_INTERRUPT;
 
 WDF_DECLARE_CONTEXT_TYPE_WITH_NAME(IGB_INTERRUPT, IgbGetInterruptContext);
@@ -27,7 +20,9 @@ IgbInterruptCreate(
 
 void
 IgbInterruptInitialize(
-	_In_ IGB_INTERRUPT* interrupt);
+	_In_ IGB_ADAPTER* adapter);
 
 EVT_WDF_INTERRUPT_ISR EvtInterruptIsr;
 EVT_WDF_INTERRUPT_DPC EvtInterruptDpc;
+EVT_WDF_INTERRUPT_ISR EvtQueueInterruptIsr;
+EVT_WDF_INTERRUPT_DPC EvtQueueInterruptDpc;