Merge pull request #3 from filipnavara/interrupts

RSS & extended interrupts

Author: filipnavara

README.md

@@ -10,12 +10,20 @@ The driver contains code from the Intel FreeBSD igb driver version 2.5.30.
 
 Thanks to *CoolStar* for writing open source [RealTek driver](https://github.com/coolstar/if_re-win), and to *alotipac* for the [Raspberry Pi GENET Ethernet driver](https://github.com/raspberrypi/windows-drivers/). Both of them were invaluable resources that helped me bootstrap the prototype.
 
+## Technical details
+
+The driver supports one 1 TX queue and up to 4 RX queues. This was chosen to balance the supported send offloading methods, receive side scaling, and the number of interrupt vectors supported by the card. For transmission the NIC supports offloading of sending large IPv4, IPv6, TCP, and UDP packets. Since the NIC doesn't support receive side coalescing, we rely mostly on receive side scaling to distribute the load of RX queues.
+
+### Interrupt mapping
+
+In MSI/Legacy mode the RX queues are mapped into the first 4 bits of the Extended Interrupt Cause register. The TX queue uses the legacy bits.
+
+In MSI-X mode we map the RX queues to the first 4 interrupts. The 5th interrupt is used for link status changes and the TX queue.
+
 ## TODO
 
-- [ ] Efficient MSI/MSI-X interrupt handling
-- [x] Offloading
-  - [x] Checksums
-  - [x] Large send offload
 - [ ] Energy Efficient Ethernet
-- [ ] RSS
+- [ ] Wake on LAN
+- [ ] Manual Duplex/Speed setting
+- [ ] VLAN filtering
 - [ ] Enable support for more NIC models

adapter.cpp

@@ -28,10 +28,6 @@ IgbInitializeAdapterContext(
 	GOTO_IF_NOT_NT_SUCCESS(Exit, status,
 		WdfSpinLockCreate(&attributes, &adapter->Lock));
 
-	WDF_OBJECT_ATTRIBUTES timerAttributes;
-	WDF_OBJECT_ATTRIBUTES_INIT(&timerAttributes);
-	timerAttributes.ParentObject = device;
-
 Exit:
 	DBGPRINT("IntelInitializeAdapterContext - %x\n", status);
 
@@ -141,23 +137,28 @@ IgbAdapterSetDatapathCapabilities(
 	NET_ADAPTER_TX_CAPABILITIES_INIT_FOR_DMA(
 		&txCapabilities,
 		&txDmaCapabilities,
-		1);
+		IGB_MAX_TX_QUEUES);
 
-	txCapabilities.FragmentRingNumberOfElementsHint = /*RE_TX_BUF_NUM*/1024;
-	txCapabilities.MaximumNumberOfFragments = /*RE_NTXSEGS*/1;
+	txCapabilities.FragmentRingNumberOfElementsHint = IGB_TX_BUF_NUM;
+	txCapabilities.MaximumNumberOfFragments = IGB_MAX_PHYS_BUF_COUNT;
 
 	NET_ADAPTER_DMA_CAPABILITIES rxDmaCapabilities;
 	NET_ADAPTER_DMA_CAPABILITIES_INIT(&rxDmaCapabilities, adapter->DmaEnabler);
 
+	SIZE_T maxRxQueues = IGB_MAX_RX_QUEUES;
+
+	if (adapter->MsiInterrupts > 1)
+		maxRxQueues = min(maxRxQueues, adapter->MsiInterrupts - 1);
+
 	NET_ADAPTER_RX_CAPABILITIES rxCapabilities;
 	NET_ADAPTER_RX_CAPABILITIES_INIT_SYSTEM_MANAGED_DMA(
 		&rxCapabilities,
 		&rxDmaCapabilities,
 		IGB_BUF_SIZE,
-		1);
+		maxRxQueues);
 
-	rxCapabilities.FragmentBufferAlignment = /*RE_RX_BUFFER_ALIGN*/128;
-	rxCapabilities.FragmentRingNumberOfElementsHint = /*RE_RX_BUF_NUM*/1024;
+	rxCapabilities.FragmentBufferAlignment = IGB_RX_BUFFER_ALIGN;
+	rxCapabilities.FragmentRingNumberOfElementsHint = IGB_RX_BUF_NUM;
 
 	NetAdapterSetDataPathCapabilities(adapter->NetAdapter, &txCapabilities, &rxCapabilities);
 }
@@ -221,6 +222,100 @@ EvtAdapterOffloadSetRxChecksum(
 	adapter->RxUdpHwChkSum = NetOffloadIsRxChecksumUdpEnabled(offload);
 }
 
+
+static
+NTSTATUS
+EvtAdapterReceiveScalingEnable(
+	_In_ NETADAPTER netAdapter,
+	_In_ NET_ADAPTER_RECEIVE_SCALING_HASH_TYPE hashType,
+	_In_ NET_ADAPTER_RECEIVE_SCALING_PROTOCOL_TYPE protocolType)
+{
+	IGB_ADAPTER* adapter = IgbGetAdapterContext(netAdapter);
+
+	DBGPRINT("EvtAdapterReceiveScalingEnable\n");
+
+	u32 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
+
+	if (protocolType & NetAdapterReceiveScalingProtocolTypeIPv4)
+	{
+		mrqc |= E1000_MRQC_RSS_FIELD_IPV4;
+		if (protocolType & NetAdapterReceiveScalingProtocolTypeTcp)
+			mrqc |= E1000_MRQC_RSS_FIELD_IPV4_TCP;
+	}
+
+	if (protocolType & NetAdapterReceiveScalingProtocolTypeIPv6)
+	{
+		mrqc |= E1000_MRQC_RSS_FIELD_IPV6;
+		if (protocolType & NetAdapterReceiveScalingProtocolTypeTcp)
+			mrqc |= E1000_MRQC_RSS_FIELD_IPV6_TCP;
+	}
+
+	// TODO: E1000_MRQC_RSS_FIELD_IPV4_UDP,	E1000_MRQC_RSS_FIELD_IPV6_UDP,
+	// E1000_MRQC_RSS_FIELD_IPV6_UDP_EX, E1000_MRQC_RSS_FIELD_IPV6_TCP_EX
+
+	E1000_WRITE_REG(&adapter->Hw, E1000_MRQC, mrqc);
+
+	return STATUS_SUCCESS;
+}
+
+static
+VOID
+EvtAdapterReceiveScalingDisable(
+	_In_ NETADAPTER netAdapter)
+{
+	IGB_ADAPTER* adapter = IgbGetAdapterContext(netAdapter);
+
+	DBGPRINT("EvtAdapterReceiveScalingDisable\n");
+
+	E1000_WRITE_REG(&adapter->Hw, E1000_MRQC, 0);
+}
+
+static
+NTSTATUS
+EvtAdapterReceiveScalingSetHashSecretKey(
+	_In_ NETADAPTER netAdapter,
+	_In_ NET_ADAPTER_RECEIVE_SCALING_HASH_SECRET_KEY const* hashSecretKey)
+{
+	IGB_ADAPTER* adapter = IgbGetAdapterContext(netAdapter);
+
+	DBGPRINT("EvtAdapterReceiveScalingSetHashSecretKey\n");
+
+	for (int i = 0; i < 10 && i < hashSecretKey->Length; i++)
+	{
+		E1000_WRITE_REG_ARRAY(&adapter->Hw, E1000_RSSRK(0), i, hashSecretKey->Key[i]);
+	}
+
+	return STATUS_SUCCESS;
+}
+
+static
+NTSTATUS
+EvtAdapterReceiveScalingSetIndirectionEntries(
+	_In_ NETADAPTER netAdapter,
+	_In_ NET_ADAPTER_RECEIVE_SCALING_INDIRECTION_ENTRIES* indirectionEntries)
+{
+	IGB_ADAPTER* adapter = IgbGetAdapterContext(netAdapter);
+
+	DBGPRINT("EvtAdapterReceiveScalingSetIndirectionEntries\n");
+
+	for (size_t i = 0; i < indirectionEntries->Length; i++)
+	{
+		const ULONG queueId = IgbGetRxQueueContext(indirectionEntries->Entries[i].PacketQueue)->QueueId;
+		const UINT32 index = indirectionEntries->Entries[i].Index;
+		if (index < 128)
+		{
+			u32 reta = E1000_READ_REG(&adapter->Hw, E1000_RETA(index >> 2));
+			u32 shift = (index & 2) << 3;
+			reta ^= reta & (0xff << shift);
+			reta |= queueId << shift;
+			E1000_WRITE_REG(&adapter->Hw, E1000_RETA(index >> 2), reta);
+		}
+	}
+
+	return STATUS_SUCCESS;
+
+}
+
 static
 void
 IgbAdapterSetOffloadCapabilities(
@@ -271,6 +366,28 @@ IgbAdapterSetOffloadCapabilities(
 	NetAdapterOffloadSetIeee8021qTagCapabilities(adapter->NetAdapter, &ieee8021qTagOffloadCapabilities);
 }
 
+static
+void
+IgbAdapterSetRssCapabilities(
+	_In_ IGB_ADAPTER const* adapter)
+{
+	NET_ADAPTER_RECEIVE_SCALING_CAPABILITIES receiveScalingCapabilities;
+	NET_ADAPTER_RECEIVE_SCALING_CAPABILITIES_INIT(
+		&receiveScalingCapabilities,
+		IGB_MAX_RX_QUEUES,
+		NetAdapterReceiveScalingUnhashedTargetTypeHashIndex,
+		NetAdapterReceiveScalingHashTypeToeplitz,
+		NetAdapterReceiveScalingProtocolTypeIPv4 |
+		NetAdapterReceiveScalingProtocolTypeIPv6 |
+		NetAdapterReceiveScalingProtocolTypeTcp,
+		EvtAdapterReceiveScalingEnable,
+		EvtAdapterReceiveScalingDisable,
+		EvtAdapterReceiveScalingSetHashSecretKey,
+		EvtAdapterReceiveScalingSetIndirectionEntries);
+	receiveScalingCapabilities.SynchronizeSetIndirectionEntries = true;
+	NetAdapterSetReceiveScalingCapabilities(adapter->NetAdapter, &receiveScalingCapabilities);
+}
+
 _Use_decl_annotations_
 NTSTATUS
 IgbAdapterStart(
@@ -284,6 +401,7 @@ IgbAdapterStart(
 	IgbAdapterSetReceiveFilterCapabilities(adapter);
 	IgbAdapterSetDatapathCapabilities(adapter);
 	IgbAdapterSetOffloadCapabilities(adapter);
+	IgbAdapterSetRssCapabilities(adapter);
 
 	IgbUpdateReceiveFilters(adapter);
 

adapter.h

@@ -5,10 +5,16 @@ extern "C"
 #include "e1000_api.h"
 }
 
-#define IGB_MAX_TX_QUEUES 4
+#define IGB_MAX_TX_QUEUES 1
 #define IGB_MAX_RX_QUEUES 4
+#define IGB_MAX_INTERRUPTS 5
 #define IGB_MAX_MCAST_LIST 32
 
+#define IGB_TX_BUF_NUM 1024
+#define IGB_RX_BUF_NUM 1024
+#define IGB_RX_BUFFER_ALIGN 128
+#define IGB_MAX_PHYS_BUF_COUNT 1
+
 #define IGB_BUF_SIZE 2048
 
 typedef struct _IGB_ADAPTER
@@ -27,7 +33,9 @@ typedef struct _IGB_ADAPTER
 	SIZE_T MMIOSize;
 	BUS_INTERFACE_STANDARD PciConfig;
 
-	IGB_INTERRUPT* Interrupt;
+	PIGB_INTERRUPT MiscInterrupt;
+	PIGB_INTERRUPT Interrupts[IGB_MAX_INTERRUPTS];
+	UINT MsiInterrupts;
 
 	NET_PACKET_FILTER_FLAGS PacketFilterFlags;
 	UINT MCAddressLength;

device.cpp

@@ -5,7 +5,7 @@
 #include "trace.h"
 #include "device.h"
 #include "adapter.h"
-//#include "configuration.h"
+#include "interrupt.h"
 #include "link.h"
 
 NTSTATUS
@@ -20,6 +20,7 @@ IgbGetResources(
 
 	ULONG errorCode = 0;
 	ULONG memRegCnt = 0;
+	ULONG intCnt = 0;
 
 	// According to https://msdn.microsoft.com/windows/hardware/drivers/wdf/raw-and-translated-resources
 	// "Both versions represent the same set of hardware resources, in the same order."
@@ -44,9 +45,21 @@ IgbGetResources(
 
 			memRegCnt++;
 		}
+		else if (rawDescriptor->Type == CmResourceTypeInterrupt)
+		{
+			if (intCnt < IGB_MAX_INTERRUPTS)
+			{
+				GOTO_IF_NOT_NT_SUCCESS(Exit, status,
+					IgbInterruptCreate(adapter->WdfDevice, adapter, rawDescriptor, translatedDescriptor, &adapter->Interrupts[intCnt]));
+
+				if (translatedDescriptor->Flags & CM_RESOURCE_INTERRUPT_MESSAGE)
+					adapter->MsiInterrupts++;
+			}
+			intCnt++;
+		}
 	}
 
-	if (!adapter->MMIOAddress)
+	if (!adapter->MMIOAddress || intCnt == 0)
 	{
 		status = STATUS_RESOURCE_TYPE_NOT_FOUND;
 		errorCode = NDIS_ERROR_CODE_RESOURCE_CONFLICT;
@@ -71,7 +84,6 @@ NTSTATUS
 IgbRegisterScatterGatherDma(
 	_In_ IGB_ADAPTER* adapter)
 {
-	//TraceEntryRtAdapter(adapter);
 	DBGPRINT("IntelRegisterScatterGatherDma\n");
 
 	WDF_DMA_ENABLER_CONFIG dmaEnablerConfig;

driver.cpp

@@ -111,8 +111,6 @@ EvtDriverDeviceAdd(
 
 	GOTO_IF_NOT_NT_SUCCESS(Exit, status,
 		IgbInitializeAdapterContext(adapter, wdfDevice, netAdapter));
-	GOTO_IF_NOT_NT_SUCCESS(Exit, status,
-		IgbInterruptCreate(wdfDevice, adapter, &adapter->Interrupt));
 
 Exit:
 	if (adapterInit != nullptr)

forward.h

@@ -1,5 +1,6 @@
 #pragma once
 
-typedef struct _IGB_ADAPTER IGB_ADAPTER;
-typedef struct _IGB_DEVICE IGB_DEVICE;
-typedef struct _IGB_INTERRUPT IGB_INTERRUPT;
+typedef struct _IGB_ADAPTER IGB_ADAPTER, *PIGB_ADAPTER;
+typedef struct _IGB_DEVICE IGB_DEVICE, *PIGB_DEVICE;
+typedef struct _IGB_INTERRUPT IGB_INTERRUPT, *PIGB_INTERRUPT;
+typedef struct _IGB_QUEUE_INTERRUPT IGB_QUEUE_INTERRUPT, *PIGB_QUEUE_INTERRUPT;

igb.inx

@@ -48,6 +48,10 @@ AddReg                  = NetworkAddress.kw
 AddReg                  = OffloadChecksum.kw
 AddReg                  = PriorityVlanTag.kw
 AddReg                  = LSO.kw
+AddReg                  = ReceiveSideScaling.kw
+
+[IGB.ndi.NT.HW]
+AddReg                  = MSIInterrupt.kw
 
 [ndi.reg]
 ; TODO: Update these if your device is not Ethernet.
@@ -193,6 +197,17 @@ HKR,Ndi\Params\*LsoV2Ipv6,                      Default,        0,  "1"
 HKR,Ndi\Params\*LsoV2Ipv6\enum,                 "0",            0,  %Disabled%
 HKR,Ndi\Params\*LsoV2Ipv6\enum,                 "1",            0,  %Enabled%
 
+[ReceiveSideScaling.kw]
+HKR,Ndi\Params\*RSS,                            ParamDesc,      0, "%ReceiveSideScaling%"
+HKR,Ndi\Params\*RSS,                            default,        0, "1"
+HKR,Ndi\Params\*RSS,                            type,           0, "enum"
+HKR,Ndi\Params\*RSS,                            optional,       0, "0"
+HKR,Ndi\Params\*RSS\enum,                       "0",            0,  %Disabled%"
+HKR,Ndi\Params\*RSS\enum,                       "1",            0,  %Enabled%"
+
+[MSIInterrupt.kw]
+HKR,Interrupt Management\MessageSignaledInterruptProperties, MSISupported, 0x00010001, 1
+
 [Strings]
 AutoDetect               = "Auto Negotiation"
 SpeedDuplex              = "Speed & Duplex"
@@ -220,6 +235,7 @@ PriorityVLANDisabled     = "Priority & VLAN Disabled"
 PriorityVLANEnabled      = "Priority & VLAN Enabled"
 PriorityEnabled          = "Packet Priority Enabled"
 VLANEnabled              = "VLAN Enabled"
+ReceiveSideScaling       = "Receive Side Scaling"
 
 LSOv2Ipv4                = "Large Send Offload v2 (IPv4)"
 LSOv2Ipv6                = "Large Send Offload v2 (IPv6)"