diff --git a/eng/pipelines/templates/build-job.yml b/eng/pipelines/templates/build-job.yml index 2681c95a..5580d40a 100644 --- a/eng/pipelines/templates/build-job.yml +++ b/eng/pipelines/templates/build-job.yml @@ -78,6 +78,12 @@ jobs: - script: sudo gem install fpm displayName: Install fpm + - ${{ if eq(parameters.osGroup, 'Windows') }}: + - script: | + mkdir src\msquic\artifacts\xdp + xcopy /s /y src\xdp\* src\msquic\artifacts\xdp + displayName: Install xdp + - script: $(_crossBuild) $(_buildScript) $(_buildargs) displayName: Build binaries diff --git a/src/msquic b/src/msquic index 8652204d..7ea0e2e6 160000 --- a/src/msquic +++ b/src/msquic @@ -1 +1 @@ -Subproject commit 8652204d85b4a75fb7a4faf34217a2cb0a001111 +Subproject commit 7ea0e2e640008ad5c86d8d31e0cd186399b6b404 diff --git a/src/xdp/include/afxdp.h b/src/xdp/include/afxdp.h new file mode 100644 index 00000000..45e88cfa --- /dev/null +++ b/src/xdp/include/afxdp.h @@ -0,0 +1,241 @@ +// +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// + +#ifndef AFXDP_H +#define AFXDP_H + +#include + +#ifndef XDPAPI +#define XDPAPI __declspec(dllimport) +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +typedef union _XSK_BUFFER_ADDRESS { + struct { + UINT64 BaseAddress : 48; + UINT64 Offset : 16; +#pragma warning(suppress:4201) // nonstandard extension used: nameless struct/union + } DUMMYUNIONNAME; + UINT64 AddressAndOffset; +} XSK_BUFFER_ADDRESS; + +C_ASSERT(sizeof(XSK_BUFFER_ADDRESS) == sizeof(UINT64)); + +typedef struct _XSK_BUFFER_DESCRIPTOR { + XSK_BUFFER_ADDRESS Address; + UINT32 Length; + UINT32 Reserved; +} XSK_BUFFER_DESCRIPTOR; + +typedef struct _XSK_FRAME_DESCRIPTOR { + XSK_BUFFER_DESCRIPTOR Buffer; + // + // Followed by various descriptor extensions. + // +} XSK_FRAME_DESCRIPTOR; + +// +// Ensure frame-unaware apps can treat the frame ring as a buffer ring. +// +C_ASSERT(FIELD_OFFSET(XSK_FRAME_DESCRIPTOR, Buffer) == 0); + +typedef enum _XSK_RING_FLAGS { + XSK_RING_FLAG_NONE = 0x0, + XSK_RING_FLAG_ERROR = 0x1, + XSK_RING_FLAG_NEED_POKE = 0x2, + XSK_RING_FLAG_AFFINITY_CHANGED = 0x4, +} XSK_RING_FLAGS; + +DEFINE_ENUM_FLAG_OPERATORS(XSK_RING_FLAGS); +C_ASSERT(sizeof(XSK_RING_FLAGS) == sizeof(UINT32)); + +typedef +HRESULT +XSK_CREATE_FN( + _Out_ HANDLE* Socket + ); + +typedef enum _XSK_BIND_FLAGS { + XSK_BIND_FLAG_NONE = 0x0, + XSK_BIND_FLAG_RX = 0x1, + XSK_BIND_FLAG_TX = 0x2, + XSK_BIND_FLAG_GENERIC = 0x4, + XSK_BIND_FLAG_NATIVE = 0x8, +} XSK_BIND_FLAGS; + +DEFINE_ENUM_FLAG_OPERATORS(XSK_BIND_FLAGS) +C_ASSERT(sizeof(XSK_BIND_FLAGS) == sizeof(UINT32)); + +typedef +HRESULT +XSK_BIND_FN( + _In_ HANDLE Socket, + _In_ UINT32 IfIndex, + _In_ UINT32 QueueId, + _In_ XSK_BIND_FLAGS Flags + ); + +typedef enum _XSK_ACTIVATE_FLAGS { + XSK_ACTIVATE_FLAG_NONE = 0x0, +} XSK_ACTIVATE_FLAGS; + +DEFINE_ENUM_FLAG_OPERATORS(XSK_ACTIVATE_FLAGS) +C_ASSERT(sizeof(XSK_ACTIVATE_FLAGS) == sizeof(UINT32)); + +typedef +HRESULT +XSK_ACTIVATE_FN( + _In_ HANDLE Socket, + _In_ XSK_ACTIVATE_FLAGS Flags + ); + +typedef enum _XSK_NOTIFY_FLAGS { + XSK_NOTIFY_FLAG_NONE = 0x0, + XSK_NOTIFY_FLAG_POKE_RX = 0x1, + XSK_NOTIFY_FLAG_POKE_TX = 0x2, + XSK_NOTIFY_FLAG_WAIT_RX = 0x4, + XSK_NOTIFY_FLAG_WAIT_TX = 0x8, +} XSK_NOTIFY_FLAGS; + +DEFINE_ENUM_FLAG_OPERATORS(XSK_NOTIFY_FLAGS) +C_ASSERT(sizeof(XSK_NOTIFY_FLAGS) == sizeof(UINT32)); + +typedef enum _XSK_NOTIFY_RESULT_FLAGS { + XSK_NOTIFY_RESULT_FLAG_NONE = 0x0, + XSK_NOTIFY_RESULT_FLAG_RX_AVAILABLE = 0x1, + XSK_NOTIFY_RESULT_FLAG_TX_COMP_AVAILABLE = 0x2, +} XSK_NOTIFY_RESULT_FLAGS; + +DEFINE_ENUM_FLAG_OPERATORS(XSK_NOTIFY_RESULT_FLAGS) +C_ASSERT(sizeof(XSK_NOTIFY_RESULT_FLAGS) == sizeof(UINT32)); + +typedef +HRESULT +XSK_NOTIFY_SOCKET_FN( + _In_ HANDLE Socket, + _In_ XSK_NOTIFY_FLAGS Flags, + _In_ UINT32 WaitTimeoutMilliseconds, + _Out_ XSK_NOTIFY_RESULT_FLAGS *Result + ); + +typedef struct _OVERLAPPED OVERLAPPED; + +typedef +HRESULT +XSK_NOTIFY_ASYNC_FN( + _In_ HANDLE Socket, + _In_ XSK_NOTIFY_FLAGS Flags, + _Inout_ OVERLAPPED *Overlapped + ); + +typedef +HRESULT +XSK_GET_NOTIFY_ASYNC_RESULT_FN( + _In_ OVERLAPPED *Overlapped, + _Out_ XSK_NOTIFY_RESULT_FLAGS *Result + ); + +typedef +HRESULT +XSK_SET_SOCKOPT_FN( + _In_ HANDLE Socket, + _In_ UINT32 OptionName, + _In_reads_bytes_opt_(OptionLength) const VOID *OptionValue, + _In_ UINT32 OptionLength + ); + +typedef +HRESULT +XSK_GET_SOCKOPT_FN( + _In_ HANDLE Socket, + _In_ UINT32 OptionName, + _Out_writes_bytes_(*OptionLength) VOID *OptionValue, + _Inout_ UINT32 *OptionLength + ); + +typedef +HRESULT +XSK_IOCTL_FN( + _In_ HANDLE Socket, + _In_ UINT32 OptionName, + _In_reads_bytes_opt_(InputLength) const VOID *InputValue, + _In_ UINT32 InputLength, + _Out_writes_bytes_(*OutputLength) VOID *OutputValue, + _Inout_ UINT32 *OutputLength + ); + +// +// Socket options +// + +#define XSK_SOCKOPT_UMEM_REG 1 + +typedef struct _XSK_UMEM_REG { + UINT64 TotalSize; + UINT32 ChunkSize; + UINT32 Headroom; + VOID *Address; +} XSK_UMEM_REG; + +#define XSK_SOCKOPT_RX_RING_SIZE 2 +#define XSK_SOCKOPT_RX_FILL_RING_SIZE 3 +#define XSK_SOCKOPT_TX_RING_SIZE 4 +#define XSK_SOCKOPT_TX_COMPLETION_RING_SIZE 5 + +#define XSK_SOCKOPT_RING_INFO 6 + +typedef struct _XSK_RING_INFO { + BYTE *Ring; + UINT32 DescriptorsOffset; + UINT32 ProducerIndexOffset; + UINT32 ConsumerIndexOffset; + UINT32 FlagsOffset; + UINT32 Size; + UINT32 ElementStride; + UINT32 Reserved; +} XSK_RING_INFO; + +typedef struct _XSK_RING_INFO_SET { + XSK_RING_INFO Fill; + XSK_RING_INFO Completion; + XSK_RING_INFO Rx; + XSK_RING_INFO Tx; +} XSK_RING_INFO_SET; + +#define XSK_SOCKOPT_STATISTICS 7 + +typedef struct _XSK_STATISTICS { + UINT64 RxDropped; + UINT64 RxTruncated; + UINT64 RxInvalidDescriptors; + UINT64 TxInvalidDescriptors; +} XSK_STATISTICS; + +#define XSK_SOCKOPT_RX_HOOK_ID 8 +#define XSK_SOCKOPT_TX_HOOK_ID 9 + +#define XSK_SOCKOPT_RX_ERROR 10 +#define XSK_SOCKOPT_RX_FILL_ERROR 11 +#define XSK_SOCKOPT_TX_ERROR 12 +#define XSK_SOCKOPT_TX_COMPLETION_ERROR 13 + +typedef enum _XSK_ERROR { + XSK_NO_ERROR = 0, + XSK_ERROR_INTERFACE_DETACH = 0x80000000, + XSK_ERROR_INVALID_RING = 0xC0000000, +} XSK_ERROR; + +#define XSK_SOCKOPT_RX_PROCESSOR_AFFINITY 14 +#define XSK_SOCKOPT_TX_PROCESSOR_AFFINITY 15 + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif diff --git a/src/xdp/include/afxdp_experimental.h b/src/xdp/include/afxdp_experimental.h new file mode 100644 index 00000000..8886ab74 --- /dev/null +++ b/src/xdp/include/afxdp_experimental.h @@ -0,0 +1,105 @@ +// +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// + +// +// This header declares experimental AF_XDP interfaces. All definitions within +// this file are subject to breaking changes, including removal. +// + +#ifndef AFXDP_EXPERIMENTAL_H +#define AFXDP_EXPERIMENTAL_H + +#ifdef __cplusplus +extern "C" { +#endif + +// +// XSK_SOCKOPT_POLL_MODE +// +// Supports: set +// Optval type: XSK_POLL_MODE +// Description: Sets the poll mode of a socket. +// + +#define XSK_SOCKOPT_POLL_MODE 1000 + +typedef enum _XSK_POLL_MODE { + // + // Sets the XSK polling mode to the system default. + // + // Expectation: XSK_RING_FLAG_NEED_POKE varies. + // + XSK_POLL_MODE_DEFAULT, + + // + // Sets the XSK polling mode to a kernel busy loop. + // + // Expectation: XSK_RING_FLAG_NEED_POKE is usually FALSE. + // + XSK_POLL_MODE_BUSY, + + // + // Sets the XSK polling mode to poll only in the context of XskNotifySocket. + // + // Expectation: XSK_RING_FLAG_NEED_POKE is usually TRUE. + // + XSK_POLL_MODE_SOCKET, +} XSK_POLL_MODE; + +// +// XSK_SOCKOPT_TX_FRAME_LAYOUT_EXTENSION +// +// Supports: get +// Optval type: XDP_EXTENSION +// Description: Gets the XDP_FRAME_LAYOUT descriptor extension for the TX frame +// ring. This requires the socket is bound, the TX ring size is +// set, and at least one socket option has enabled the frame layout +// extension. +// +#define XSK_SOCKOPT_TX_FRAME_LAYOUT_EXTENSION 1001 + +// +// XSK_SOCKOPT_TX_FRAME_CHECKSUM_EXTENSION +// +// Supports: get +// Optval type: XDP_EXTENSION +// Description: Gets the XDP_FRAME_CHECKSUM descriptor extension for the TX +// frame ring. This requires the socket is bound, the TX ring size +// is set, and at least one socket option has enabled the frame +// layout extension. +// +#define XSK_SOCKOPT_TX_FRAME_CHECKSUM_EXTENSION 1002 + +// +// XSK_SOCKOPT_OFFLOAD_UDP_CHECKSUM_TX +// +// Supports: set +// Optval type: BOOLEAN +// Description: Sets whether UDP checksum transmit offload is enabled. This +// option requires the socket is bound and the TX frame ring size +// is not set. This option enables the XDP_FRAME_LAYOUT and +// XDP_FRAME_CHECKSUM extensions on the TX frame ring. +// +#define XSK_SOCKOPT_OFFLOAD_UDP_CHECKSUM_TX 1003 + +// +// XSK_SOCKOPT_OFFLOAD_UDP_CHECKSUM_TX_CAPABILITIES +// +// Supports: get +// Optval type: XSK_OFFLOAD_UDP_CHECKSUM_TX_CAPABILITIES +// Description: Returns the UDP checksum transmit offload capabilities. This +// option requires the socket is bound. +// +#define XSK_SOCKOPT_OFFLOAD_UDP_CHECKSUM_TX_CAPABILITIES 1004 + +typedef struct _XSK_OFFLOAD_UDP_CHECKSUM_TX_CAPABILITIES { + BOOLEAN Supported; +} XSK_OFFLOAD_UDP_CHECKSUM_TX_CAPABILITIES; + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif diff --git a/src/xdp/include/afxdp_helper.h b/src/xdp/include/afxdp_helper.h new file mode 100644 index 00000000..cf1ccf06 --- /dev/null +++ b/src/xdp/include/afxdp_helper.h @@ -0,0 +1,143 @@ +// +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// + +#ifndef AFXDP_HELPER_H +#define AFXDP_HELPER_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct _XSK_RING { + UINT32 *SharedProducer; + UINT32 *SharedConsumer; + UINT32 *SharedFlags; + VOID *SharedElements; + UINT32 Mask; + UINT32 Size; + UINT32 ElementStride; +} XSK_RING; + +inline +VOID +XskRingInitialize( + _Out_ XSK_RING *Ring, + _In_ CONST XSK_RING_INFO *RingInfo + ) +{ + RtlZeroMemory(Ring, sizeof(*Ring)); + + Ring->SharedProducer = (UINT32 *)(RingInfo->Ring + RingInfo->ProducerIndexOffset); + Ring->SharedConsumer = (UINT32 *)(RingInfo->Ring + RingInfo->ConsumerIndexOffset); + Ring->SharedFlags = (UINT32 *)(RingInfo->Ring + RingInfo->FlagsOffset); + Ring->SharedElements = RingInfo->Ring + RingInfo->DescriptorsOffset; + + Ring->Mask = RingInfo->Size - 1; + Ring->Size = RingInfo->Size; + Ring->ElementStride = RingInfo->ElementStride; +} + +inline +VOID * +XskRingGetElement( + _In_ CONST XSK_RING *Ring, + _In_ UINT32 Index + ) +{ + return (UCHAR *)Ring->SharedElements + (Index & Ring->Mask) * (SIZE_T)Ring->ElementStride; +} + +inline +UINT32 +XskRingGetFlags( + _In_ CONST XSK_RING *Ring + ) +{ + return ReadUInt32Acquire(Ring->SharedFlags); +} + +inline +UINT32 +XskRingConsumerReserve( + _In_ XSK_RING *Ring, + _In_ UINT32 MaxCount, + _Out_ UINT32 *Index + ) +{ + UINT32 Consumer = *Ring->SharedConsumer; + UINT32 Available = ReadUInt32Acquire(Ring->SharedProducer) - Consumer; + *Index = Consumer; + return Available < MaxCount ? Available : MaxCount; +} + +inline +VOID +XskRingConsumerRelease( + _Inout_ XSK_RING *Ring, + _In_ UINT32 Count + ) +{ + *Ring->SharedConsumer += Count; +} + +inline +UINT32 +XskRingProducerReserve( + _In_ XSK_RING *Ring, + _In_ UINT32 MaxCount, + _Out_ UINT32 *Index + ) +{ + UINT32 Producer = *Ring->SharedProducer; + UINT32 Available = Ring->Size - (Producer - ReadUInt32Acquire(Ring->SharedConsumer)); + *Index = Producer; + return Available < MaxCount ? Available : MaxCount; +} + +inline +VOID +XskRingProducerSubmit( + _Inout_ XSK_RING *Ring, + _In_ UINT32 Count + ) +{ + WriteUInt32Release(Ring->SharedProducer, *Ring->SharedProducer + Count); +} + +inline +BOOLEAN +XskRingError( + _In_ CONST XSK_RING *Ring + ) +{ + return !!(XskRingGetFlags(Ring) & XSK_RING_FLAG_ERROR); +} + +inline +BOOLEAN +XskRingProducerNeedPoke( + _In_ CONST XSK_RING *Ring + ) +{ + return !!(XskRingGetFlags(Ring) & XSK_RING_FLAG_NEED_POKE); +} + +inline +BOOLEAN +XskRingAffinityChanged( + _In_ CONST XSK_RING *Ring + ) +{ + return !!(XskRingGetFlags(Ring) & XSK_RING_FLAG_AFFINITY_CHANGED); +} + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif diff --git a/src/xdp/include/xdp/apiversion.h b/src/xdp/include/xdp/apiversion.h new file mode 100644 index 00000000..8f3069c2 --- /dev/null +++ b/src/xdp/include/xdp/apiversion.h @@ -0,0 +1,23 @@ +// +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// + +#ifndef APIVERSION_H +#define APIVERSION_H + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct _XDP_VERSION { + UINT32 Major; + UINT32 Minor; + UINT32 Patch; +} XDP_VERSION; + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif diff --git a/src/xdp/include/xdp/bufferinterfacecontext.h b/src/xdp/include/xdp/bufferinterfacecontext.h new file mode 100644 index 00000000..79bff41d --- /dev/null +++ b/src/xdp/include/xdp/bufferinterfacecontext.h @@ -0,0 +1,28 @@ +// +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// + +#pragma once + +EXTERN_C_START + +typedef VOID XDP_BUFFER_INTERFACE_CONTEXT; + +#define XDP_BUFFER_EXTENSION_INTERFACE_CONTEXT_NAME L"ms_buffer_interface_context" +#define XDP_BUFFER_EXTENSION_INTERFACE_CONTEXT_VERSION_1 1U + +#include +#include + +inline +XDP_BUFFER_INTERFACE_CONTEXT * +XdpGetBufferInterfaceContextExtension( + _In_ XDP_BUFFER *Buffer, + _In_ XDP_EXTENSION *Extension + ) +{ + return (XDP_BUFFER_INTERFACE_CONTEXT *)XdpGetExtensionData(Buffer, Extension); +} + +EXTERN_C_END diff --git a/src/xdp/include/xdp/bufferlogicaladdress.h b/src/xdp/include/xdp/bufferlogicaladdress.h new file mode 100644 index 00000000..3d3357a2 --- /dev/null +++ b/src/xdp/include/xdp/bufferlogicaladdress.h @@ -0,0 +1,37 @@ +// +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// + +#pragma once + +EXTERN_C_START + +#pragma warning(push) +#pragma warning(default:4820) // warn if the compiler inserted padding + +typedef struct _XDP_BUFFER_LOGICAL_ADDRESS { + UINT64 LogicalAddress; +} XDP_BUFFER_LOGICAL_ADDRESS; + +C_ASSERT(sizeof(XDP_BUFFER_LOGICAL_ADDRESS) == sizeof(VOID *)); + +#pragma warning(pop) + +#define XDP_BUFFER_EXTENSION_LOGICAL_ADDRESS_NAME L"ms_buffer_logical_address" +#define XDP_BUFFER_EXTENSION_LOGICAL_ADDRESS_VERSION_1 1U + +#include +#include + +inline +XDP_BUFFER_LOGICAL_ADDRESS * +XdpGetLogicalAddressExtension( + _In_ XDP_BUFFER *Buffer, + _In_ XDP_EXTENSION *Extension + ) +{ + return (XDP_BUFFER_LOGICAL_ADDRESS *)XdpGetExtensionData(Buffer, Extension); +} + +EXTERN_C_END diff --git a/src/xdp/include/xdp/buffermdl.h b/src/xdp/include/xdp/buffermdl.h new file mode 100644 index 00000000..eba4b521 --- /dev/null +++ b/src/xdp/include/xdp/buffermdl.h @@ -0,0 +1,38 @@ +// +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// + +#pragma once + +EXTERN_C_START + +#pragma warning(push) +#pragma warning(default:4820) // warn if the compiler inserted padding + +typedef struct _XDP_BUFFER_MDL { + MDL *Mdl; + SIZE_T MdlOffset; +} XDP_BUFFER_MDL; + +C_ASSERT(sizeof(XDP_BUFFER_MDL) == 2 * sizeof(VOID *)); + +#pragma warning(pop) + +#define XDP_BUFFER_EXTENSION_MDL_NAME L"ms_buffer_mdl" +#define XDP_BUFFER_EXTENSION_MDL_VERSION_1 1U + +#include +#include + +inline +XDP_BUFFER_MDL * +XdpGetMdlExtension( + _In_ XDP_BUFFER *Buffer, + _In_ XDP_EXTENSION *Extension + ) +{ + return (XDP_BUFFER_MDL *)XdpGetExtensionData(Buffer, Extension); +} + +EXTERN_C_END diff --git a/src/xdp/include/xdp/buffervirtualaddress.h b/src/xdp/include/xdp/buffervirtualaddress.h new file mode 100644 index 00000000..0b6509dd --- /dev/null +++ b/src/xdp/include/xdp/buffervirtualaddress.h @@ -0,0 +1,37 @@ +// +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// + +#pragma once + +EXTERN_C_START + +#pragma warning(push) +#pragma warning(default:4820) // warn if the compiler inserted padding + +typedef struct XDP_BUFFER_VIRTUAL_ADDRESS { + UCHAR *VirtualAddress; +} XDP_BUFFER_VIRTUAL_ADDRESS; + +C_ASSERT(sizeof(XDP_BUFFER_VIRTUAL_ADDRESS) == sizeof(VOID *)); + +#pragma warning(pop) + +#define XDP_BUFFER_EXTENSION_VIRTUAL_ADDRESS_NAME L"ms_buffer_virtual_address" +#define XDP_BUFFER_EXTENSION_VIRTUAL_ADDRESS_VERSION_1 1U + +#include +#include + +inline +XDP_BUFFER_VIRTUAL_ADDRESS * +XdpGetVirtualAddressExtension( + _In_ XDP_BUFFER *Buffer, + _In_ XDP_EXTENSION *Extension + ) +{ + return (XDP_BUFFER_VIRTUAL_ADDRESS *)XdpGetExtensionData(Buffer, Extension); +} + +EXTERN_C_END diff --git a/src/xdp/include/xdp/control.h b/src/xdp/include/xdp/control.h new file mode 100644 index 00000000..fb2f9ecc --- /dev/null +++ b/src/xdp/include/xdp/control.h @@ -0,0 +1,204 @@ +// +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// + +// +// This file contains declarations for the XDP Driver API control path. +// + +#pragma once + +#include +#include +#include +#include +#include +#include + +EXTERN_C_START + +#pragma warning(push) +#pragma warning(disable:4214) // nonstandard extension used: bit field types other than int + +// +// Forward declaration of XDP interface dispatch table. +// +typedef struct _XDP_INTERFACE_DISPATCH XDP_INTERFACE_DISPATCH; + +DECLARE_HANDLE(XDP_REGISTRATION_HANDLE); + +typedef +_When_(DriverApiVersionCount == 0, _Struct_size_bytes_(Header.Size)) +_When_(DriverApiVersionCount > 0, _Struct_size_bytes_(DriverApiVersionsOffset + DriverApiVersionCount * sizeof(XDP_VERSION))) +struct _XDP_CAPABILITIES_EX { + XDP_OBJECT_HEADER Header; + + UINT32 DriverApiVersionsOffset; + UINT32 DriverApiVersionCount; + XDP_VERSION DdkDriverApiVersion; + GUID InstanceId; +} XDP_CAPABILITIES_EX; + +#define XDP_CAPABILITIES_EX_REVISION_1 1 + +#define XDP_SIZEOF_CAPABILITIES_EX_REVISION_1 \ + RTL_SIZEOF_THROUGH_FIELD(XDP_CAPABILITIES_EX, InstanceId) + +typedef struct _XDP_CAPABILITIES { + XDP_CAPABILITIES_EX CapabilitiesEx; + XDP_VERSION DriverApiVersion; +} XDP_CAPABILITIES; + +inline +NTSTATUS +XdpInitializeCapabilities( + _Out_ XDP_CAPABILITIES *Capabilities, + _In_ CONST XDP_VERSION *DriverApiVersion + ) +{ + CONST XDP_VERSION DdkDriverApiVersion = { + XDP_DRIVER_API_MAJOR_VER, + XDP_DRIVER_API_MINOR_VER, + XDP_DRIVER_API_PATCH_VER + }; + + RtlZeroMemory(Capabilities, sizeof(*Capabilities)); + Capabilities->CapabilitiesEx.Header.Revision = XDP_CAPABILITIES_EX_REVISION_1; + Capabilities->CapabilitiesEx.Header.Size = XDP_SIZEOF_CAPABILITIES_EX_REVISION_1; + + Capabilities->CapabilitiesEx.DriverApiVersionsOffset = + FIELD_OFFSET(XDP_CAPABILITIES, DriverApiVersion); + Capabilities->CapabilitiesEx.DriverApiVersionCount = 1; + + Capabilities->DriverApiVersion = *DriverApiVersion; + Capabilities->CapabilitiesEx.DdkDriverApiVersion = DdkDriverApiVersion; + + return XdpGuidCreate(&Capabilities->CapabilitiesEx.InstanceId); +} + +NTSTATUS +XdpRegisterInterfaceEx( + _In_ UINT32 InterfaceIndex, + _In_ CONST XDP_CAPABILITIES_EX *CapabilitiesEx, + _In_ VOID *InterfaceContext, + _In_ CONST XDP_INTERFACE_DISPATCH *InterfaceDispatch, + _Out_ XDP_REGISTRATION_HANDLE *RegistrationHandle + ); + +inline +NTSTATUS +XdpRegisterInterface( + _In_ UINT32 InterfaceIndex, + _In_ CONST XDP_CAPABILITIES *Capabilities, + _In_ VOID *InterfaceContext, + _In_ CONST XDP_INTERFACE_DISPATCH *InterfaceDispatch, + _Out_ XDP_REGISTRATION_HANDLE *RegistrationHandle + ) +{ + return + XdpRegisterInterfaceEx( + InterfaceIndex, &Capabilities->CapabilitiesEx, InterfaceContext, + InterfaceDispatch, RegistrationHandle); +} + +VOID +XdpDeregisterInterface( + _In_ XDP_REGISTRATION_HANDLE RegistrationHandle + ); + +typedef +_IRQL_requires_(PASSIVE_LEVEL) +NTSTATUS +XDP_OPEN_INTERFACE( + _In_ XDP_INTERFACE_HANDLE InterfaceContext, + _Inout_ XDP_INTERFACE_CONFIG InterfaceConfig + ); + +typedef +_IRQL_requires_(PASSIVE_LEVEL) +VOID +XDP_CLOSE_INTERFACE( + _In_ XDP_INTERFACE_HANDLE InterfaceContext + ); + +typedef struct _XDP_INTERFACE_RX_QUEUE_DISPATCH { + XDP_INTERFACE_NOTIFY_QUEUE *InterfaceNotifyQueue; +} XDP_INTERFACE_RX_QUEUE_DISPATCH; + +typedef +_IRQL_requires_(PASSIVE_LEVEL) +NTSTATUS +XDP_CREATE_RX_QUEUE( + _In_ XDP_INTERFACE_HANDLE InterfaceContext, + _Inout_ XDP_RX_QUEUE_CONFIG_CREATE Config, + _Out_ XDP_INTERFACE_HANDLE *InterfaceRxQueue, + _Out_ CONST XDP_INTERFACE_RX_QUEUE_DISPATCH **InterfaceRxQueueDispatch + ); + +typedef +_IRQL_requires_(PASSIVE_LEVEL) +NTSTATUS +XDP_ACTIVATE_RX_QUEUE( + _In_ XDP_INTERFACE_HANDLE InterfaceRxQueue, + _In_ XDP_RX_QUEUE_HANDLE XdpRxQueue, + _In_ XDP_RX_QUEUE_CONFIG_ACTIVATE Config + ); + +typedef +_IRQL_requires_(PASSIVE_LEVEL) +VOID +XDP_DELETE_RX_QUEUE( + _In_ XDP_INTERFACE_HANDLE InterfaceRxQueue + ); + +typedef struct _XDP_INTERFACE_TX_QUEUE_DISPATCH { + XDP_INTERFACE_NOTIFY_QUEUE *InterfaceNotifyQueue; +} XDP_INTERFACE_TX_QUEUE_DISPATCH; + +typedef +_IRQL_requires_(PASSIVE_LEVEL) +NTSTATUS +XDP_CREATE_TX_QUEUE( + _In_ XDP_INTERFACE_HANDLE InterfaceContext, + _Inout_ XDP_TX_QUEUE_CONFIG_CREATE Config, + _Out_ XDP_INTERFACE_HANDLE *InterfaceTxQueue, + _Out_ CONST XDP_INTERFACE_TX_QUEUE_DISPATCH **InterfaceTxQueueDispatch + ); + +typedef +_IRQL_requires_(PASSIVE_LEVEL) +NTSTATUS +XDP_ACTIVATE_TX_QUEUE( + _In_ XDP_INTERFACE_HANDLE InterfaceTxQueue, + _In_ XDP_TX_QUEUE_HANDLE XdpTxQueue, + _In_ XDP_TX_QUEUE_CONFIG_ACTIVATE Config + ); + +typedef +_IRQL_requires_(PASSIVE_LEVEL) +VOID +XDP_DELETE_TX_QUEUE( + _In_ XDP_INTERFACE_HANDLE InterfaceTxQueue + ); + +typedef struct _XDP_INTERFACE_DISPATCH { + XDP_OBJECT_HEADER Header; + XDP_OPEN_INTERFACE *OpenInterface; + XDP_CLOSE_INTERFACE *CloseInterface; + XDP_CREATE_RX_QUEUE *CreateRxQueue; + XDP_ACTIVATE_RX_QUEUE *ActivateRxQueue; + XDP_DELETE_RX_QUEUE *DeleteRxQueue; + XDP_CREATE_TX_QUEUE *CreateTxQueue; + XDP_ACTIVATE_TX_QUEUE *ActivateTxQueue; + XDP_DELETE_TX_QUEUE *DeleteTxQueue; +} XDP_INTERFACE_DISPATCH; + +#define XDP_INTERFACE_DISPATCH_REVISION_1 1 + +#define XDP_SIZEOF_INTERFACE_DISPATCH_REVISION_1 \ + RTL_SIZEOF_THROUGH_FIELD(XDP_INTERFACE_DISPATCH, DeleteTxQueue) + +#pragma warning(pop) + +EXTERN_C_END diff --git a/src/xdp/include/xdp/datapath.h b/src/xdp/include/xdp/datapath.h new file mode 100644 index 00000000..d877dedc --- /dev/null +++ b/src/xdp/include/xdp/datapath.h @@ -0,0 +1,129 @@ +// +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// + +#pragma once + +EXTERN_C_START + +DECLARE_HANDLE(XDP_RX_QUEUE_HANDLE); +DECLARE_HANDLE(XDP_TX_QUEUE_HANDLE); +DECLARE_HANDLE(XDP_INTERFACE_HANDLE); + +#pragma warning(push) +#pragma warning(disable:4324) // structure was padded due to alignment specifier + +typedef struct DECLSPEC_CACHEALIGN _XDP_RING { + UINT32 ProducerIndex; + UINT32 ConsumerIndex; + UINT32 InterfaceReserved; + UINT32 Reserved; + UINT32 Mask; + UINT32 ElementStride; + // + // Followed by power-of-two array of ring elements. + // +} XDP_RING; + +#pragma warning(pop) + +C_ASSERT(sizeof(XDP_RING) == SYSTEM_CACHE_ALIGNMENT_SIZE); + +inline +VOID * +XdpRingGetElement( + _In_ XDP_RING *Ring, + _In_ UINT32 Index + ) +{ + ASSERT(Index <= Ring->Mask); + return (PUCHAR)&Ring[1] + (SIZE_T)Index * Ring->ElementStride; +} + +inline +UINT32 +XdpRingCount( + _In_ XDP_RING *Ring + ) +{ + return Ring->ProducerIndex - Ring->ConsumerIndex; +} + +inline +UINT32 +XdpRingFree( + _In_ XDP_RING *Ring + ) +{ + return Ring->Mask + 1 - XdpRingCount(Ring); +} + +typedef struct _XDP_BUFFER { + UINT32 DataOffset; + UINT32 DataLength; + UINT32 BufferLength; + UINT32 Reserved; + // + // Followed by various XDP descriptor extensions. + // +} XDP_BUFFER; + +C_ASSERT(sizeof(XDP_BUFFER) == 16); + +typedef struct _XDP_FRAME { + XDP_BUFFER Buffer; + // + // Followed by various XDP descriptor extensions. + // +} XDP_FRAME; + +typedef struct _XDP_TX_FRAME_COMPLETION { + UINT64 BufferAddress; +} XDP_TX_FRAME_COMPLETION; + +typedef enum _XDP_RX_ACTION { + XDP_RX_ACTION_DROP, + XDP_RX_ACTION_PASS, + XDP_RX_ACTION_TX, +} XDP_RX_ACTION; + +_IRQL_requires_max_(DISPATCH_LEVEL) +VOID +XdpReceive( + _In_ XDP_RX_QUEUE_HANDLE XdpRxQueue + ); + +_IRQL_requires_max_(DISPATCH_LEVEL) +VOID +XdpFlushReceive( + _In_ XDP_RX_QUEUE_HANDLE XdpRxQueue + ); + +_IRQL_requires_max_(DISPATCH_LEVEL) +VOID +XdpFlushTransmit( + _In_ XDP_TX_QUEUE_HANDLE XdpTxQueue + ); + +typedef enum _XDP_NOTIFY_QUEUE_FLAGS { + XDP_NOTIFY_QUEUE_FLAG_NONE = 0x0, + XDP_NOTIFY_QUEUE_FLAG_RX = 0x1, + XDP_NOTIFY_QUEUE_FLAG_TX = 0x2, + XDP_NOTIFY_QUEUE_FLAG_RX_FLUSH = 0x4, + XDP_NOTIFY_QUEUE_FLAG_TX_FLUSH = 0x8, +} XDP_NOTIFY_QUEUE_FLAGS; + +DEFINE_ENUM_FLAG_OPERATORS(XDP_NOTIFY_QUEUE_FLAGS); + +typedef +_IRQL_requires_max_(PASSIVE_LEVEL) +VOID +XDP_INTERFACE_NOTIFY_QUEUE( + _In_ XDP_INTERFACE_HANDLE InterfaceQueue, + _In_ XDP_NOTIFY_QUEUE_FLAGS Flags + ); + +#include + +EXTERN_C_END diff --git a/src/xdp/include/xdp/details/datapath.h b/src/xdp/include/xdp/details/datapath.h new file mode 100644 index 00000000..08be4105 --- /dev/null +++ b/src/xdp/include/xdp/details/datapath.h @@ -0,0 +1,76 @@ +// +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// + +#pragma once + +EXTERN_C_START + +#include +#include + +typedef +_IRQL_requires_max_(DISPATCH_LEVEL) +VOID +XDP_RECEIVE( + _In_ XDP_RX_QUEUE_HANDLE XdpRxQueue + ); + +typedef +_IRQL_requires_max_(DISPATCH_LEVEL) +VOID +XDP_FLUSH_RECEIVE( + _In_ XDP_RX_QUEUE_HANDLE XdpRxQueue + ); + +typedef struct _XDP_RX_QUEUE_DISPATCH { + XDP_RECEIVE *Receive; + XDP_FLUSH_RECEIVE *FlushReceive; +} XDP_RX_QUEUE_DISPATCH; + +inline +_IRQL_requires_max_(DISPATCH_LEVEL) +VOID +XDPEXPORT(XdpReceive)( + _In_ XDP_RX_QUEUE_HANDLE XdpRxQueue + ) +{ + CONST XDP_RX_QUEUE_DISPATCH *Dispatch = (CONST XDP_RX_QUEUE_DISPATCH *)XdpRxQueue; + Dispatch->Receive(XdpRxQueue); +} + +inline +_IRQL_requires_max_(DISPATCH_LEVEL) +VOID +XDPEXPORT(XdpFlushReceive)( + _In_ XDP_RX_QUEUE_HANDLE XdpRxQueue + ) +{ + CONST XDP_RX_QUEUE_DISPATCH *Dispatch = (CONST XDP_RX_QUEUE_DISPATCH *)XdpRxQueue; + Dispatch->FlushReceive(XdpRxQueue); +} + +typedef +_IRQL_requires_max_(DISPATCH_LEVEL) +VOID +XDP_FLUSH_TRANSMIT( + _In_ XDP_TX_QUEUE_HANDLE XdpTxQueue + ); + +typedef struct _XDP_TX_QUEUE_DISPATCH { + XDP_FLUSH_TRANSMIT *FlushTransmit; +} XDP_TX_QUEUE_DISPATCH; + +inline +_IRQL_requires_max_(DISPATCH_LEVEL) +VOID +XDPEXPORT(XdpFlushTransmit)( + _In_ XDP_TX_QUEUE_HANDLE XdpTxQueue + ) +{ + CONST XDP_TX_QUEUE_DISPATCH *Dispatch = (CONST XDP_TX_QUEUE_DISPATCH *)XdpTxQueue; + Dispatch->FlushTransmit(XdpTxQueue); +} + +EXTERN_C_END diff --git a/src/xdp/include/xdp/details/export.h b/src/xdp/include/xdp/details/export.h new file mode 100644 index 00000000..58b1c5a1 --- /dev/null +++ b/src/xdp/include/xdp/details/export.h @@ -0,0 +1,21 @@ +// +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// + +#ifndef XDP_EXPORT_H +#define XDP_EXPORT_H + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef XDPEXPORT +#define XDPEXPORT(X) X +#endif + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif diff --git a/src/xdp/include/xdp/details/extension.h b/src/xdp/include/xdp/details/extension.h new file mode 100644 index 00000000..da31bde6 --- /dev/null +++ b/src/xdp/include/xdp/details/extension.h @@ -0,0 +1,29 @@ +// +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// + +#ifndef XDP_EXTENSION_DETAILS_H +#define XDP_EXTENSION_DETAILS_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +inline +VOID * +XdpGetExtensionData( + _In_ VOID *Descriptor, + _In_ XDP_EXTENSION *Extension + ) +{ + return (VOID *)((UCHAR *)Descriptor + Extension->Reserved); +} + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif diff --git a/src/xdp/include/xdp/details/interfaceconfig.h b/src/xdp/include/xdp/details/interfaceconfig.h new file mode 100644 index 00000000..f699ad99 --- /dev/null +++ b/src/xdp/include/xdp/details/interfaceconfig.h @@ -0,0 +1,46 @@ +// +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// + +#pragma once + +EXTERN_C_START + +#include +#include +#include +#include + +typedef +CONST XDP_VERSION * +XDP_GET_DRIVER_API_VERSION( + _In_ XDP_INTERFACE_CONFIG InterfaceConfig + ); + +typedef struct _XDP_INTERFACE_CONFIG_DISPATCH { + XDP_OBJECT_HEADER Header; + XDP_GET_DRIVER_API_VERSION *GetDriverApiVersion; +} XDP_INTERFACE_CONFIG_DISPATCH; + +#define XDP_INTERFACE_CONFIG_DISPATCH_REVISION_1 1 + +#define XDP_SIZEOF_INTERFACE_CONFIG_DISPATCH_REVISION_1 \ + RTL_SIZEOF_THROUGH_FIELD(XDP_INTERFACE_CONFIG_DISPATCH, GetDriverApiVersion) + +typedef struct _XDP_INTERFACE_CONFIG_DETAILS { + CONST XDP_INTERFACE_CONFIG_DISPATCH *Dispatch; +} XDP_INTERFACE_CONFIG_DETAILS; + +inline +CONST XDP_VERSION * +XDPEXPORT(XdpGetDriverApiVersion)( + _In_ XDP_INTERFACE_CONFIG InterfaceConfig + ) +{ + XDP_INTERFACE_CONFIG_DETAILS *Details = + (XDP_INTERFACE_CONFIG_DETAILS *)InterfaceConfig; + return Details->Dispatch->GetDriverApiVersion(InterfaceConfig); +} + +EXTERN_C_END diff --git a/src/xdp/include/xdp/details/rxqueueconfig.h b/src/xdp/include/xdp/details/rxqueueconfig.h new file mode 100644 index 00000000..87a3dfd6 --- /dev/null +++ b/src/xdp/include/xdp/details/rxqueueconfig.h @@ -0,0 +1,206 @@ +// +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// + +#pragma once + +EXTERN_C_START + +#include +#include + +typedef +CONST XDP_QUEUE_INFO * +XDP_RX_QUEUE_GET_TARGET_QUEUE_INFO( + _In_ XDP_RX_QUEUE_CONFIG_CREATE RxQueueConfig + ); + +typedef +BOOLEAN +XDP_RX_QUEUE_CREATE_IS_ENABLED( + _In_ XDP_RX_QUEUE_CONFIG_CREATE RxQueueConfig + ); + +typedef +VOID +XDP_RX_QUEUE_REGISTER_EXTENSION_VERSION( + _In_ XDP_RX_QUEUE_CONFIG_CREATE RxQueueConfig, + _In_ XDP_EXTENSION_INFO *ExtensionInfo + ); + +typedef +VOID +XDP_RX_QUEUE_SET_CAPABILITIES( + _In_ XDP_RX_QUEUE_CONFIG_CREATE RxQueueConfig, + _In_ XDP_RX_CAPABILITIES *Capabilities + ); + +typedef +VOID +XDP_RX_QUEUE_SET_DESCRIPTOR_CONTEXTS( + _In_ XDP_RX_QUEUE_CONFIG_CREATE RxQueueConfig, + _In_ XDP_RX_DESCRIPTOR_CONTEXTS *DescriptorContexts + ); + +typedef +VOID +XDP_RX_QUEUE_SET_POLL_INFO( + _In_ XDP_RX_QUEUE_CONFIG_CREATE RxQueueConfig, + _In_ XDP_POLL_INFO *PollInfo + ); + +typedef +VOID +XDP_RX_QUEUE_GET_EXTENSION( + _In_ XDP_RX_QUEUE_CONFIG_ACTIVATE RxQueueConfig, + _In_ XDP_EXTENSION_INFO *ExtensionInfo, + _Out_ XDP_EXTENSION *Extension + ); + +typedef +XDP_RING * +XDP_RX_QUEUE_GET_RING( + _In_ XDP_RX_QUEUE_CONFIG_ACTIVATE RxQueueConfig + ); + +typedef +BOOLEAN +XDP_RX_QUEUE_ACTIVATE_IS_ENABLED( + _In_ XDP_RX_QUEUE_CONFIG_ACTIVATE RxQueueConfig + ); + +typedef struct _XDP_RX_QUEUE_CONFIG_CREATE_DISPATCH { + XDP_OBJECT_HEADER Header; + CONST VOID *Reserved; + XDP_RX_QUEUE_GET_TARGET_QUEUE_INFO *GetTargetQueueInfo; + XDP_RX_QUEUE_REGISTER_EXTENSION_VERSION *RegisterExtensionVersion; + XDP_RX_QUEUE_SET_CAPABILITIES *SetRxQueueCapabilities; + XDP_RX_QUEUE_SET_DESCRIPTOR_CONTEXTS *SetRxDescriptorContexts; + XDP_RX_QUEUE_SET_POLL_INFO *SetPollInfo; +} XDP_RX_QUEUE_CONFIG_CREATE_DISPATCH; + +#define XDP_RX_QUEUE_CONFIG_CREATE_DISPATCH_REVISION_1 1 + +#define XDP_SIZEOF_RX_QUEUE_CONFIG_CREATE_DISPATCH_REVISION_1 \ + RTL_SIZEOF_THROUGH_FIELD(XDP_RX_QUEUE_CONFIG_CREATE_DISPATCH, SetPollInfo) + +typedef struct _XDP_RX_QUEUE_CONFIG_CREATE_DETAILS { + CONST XDP_RX_QUEUE_CONFIG_CREATE_DISPATCH *Dispatch; +} XDP_RX_QUEUE_CONFIG_CREATE_DETAILS; + +typedef struct _XDP_RX_QUEUE_CONFIG_ACTIVATE_DISPATCH { + XDP_OBJECT_HEADER Header; + CONST VOID *Reserved; + XDP_RX_QUEUE_GET_RING *GetFrameRing; + XDP_RX_QUEUE_GET_RING *GetFragmentRing; + XDP_RX_QUEUE_GET_EXTENSION *GetExtension; + XDP_RX_QUEUE_ACTIVATE_IS_ENABLED *IsVirtualAddressEnabled; +} XDP_RX_QUEUE_CONFIG_ACTIVATE_DISPATCH; + +#define XDP_RX_QUEUE_CONFIG_ACTIVATE_DISPATCH_REVISION_1 1 + +#define XDP_SIZEOF_RX_QUEUE_CONFIG_ACTIVATE_DISPATCH_REVISION_1 \ + RTL_SIZEOF_THROUGH_FIELD(XDP_RX_QUEUE_CONFIG_ACTIVATE_DISPATCH, IsVirtualAddressEnabled) + +typedef struct _XDP_RX_QUEUE_CONFIG_ACTIVATE_DETAILS { + CONST XDP_RX_QUEUE_CONFIG_ACTIVATE_DISPATCH *Dispatch; +} XDP_RX_QUEUE_CONFIG_ACTIVATE_DETAILS; + +inline +CONST XDP_QUEUE_INFO * +XDPEXPORT(XdpRxQueueGetTargetQueueInfo)( + _In_ XDP_RX_QUEUE_CONFIG_CREATE RxQueueConfig + ) +{ + XDP_RX_QUEUE_CONFIG_CREATE_DETAILS *Details = (XDP_RX_QUEUE_CONFIG_CREATE_DETAILS *)RxQueueConfig; + return Details->Dispatch->GetTargetQueueInfo(RxQueueConfig); +} + +inline +VOID +XDPEXPORT(XdpRxQueueSetCapabilities)( + _In_ XDP_RX_QUEUE_CONFIG_CREATE RxQueueConfig, + _In_ XDP_RX_CAPABILITIES *Capabilities + ) +{ + XDP_RX_QUEUE_CONFIG_CREATE_DETAILS *Details = (XDP_RX_QUEUE_CONFIG_CREATE_DETAILS *)RxQueueConfig; + Details->Dispatch->SetRxQueueCapabilities(RxQueueConfig, Capabilities); +} + +inline +VOID +XDPEXPORT(XdpRxQueueRegisterExtensionVersion)( + _In_ XDP_RX_QUEUE_CONFIG_CREATE RxQueueConfig, + _In_ XDP_EXTENSION_INFO *ExtensionInfo + ) +{ + XDP_RX_QUEUE_CONFIG_CREATE_DETAILS *Details = (XDP_RX_QUEUE_CONFIG_CREATE_DETAILS *)RxQueueConfig; + Details->Dispatch->RegisterExtensionVersion(RxQueueConfig, ExtensionInfo); +} + +inline +VOID +XDPEXPORT(XdpRxQueueSetDescriptorContexts)( + _In_ XDP_RX_QUEUE_CONFIG_CREATE RxQueueConfig, + _In_ XDP_RX_DESCRIPTOR_CONTEXTS *DescriptorContexts + ) +{ + XDP_RX_QUEUE_CONFIG_CREATE_DETAILS *Details = (XDP_RX_QUEUE_CONFIG_CREATE_DETAILS *)RxQueueConfig; + Details->Dispatch->SetRxDescriptorContexts(RxQueueConfig, DescriptorContexts); +} + +inline +VOID +XDPEXPORT(XdpRxQueueSetPollInfo)( + _In_ XDP_RX_QUEUE_CONFIG_CREATE RxQueueConfig, + _In_ XDP_POLL_INFO *PollInfo + ) +{ + XDP_RX_QUEUE_CONFIG_CREATE_DETAILS *Details = (XDP_RX_QUEUE_CONFIG_CREATE_DETAILS *)RxQueueConfig; + Details->Dispatch->SetPollInfo(RxQueueConfig, PollInfo); +} + +inline +XDP_RING * +XDPEXPORT(XdpRxQueueGetFrameRing)( + _In_ XDP_RX_QUEUE_CONFIG_ACTIVATE RxQueueConfig + ) +{ + XDP_RX_QUEUE_CONFIG_ACTIVATE_DETAILS *Details = (XDP_RX_QUEUE_CONFIG_ACTIVATE_DETAILS *)RxQueueConfig; + return Details->Dispatch->GetFrameRing(RxQueueConfig); +} + +inline +XDP_RING * +XDPEXPORT(XdpRxQueueGetFragmentRing)( + _In_ XDP_RX_QUEUE_CONFIG_ACTIVATE RxQueueConfig + ) +{ + XDP_RX_QUEUE_CONFIG_ACTIVATE_DETAILS *Details = (XDP_RX_QUEUE_CONFIG_ACTIVATE_DETAILS *)RxQueueConfig; + return Details->Dispatch->GetFragmentRing(RxQueueConfig); +} + +inline +VOID +XDPEXPORT(XdpRxQueueGetExtension)( + _In_ XDP_RX_QUEUE_CONFIG_ACTIVATE RxQueueConfig, + _In_ XDP_EXTENSION_INFO *ExtensionInfo, + _Out_ XDP_EXTENSION *Extension + ) +{ + XDP_RX_QUEUE_CONFIG_ACTIVATE_DETAILS *Details = (XDP_RX_QUEUE_CONFIG_ACTIVATE_DETAILS *)RxQueueConfig; + Details->Dispatch->GetExtension(RxQueueConfig, ExtensionInfo, Extension); +} + +inline +BOOLEAN +XDPEXPORT(XdpRxQueueIsVirtualAddressEnabled)( + _In_ XDP_RX_QUEUE_CONFIG_ACTIVATE RxQueueConfig + ) +{ + XDP_RX_QUEUE_CONFIG_ACTIVATE_DETAILS *Details = (XDP_RX_QUEUE_CONFIG_ACTIVATE_DETAILS *)RxQueueConfig; + return Details->Dispatch->IsVirtualAddressEnabled(RxQueueConfig); +} + +EXTERN_C_END diff --git a/src/xdp/include/xdp/details/txqueueconfig.h b/src/xdp/include/xdp/details/txqueueconfig.h new file mode 100644 index 00000000..e89cb084 --- /dev/null +++ b/src/xdp/include/xdp/details/txqueueconfig.h @@ -0,0 +1,239 @@ +// +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// + +#pragma once + +EXTERN_C_START + +#include +#include + +typedef +CONST XDP_QUEUE_INFO * +XDP_TX_QUEUE_GET_TARGET_QUEUE_INFO( + _In_ XDP_TX_QUEUE_CONFIG_CREATE TxQueueConfig + ); + +typedef +BOOLEAN +XDP_TX_QUEUE_CREATE_IS_ENABLED( + _In_ XDP_TX_QUEUE_CONFIG_CREATE TxQueueConfig + ); + +typedef +VOID +XDP_TX_QUEUE_REGISTER_EXTENSION_VERSION( + _In_ XDP_TX_QUEUE_CONFIG_CREATE TxQueueConfig, + _In_ XDP_EXTENSION_INFO *ExtensionInfo + ); + +typedef +VOID +XDP_TX_QUEUE_SET_CAPABILITIES( + _In_ XDP_TX_QUEUE_CONFIG_CREATE TxQueueConfig, + _In_ XDP_TX_CAPABILITIES *Capabilities + ); + +typedef +VOID +XDP_TX_QUEUE_SET_DESCRIPTOR_CONTEXTS( + _In_ XDP_TX_QUEUE_CONFIG_CREATE TxQueueConfig, + _In_ XDP_TX_DESCRIPTOR_CONTEXTS *DescriptorContexts + ); + +typedef +VOID +XDP_TX_QUEUE_SET_POLL_INFO( + _In_ XDP_TX_QUEUE_CONFIG_CREATE TxQueueConfig, + _In_ XDP_POLL_INFO *PollInfo + ); + +typedef +VOID +XDP_TX_QUEUE_GET_EXTENSION( + _In_ XDP_TX_QUEUE_CONFIG_ACTIVATE TxQueueConfig, + _In_ XDP_EXTENSION_INFO *ExtensionInfo, + _Out_ XDP_EXTENSION *Extension + ); + +typedef +XDP_RING * +XDP_TX_QUEUE_GET_RING( + _In_ XDP_TX_QUEUE_CONFIG_ACTIVATE TxQueueConfig + ); + +typedef +BOOLEAN +XDP_TX_QUEUE_ACTIVATE_IS_ENABLED( + _In_ XDP_TX_QUEUE_CONFIG_ACTIVATE TxQueueConfig + ); + +typedef struct _XDP_TX_QUEUE_CONFIG_CREATE_DISPATCH { + XDP_OBJECT_HEADER Header; + CONST VOID *Reserved; + XDP_TX_QUEUE_GET_TARGET_QUEUE_INFO *GetTargetQueueInfo; + XDP_TX_QUEUE_REGISTER_EXTENSION_VERSION *RegisterExtensionVersion; + XDP_TX_QUEUE_SET_CAPABILITIES *SetTxQueueCapabilities; + XDP_TX_QUEUE_SET_DESCRIPTOR_CONTEXTS *SetTxDescriptorContexts; + XDP_TX_QUEUE_SET_POLL_INFO *SetPollInfo; +} XDP_TX_QUEUE_CONFIG_CREATE_DISPATCH; + +#define XDP_TX_QUEUE_CONFIG_CREATE_DISPATCH_REVISION_1 1 + +#define XDP_SIZEOF_TX_QUEUE_CONFIG_CREATE_DISPATCH_REVISION_1 \ + RTL_SIZEOF_THROUGH_FIELD(XDP_TX_QUEUE_CONFIG_CREATE_DISPATCH, SetPollInfo) + +typedef struct _XDP_TX_QUEUE_CONFIG_CREATE_DETAILS { + CONST XDP_TX_QUEUE_CONFIG_CREATE_DISPATCH *Dispatch; +} XDP_TX_QUEUE_CONFIG_CREATE_DETAILS; + +typedef struct _XDP_TX_QUEUE_CONFIG_ACTIVATE_DISPATCH { + XDP_OBJECT_HEADER Header; + CONST VOID *Reserved; + XDP_TX_QUEUE_GET_RING *GetFrameRing; + XDP_TX_QUEUE_GET_RING *GetFragmentRing; + XDP_TX_QUEUE_GET_RING *GetCompletionRing; + XDP_TX_QUEUE_GET_EXTENSION *GetExtension; + XDP_TX_QUEUE_ACTIVATE_IS_ENABLED *IsTxCompletionContextEnabled; + XDP_TX_QUEUE_ACTIVATE_IS_ENABLED *IsFragmentationEnabled; + XDP_TX_QUEUE_ACTIVATE_IS_ENABLED *IsOutOfOrderCompletionEnabled; +} XDP_TX_QUEUE_CONFIG_ACTIVATE_DISPATCH; + +#define XDP_TX_QUEUE_CONFIG_ACTIVATE_DISPATCH_REVISION_1 1 + +#define XDP_SIZEOF_TX_QUEUE_CONFIG_ACTIVATE_DISPATCH_REVISION_1 \ + RTL_SIZEOF_THROUGH_FIELD(XDP_TX_QUEUE_CONFIG_ACTIVATE_DISPATCH, IsOutOfOrderCompletionEnabled) + +typedef struct _XDP_TX_QUEUE_CONFIG_ACTIVATE_DETAILS { + CONST XDP_TX_QUEUE_CONFIG_ACTIVATE_DISPATCH *Dispatch; +} XDP_TX_QUEUE_CONFIG_ACTIVATE_DETAILS; + +inline +CONST XDP_QUEUE_INFO * +XDPEXPORT(XdpTxQueueGetTargetQueueInfo)( + _In_ XDP_TX_QUEUE_CONFIG_CREATE TxQueueConfig + ) +{ + XDP_TX_QUEUE_CONFIG_CREATE_DETAILS *Details = (XDP_TX_QUEUE_CONFIG_CREATE_DETAILS *)TxQueueConfig; + return Details->Dispatch->GetTargetQueueInfo(TxQueueConfig); +} + +inline +VOID +XDPEXPORT(XdpTxQueueSetCapabilities)( + _In_ XDP_TX_QUEUE_CONFIG_CREATE TxQueueConfig, + _In_ XDP_TX_CAPABILITIES *Capabilities + ) +{ + XDP_TX_QUEUE_CONFIG_CREATE_DETAILS *Details = (XDP_TX_QUEUE_CONFIG_CREATE_DETAILS *)TxQueueConfig; + Details->Dispatch->SetTxQueueCapabilities(TxQueueConfig, Capabilities); +} + +inline +VOID +XDPEXPORT(XdpTxQueueRegisterExtensionVersion)( + _In_ XDP_TX_QUEUE_CONFIG_CREATE TxQueueConfig, + _In_ XDP_EXTENSION_INFO *ExtensionInfo + ) +{ + XDP_TX_QUEUE_CONFIG_CREATE_DETAILS *Details = (XDP_TX_QUEUE_CONFIG_CREATE_DETAILS *)TxQueueConfig; + Details->Dispatch->RegisterExtensionVersion(TxQueueConfig, ExtensionInfo); +} + +inline +VOID +XDPEXPORT(XdpTxQueueSetDescriptorContexts)( + _In_ XDP_TX_QUEUE_CONFIG_CREATE TxQueueConfig, + _In_ XDP_TX_DESCRIPTOR_CONTEXTS *DescriptorContexts + ) +{ + XDP_TX_QUEUE_CONFIG_CREATE_DETAILS *Details = (XDP_TX_QUEUE_CONFIG_CREATE_DETAILS *)TxQueueConfig; + Details->Dispatch->SetTxDescriptorContexts(TxQueueConfig, DescriptorContexts); +} + +inline +VOID +XDPEXPORT(XdpTxQueueSetPollInfo)( + _In_ XDP_TX_QUEUE_CONFIG_CREATE TxQueueConfig, + _In_ XDP_POLL_INFO *PollInfo + ) +{ + XDP_TX_QUEUE_CONFIG_CREATE_DETAILS *Details = (XDP_TX_QUEUE_CONFIG_CREATE_DETAILS *)TxQueueConfig; + Details->Dispatch->SetPollInfo(TxQueueConfig, PollInfo); +} + +inline +XDP_RING * +XDPEXPORT(XdpTxQueueGetFrameRing)( + _In_ XDP_TX_QUEUE_CONFIG_ACTIVATE TxQueueConfig + ) +{ + XDP_TX_QUEUE_CONFIG_ACTIVATE_DETAILS *Details = (XDP_TX_QUEUE_CONFIG_ACTIVATE_DETAILS *)TxQueueConfig; + return Details->Dispatch->GetFrameRing(TxQueueConfig); +} + +inline +XDP_RING * +XDPEXPORT(XdpTxQueueGetFragmentRing)( + _In_ XDP_TX_QUEUE_CONFIG_ACTIVATE TxQueueConfig + ) +{ + XDP_TX_QUEUE_CONFIG_ACTIVATE_DETAILS *Details = (XDP_TX_QUEUE_CONFIG_ACTIVATE_DETAILS *)TxQueueConfig; + return Details->Dispatch->GetFragmentRing(TxQueueConfig); +} + +inline +XDP_RING * +XDPEXPORT(XdpTxQueueGetCompletionRing)( + _In_ XDP_TX_QUEUE_CONFIG_ACTIVATE TxQueueConfig + ) +{ + XDP_TX_QUEUE_CONFIG_ACTIVATE_DETAILS *Details = (XDP_TX_QUEUE_CONFIG_ACTIVATE_DETAILS *)TxQueueConfig; + return Details->Dispatch->GetCompletionRing(TxQueueConfig); +} + +inline +VOID +XDPEXPORT(XdpTxQueueGetExtension)( + _In_ XDP_TX_QUEUE_CONFIG_ACTIVATE TxQueueConfig, + _In_ XDP_EXTENSION_INFO *ExtensionInfo, + _Out_ XDP_EXTENSION *Extension + ) +{ + XDP_TX_QUEUE_CONFIG_ACTIVATE_DETAILS *Details = (XDP_TX_QUEUE_CONFIG_ACTIVATE_DETAILS *)TxQueueConfig; + Details->Dispatch->GetExtension(TxQueueConfig, ExtensionInfo, Extension); +} + +inline +BOOLEAN +XDPEXPORT(XdpTxQueueIsTxCompletionContextEnabled)( + _In_ XDP_TX_QUEUE_CONFIG_ACTIVATE TxQueueConfig + ) +{ + XDP_TX_QUEUE_CONFIG_ACTIVATE_DETAILS *Details = (XDP_TX_QUEUE_CONFIG_ACTIVATE_DETAILS *)TxQueueConfig; + return Details->Dispatch->IsTxCompletionContextEnabled(TxQueueConfig); +} + +inline +BOOLEAN +XDPEXPORT(XdpTxQueueIsFragmentationEnabled)( + _In_ XDP_TX_QUEUE_CONFIG_ACTIVATE TxQueueConfig + ) +{ + XDP_TX_QUEUE_CONFIG_ACTIVATE_DETAILS *Details = (XDP_TX_QUEUE_CONFIG_ACTIVATE_DETAILS *)TxQueueConfig; + return Details->Dispatch->IsFragmentationEnabled(TxQueueConfig); +} + +inline +BOOLEAN +XDPEXPORT(XdpTxQueueIsOutOfOrderCompletionEnabled)( + _In_ XDP_TX_QUEUE_CONFIG_ACTIVATE TxQueueConfig + ) +{ + XDP_TX_QUEUE_CONFIG_ACTIVATE_DETAILS *Details = (XDP_TX_QUEUE_CONFIG_ACTIVATE_DETAILS *)TxQueueConfig; + return Details->Dispatch->IsOutOfOrderCompletionEnabled(TxQueueConfig); +} + +EXTERN_C_END diff --git a/src/xdp/include/xdp/dma.h b/src/xdp/include/xdp/dma.h new file mode 100644 index 00000000..eff3c219 --- /dev/null +++ b/src/xdp/include/xdp/dma.h @@ -0,0 +1,27 @@ +// +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// + +#pragma once + +EXTERN_C_START + +typedef struct _XDP_DMA_CAPABILITIES { + ULONG Size; + DEVICE_OBJECT *PhysicalDeviceObject; +} XDP_DMA_CAPABILITIES; + +inline +VOID +XdpInitializeDmaCapabilitiesPdo( + _Out_ XDP_DMA_CAPABILITIES *Capabilities, + _In_ DEVICE_OBJECT *PhysicalDeviceObject + ) +{ + RtlZeroMemory(Capabilities, sizeof(*Capabilities)); + Capabilities->Size = sizeof(*Capabilities); + Capabilities->PhysicalDeviceObject = PhysicalDeviceObject; +} + +EXTERN_C_END diff --git a/src/xdp/include/xdp/driverapi.h b/src/xdp/include/xdp/driverapi.h new file mode 100644 index 00000000..6227488e --- /dev/null +++ b/src/xdp/include/xdp/driverapi.h @@ -0,0 +1,14 @@ +// +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// + +#pragma once + +EXTERN_C_START + +#define XDP_DRIVER_API_MAJOR_VER 0 +#define XDP_DRIVER_API_MINOR_VER 10 +#define XDP_DRIVER_API_PATCH_VER 0 + +EXTERN_C_END diff --git a/src/xdp/include/xdp/extension.h b/src/xdp/include/xdp/extension.h new file mode 100644 index 00000000..06c06798 --- /dev/null +++ b/src/xdp/include/xdp/extension.h @@ -0,0 +1,33 @@ +// +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// + +#ifndef XDP_EXTENSION_H +#define XDP_EXTENSION_H + +#ifdef __cplusplus +extern "C" { +#endif + +#pragma warning(push) +#pragma warning(default:4820) // warn if the compiler inserted padding + +typedef struct _XDP_EXTENSION { + // + // This field is reserved for XDP platform use. + // + UINT16 Reserved; +} XDP_EXTENSION; + +C_ASSERT(sizeof(XDP_EXTENSION) == 2); + +#pragma warning(pop) + +#include + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif diff --git a/src/xdp/include/xdp/extensioninfo.h b/src/xdp/include/xdp/extensioninfo.h new file mode 100644 index 00000000..5cc5207f --- /dev/null +++ b/src/xdp/include/xdp/extensioninfo.h @@ -0,0 +1,55 @@ +// +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// + +#ifndef XDP_EXTENSION_INFO_H +#define XDP_EXTENSION_INFO_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef enum _XDP_EXTENSION_TYPE { + XDP_EXTENSION_TYPE_FRAME, + XDP_EXTENSION_TYPE_BUFFER, + XDP_EXTENSION_TYPE_TX_FRAME_COMPLETION, +} XDP_EXTENSION_TYPE; + +typedef struct _XDP_EXTENSION_INFO { + XDP_OBJECT_HEADER Header; + _Null_terminated_ CONST WCHAR *ExtensionName; + UINT32 ExtensionVersion; + XDP_EXTENSION_TYPE ExtensionType; +} XDP_EXTENSION_INFO; + +#define XDP_EXTENSION_INFO_REVISION_1 1 + +#define XDP_SIZEOF_EXTENSION_INFO_REVISION_1 \ + RTL_SIZEOF_THROUGH_FIELD(XDP_EXTENSION_INFO, ExtensionType) + +inline +VOID +XdpInitializeExtensionInfo( + _Out_ XDP_EXTENSION_INFO *ExtensionInfo, + _In_z_ CONST WCHAR *ExtensionName, + _In_ UINT32 ExtensionVersion, + _In_ XDP_EXTENSION_TYPE ExtensionType + ) +{ + RtlZeroMemory(ExtensionInfo, sizeof(*ExtensionInfo)); + ExtensionInfo->Header.Revision = XDP_EXTENSION_INFO_REVISION_1; + ExtensionInfo->Header.Size = XDP_SIZEOF_EXTENSION_INFO_REVISION_1; + + ExtensionInfo->ExtensionName = ExtensionName; + ExtensionInfo->ExtensionVersion = ExtensionVersion; + ExtensionInfo->ExtensionType = ExtensionType; +} + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif diff --git a/src/xdp/include/xdp/framefragment.h b/src/xdp/include/xdp/framefragment.h new file mode 100644 index 00000000..b9be8350 --- /dev/null +++ b/src/xdp/include/xdp/framefragment.h @@ -0,0 +1,37 @@ +// +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// + +#pragma once + +EXTERN_C_START + +#pragma warning(push) +#pragma warning(default:4820) // warn if the compiler inserted padding + +typedef struct _XDP_FRAME_FRAGMENT { + UINT8 FragmentBufferCount; +} XDP_FRAME_FRAGMENT; + +C_ASSERT(sizeof(XDP_FRAME_FRAGMENT) == 1); + +#pragma warning(pop) + +#define XDP_FRAME_EXTENSION_FRAGMENT_NAME L"ms_frame_fragment" +#define XDP_FRAME_EXTENSION_FRAGMENT_VERSION_1 1U + +#include +#include + +inline +XDP_FRAME_FRAGMENT * +XdpGetFragmentExtension( + _In_ XDP_FRAME *Frame, + _In_ XDP_EXTENSION *Extension + ) +{ + return (XDP_FRAME_FRAGMENT *)XdpGetExtensionData(Frame, Extension); +} + +EXTERN_C_END diff --git a/src/xdp/include/xdp/frameinterfacecontext.h b/src/xdp/include/xdp/frameinterfacecontext.h new file mode 100644 index 00000000..1a03a5d8 --- /dev/null +++ b/src/xdp/include/xdp/frameinterfacecontext.h @@ -0,0 +1,28 @@ +// +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// + +#pragma once + +EXTERN_C_START + +typedef VOID XDP_FRAME_INTERFACE_CONTEXT; + +#define XDP_FRAME_EXTENSION_INTERFACE_CONTEXT_NAME L"ms_frame_interface_context" +#define XDP_FRAME_EXTENSION_INTERFACE_CONTEXT_VERSION_1 1U + +#include +#include + +inline +XDP_FRAME_INTERFACE_CONTEXT * +XdpGetFrameInterfaceContextExtension( + _In_ XDP_FRAME *Frame, + _In_ XDP_EXTENSION *Extension + ) +{ + return (XDP_FRAME_INTERFACE_CONTEXT *)XdpGetExtensionData(Frame, Extension); +} + +EXTERN_C_END diff --git a/src/xdp/include/xdp/framerxaction.h b/src/xdp/include/xdp/framerxaction.h new file mode 100644 index 00000000..0164ef68 --- /dev/null +++ b/src/xdp/include/xdp/framerxaction.h @@ -0,0 +1,37 @@ +// +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// + +#pragma once + +EXTERN_C_START + +#pragma warning(push) +#pragma warning(default:4820) // warn if the compiler inserted padding + +typedef struct _XDP_FRAME_RX_ACTION { + UINT8 RxAction; +} XDP_FRAME_RX_ACTION; + +C_ASSERT(sizeof(XDP_FRAME_RX_ACTION) == 1); + +#pragma warning(pop) + +#define XDP_FRAME_EXTENSION_RX_ACTION_NAME L"ms_frame_rx_action" +#define XDP_FRAME_EXTENSION_RX_ACTION_VERSION_1 1U + +#include +#include + +inline +XDP_FRAME_RX_ACTION * +XdpGetRxActionExtension( + _In_ XDP_FRAME *Frame, + _In_ XDP_EXTENSION *Extension + ) +{ + return (XDP_FRAME_RX_ACTION *)XdpGetExtensionData(Frame, Extension); +} + +EXTERN_C_END diff --git a/src/xdp/include/xdp/guid.h b/src/xdp/include/xdp/guid.h new file mode 100644 index 00000000..e047ea84 --- /dev/null +++ b/src/xdp/include/xdp/guid.h @@ -0,0 +1,16 @@ +// +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// + +#pragma once + +EXTERN_C_START + +_IRQL_requires_max_(PASSIVE_LEVEL) +NTSTATUS +XdpGuidCreate( + _Out_ GUID *Guid + ); + +EXTERN_C_END diff --git a/src/xdp/include/xdp/hookid.h b/src/xdp/include/xdp/hookid.h new file mode 100644 index 00000000..d27025e8 --- /dev/null +++ b/src/xdp/include/xdp/hookid.h @@ -0,0 +1,43 @@ +// +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// + +#ifndef HOOKID_H +#define HOOKID_H + +#ifdef __cplusplus +extern "C" { +#endif + +typedef enum _XDP_HOOK_LAYER { + XDP_HOOK_L2, +} XDP_HOOK_LAYER; + +typedef enum _XDP_HOOK_DATAPATH_DIRECTION { + XDP_HOOK_RX, + XDP_HOOK_TX, +} XDP_HOOK_DATAPATH_DIRECTION; + +typedef enum _XDP_HOOK_SUBLAYER { + XDP_HOOK_INSPECT, + XDP_HOOK_INJECT, +} XDP_HOOK_SUBLAYER; + +typedef struct _XDP_HOOK_ID { + XDP_HOOK_LAYER Layer; + XDP_HOOK_DATAPATH_DIRECTION Direction; + XDP_HOOK_SUBLAYER SubLayer; +} XDP_HOOK_ID; + +C_ASSERT( + sizeof(XDP_HOOK_ID) == + sizeof(XDP_HOOK_LAYER) + + sizeof(XDP_HOOK_DATAPATH_DIRECTION) + + sizeof(XDP_HOOK_SUBLAYER)); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif diff --git a/src/xdp/include/xdp/interfaceconfig.h b/src/xdp/include/xdp/interfaceconfig.h new file mode 100644 index 00000000..901957a6 --- /dev/null +++ b/src/xdp/include/xdp/interfaceconfig.h @@ -0,0 +1,22 @@ +// +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// + +#pragma once + +EXTERN_C_START + +#include +#include + +DECLARE_HANDLE(XDP_INTERFACE_CONFIG); + +CONST XDP_VERSION * +XdpGetDriverApiVersion( + _In_ XDP_INTERFACE_CONFIG InterfaceConfig + ); + +#include + +EXTERN_C_END diff --git a/src/xdp/include/xdp/ndis6.h b/src/xdp/include/xdp/ndis6.h new file mode 100644 index 00000000..8e79658a --- /dev/null +++ b/src/xdp/include/xdp/ndis6.h @@ -0,0 +1,23 @@ +// +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// + +#pragma once + +EXTERN_C_START + +// +// Redefine NDIS6 OID in XDP headers until an updated DDK is released. +// + +// +// An XDP-capable NDIS6 interface must respond to OID_XDP_QUERY_CAPABILITIES by +// providing the interface's registered XDP_CAPABILITIES structure. This OID is +// an NDIS query OID. +// +#ifndef OID_XDP_QUERY_CAPABILITIES +#define OID_XDP_QUERY_CAPABILITIES 0x00A00204 +#endif + +EXTERN_C_END diff --git a/src/xdp/include/xdp/ndis6poll.h b/src/xdp/include/xdp/ndis6poll.h new file mode 100644 index 00000000..0cb942cf --- /dev/null +++ b/src/xdp/include/xdp/ndis6poll.h @@ -0,0 +1,85 @@ +// +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// + +#pragma once + +// +// This file contains tentative definitions of NDIS polling API extensions for +// XDP as well as a compatibility shim for older NDIS polling API versions. +// + +EXTERN_C_START + +#if NDIS_SUPPORT_NDIS685 + +#include + +typedef struct _XDP_POLL_TRANSMIT_DATA { + // + // The number of TX frames completed to the XDP platform. + // + UINT32 FramesCompleted; + + // + // The number of TX frames injected from the XDP platform. + // + UINT32 FramesTransmitted; +} XDP_POLL_TRANSMIT_DATA; + +typedef struct _XDP_POLL_RECEIVE_DATA { + // + // The number of RX frames absorbed by the XDP platform. + // + UINT32 FramesAbsorbed; +} XDP_POLL_RECEIVE_DATA; + +typedef struct _XDP_POLL_DATA { + XDP_POLL_TRANSMIT_DATA Transmit; + XDP_POLL_RECEIVE_DATA Receive; +} XDP_POLL_DATA; + +typedef +_IRQL_requires_max_(HIGH_LEVEL) +VOID +XDP_NDIS_REQUEST_POLL( + _In_ NDIS_POLL_HANDLE PollHandle, + _Reserved_ VOID *Reserved + ); + +// +// This routine provides compatibility with NDIS polling APIs that lack support +// for XDP polling extensions. XDP interface drivers must invoke this helper (or +// a similar custom routine) prior to returning from their NDIS poll callback. +// +inline +_IRQL_requires_max_(DISPATCH_LEVEL) +VOID +XdpCompleteNdisPoll( + _In_ NDIS_POLL_HANDLE PollHandle, + _In_ NDIS_POLL_DATA *Poll, + _In_ XDP_POLL_DATA *XdpPoll, + _In_ XDP_NDIS_REQUEST_POLL *RequestPoll + ) +{ + if (Poll->Receive.IndicatedNblChain != NULL || Poll->Transmit.CompletedNblChain != NULL) { + // + // If NBL chains are returned to NDIS, a poll is implicitly requested. + // + return; + } + + if (XdpPoll->Transmit.FramesCompleted > 0 || XdpPoll->Transmit.FramesTransmitted > 0 || + XdpPoll->Receive.FramesAbsorbed > 0) { + // + // XDP made forward progress, and this was not observable to NDIS. + // Explicitly request another poll. + // + RequestPoll(PollHandle, NULL); + } +} + +#endif // NDIS_SUPPORT_NDIS685 + +EXTERN_C_END diff --git a/src/xdp/include/xdp/objectheader.h b/src/xdp/include/xdp/objectheader.h new file mode 100644 index 00000000..4ca6ecfc --- /dev/null +++ b/src/xdp/include/xdp/objectheader.h @@ -0,0 +1,22 @@ +// +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// + +#ifndef XDP_OBJECT_HEADER_H +#define XDP_OBJECT_HEADER_H + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct _XDP_OBJECT_HEADER { + UINT32 Revision; + UINT32 Size; +} XDP_OBJECT_HEADER; + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif diff --git a/src/xdp/include/xdp/offload.h b/src/xdp/include/xdp/offload.h new file mode 100644 index 00000000..f7fdd92f --- /dev/null +++ b/src/xdp/include/xdp/offload.h @@ -0,0 +1,156 @@ +// +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// + +#pragma once + +EXTERN_C_START + +// +// NOTE: The definitions in this header are for informational purposes only. +// All offload structures are currently unsupported by XDP and subject to +// significant changes. +// + +#pragma warning(push) +#pragma warning(default:4820) // warn if the compiler inserted padding +#pragma warning(disable:4201) // nonstandard extension used: nameless struct/union +#pragma warning(disable:4214) // nonstandard extension used: bit field types other than int + +typedef enum _XDP_FRAME_LAYER2_TYPE { + XdpFrameLayer2TypeUnspecified, + XdpFrameLayer2TypeNull, + XdpFrameLayer2TypeEthernet, +} XDP_FRAME_LAYER2_TYPE; + +typedef enum _XDP_FRAME_LAYER3_TYPE { + XdpFrameLayer3TypeUnspecified, + XdpFrameLayer3TypeIPv4UnspecifiedOptions, + XdpFrameLayer3TypeIPv4WithOptions, + XdpFrameLayer3TypeIPv4NoOptions, + XdpFrameLayer3TypeIPv6UnspecifiedExtensions, + XdpFrameLayer3TypeIPv6WithExtensions, + XdpFrameLayer3TypeIPv6NoExtensions, +} XDP_FRAME_LAYER3_TYPE; + +typedef enum _XDP_FRAME_LAYER4_TYPE { + XdpFrameLayer4TypeUnspecified, + XdpFrameLayer4TypeTcp, + XdpFrameLayer4TypeUdp, + XdpFrameLayer4TypeIPFragment, + XdpFrameLayer4TypeIPNotFragment, +} XDP_FRAME_LAYER4_TYPE; + +#include +typedef struct _XDP_FRAME_LAYOUT { + UINT16 Layer2HeaderLength : 7; + UINT16 Layer3HeaderLength : 9; + UINT8 Layer4HeaderLength : 8; + + // One of the XDP_FRAME_LAYER2_TYPE values + UINT8 Layer2Type : 4; + + // One of the XDP_FRAME_LAYER3_TYPE values + UINT8 Layer3Type : 4; + + // One of the XDP_FRAME_LAYER4_TYPE values + UINT8 Layer4Type : 4; + + UINT8 Reserved0 : 4; +} XDP_FRAME_LAYOUT; +#include + +C_ASSERT(sizeof(XDP_FRAME_LAYOUT) == 5); + +#pragma warning(pop) + +typedef enum _XDP_FRAME_TX_CHECKSUM_ACTION { + XdpFrameTxChecksumActionPassthrough = 0, + XdpFrameTxChecksumActionRequired = 1, +} XDP_FRAME_TX_CHECKSUM_ACTION; + +typedef enum _XDP_FRAME_RX_CHECKSUM_EVALUATION { + XdpFrameRxChecksumEvaluationNotChecked = 0, + XdpFrameRxChecksumEvaluationSucceeded = 1, + XdpFrameRxChecksumEvaluationFailed = 2, + XdpFrameRxChecksumEvaluationInvalid = 3, +} XDP_FRAME_RX_CHECKSUM_EVALUATION; + +#pragma warning(push) +#pragma warning(default:4820) // warn if the compiler inserted padding + +typedef struct _XDP_FRAME_CHECKSUM { + // One of XDP_FRAME_TX_CHECKSUM_ACTION or XDP_FRAME_RX_CHECKSUM_EVALUATION + UINT8 Layer3 : 2; + + // One of XDP_FRAME_TX_CHECKSUM_ACTION or XDP_FRAME_RX_CHECKSUM_EVALUATION + UINT8 Layer4 : 2; + + UINT8 Reserved : 4; +} XDP_FRAME_CHECKSUM; + +C_ASSERT(sizeof(XDP_FRAME_CHECKSUM) == 1); + +#pragma warning(pop) + +#pragma warning(push) +#pragma warning(default:4820) // warn if the compiler inserted padding +#pragma warning(disable:4214) // nonstandard extension used: bit field types other than int +#pragma warning(disable:4201) // nonstandard extension used: nameless struct/union + +typedef struct _XDP_FRAME_GSO { + union { + struct { + UINT32 Mss : 20; + UINT32 Reserved0 : 12; + } TCP; + struct { + UINT32 Mss : 20; + UINT32 Reserved0 : 12; + } UDP; + } DUMMYUNIONNAME; +} XDP_FRAME_GSO; + +C_ASSERT(sizeof(XDP_FRAME_GSO) == 4); + +#pragma warning(pop) + +#pragma warning(push) +#pragma warning(default:4820) // warn if the compiler inserted padding +#pragma warning(disable:4201) // nonstandard extension used: nameless struct/union + +typedef struct _XDP_FRAME_GRO { + union { + struct { + UINT16 CoalescedSegmentCount; + } TCP; + struct { + UINT16 MessageSize; + } UDP; + } DUMMYUNIONNAME; +} XDP_FRAME_GRO; + +C_ASSERT(sizeof(XDP_FRAME_GRO) == 2); + +typedef struct _XDP_FRAME_GRO_TIMESTAMP { + union { + struct { + UINT32 TcpTimestampDelta; + } TCP; + } DUMMYUNIONNAME; +} XDP_FRAME_GRO_TIMESTAMP; + +C_ASSERT(sizeof(XDP_FRAME_GRO_TIMESTAMP) == 4); + +#pragma warning(pop) + +typedef struct _XDP_FRAME_TIMESTAMP { + UINT64 Timestamp; +} XDP_FRAME_TIMESTAMP; + +C_ASSERT(sizeof(XDP_FRAME_TIMESTAMP) == 8); + +#pragma warning(pop) + +EXTERN_C_END diff --git a/src/xdp/include/xdp/pollinfo.h b/src/xdp/include/xdp/pollinfo.h new file mode 100644 index 00000000..7d91414c --- /dev/null +++ b/src/xdp/include/xdp/pollinfo.h @@ -0,0 +1,53 @@ +// +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// + +#pragma once + +EXTERN_C_START + +#include + +typedef struct _XDP_POLL_INFO { + XDP_OBJECT_HEADER Header; + NDIS_HANDLE PollHandle; + BOOLEAN Shared; +} XDP_POLL_INFO; + +#define XDP_POLL_INFO_REVISION_1 1 + +#define XDP_SIZEOF_POLL_INFO_REVISION_1 \ + RTL_SIZEOF_THROUGH_FIELD(XDP_POLL_INFO, Shared) + +inline +VOID +XdpInitializeSharedPollInfo( + _Out_ XDP_POLL_INFO *PollInfo, + _In_ NDIS_HANDLE PollHandle + ) +{ + RtlZeroMemory(PollInfo, sizeof(*PollInfo)); + PollInfo->Header.Revision = XDP_POLL_INFO_REVISION_1; + PollInfo->Header.Size = XDP_SIZEOF_POLL_INFO_REVISION_1; + + PollInfo->PollHandle = PollHandle; + PollInfo->Shared = TRUE; +} + +inline +VOID +XdpInitializeExclusivePollInfo( + _Out_ XDP_POLL_INFO *PollInfo, + _In_ NDIS_HANDLE PollHandle + ) +{ + RtlZeroMemory(PollInfo, sizeof(*PollInfo)); + PollInfo->Header.Revision = XDP_POLL_INFO_REVISION_1; + PollInfo->Header.Size = XDP_SIZEOF_POLL_INFO_REVISION_1; + + PollInfo->PollHandle = PollHandle; + PollInfo->Shared = FALSE; +} + +EXTERN_C_END diff --git a/src/xdp/include/xdp/program.h b/src/xdp/include/xdp/program.h new file mode 100644 index 00000000..09cc004e --- /dev/null +++ b/src/xdp/include/xdp/program.h @@ -0,0 +1,127 @@ +// +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// + +#ifndef XDPPROGRAM_H +#define XDPPROGRAM_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#pragma warning(push) +#pragma warning(disable:4201) // nonstandard extension used: nameless struct/union + +typedef enum _XDP_MATCH_TYPE { + XDP_MATCH_ALL, + XDP_MATCH_UDP, + XDP_MATCH_UDP_DST, + XDP_MATCH_IPV4_DST_MASK, + XDP_MATCH_IPV6_DST_MASK, + XDP_MATCH_QUIC_FLOW_SRC_CID, + XDP_MATCH_QUIC_FLOW_DST_CID, + XDP_MATCH_IPV4_UDP_TUPLE, + XDP_MATCH_IPV6_UDP_TUPLE, + XDP_MATCH_UDP_PORT_SET, + XDP_MATCH_IPV4_UDP_PORT_SET, + XDP_MATCH_IPV6_UDP_PORT_SET, + XDP_MATCH_IPV4_TCP_PORT_SET, + XDP_MATCH_IPV6_TCP_PORT_SET, + XDP_MATCH_TCP_DST, + XDP_MATCH_TCP_QUIC_FLOW_SRC_CID, + XDP_MATCH_TCP_QUIC_FLOW_DST_CID, + XDP_MATCH_TCP_CONTROL_DST, +} XDP_MATCH_TYPE; + +typedef union _XDP_INET_ADDR { + IN_ADDR Ipv4; + IN6_ADDR Ipv6; +} XDP_INET_ADDR; + +typedef struct _XDP_IP_ADDRESS_MASK { + XDP_INET_ADDR Mask; + XDP_INET_ADDR Address; +} XDP_IP_ADDRESS_MASK; + +typedef struct _XDP_TUPLE { + XDP_INET_ADDR SourceAddress; + XDP_INET_ADDR DestinationAddress; + UINT16 SourcePort; + UINT16 DestinationPort; +} XDP_TUPLE; + +#define XDP_QUIC_MAX_CID_LENGTH 20 + +typedef struct _XDP_QUIC_FLOW { + UINT16 UdpPort; + UCHAR CidLength; + UCHAR CidOffset; + UCHAR CidData[XDP_QUIC_MAX_CID_LENGTH]; // Max allowed per QUIC v1 RFC +} XDP_QUIC_FLOW; + +#define XDP_PORT_SET_BUFFER_SIZE ((MAXUINT16 + 1) / 8) + +typedef struct _XDP_PORT_SET { + const UINT8 *PortSet; + VOID *Reserved; +} XDP_PORT_SET; + +typedef struct _XDP_IP_PORT_SET { + XDP_INET_ADDR Address; + XDP_PORT_SET PortSet; +} XDP_IP_PORT_SET; + +typedef union _XDP_MATCH_PATTERN { + UINT16 Port; + XDP_IP_ADDRESS_MASK IpMask; + XDP_TUPLE Tuple; + XDP_QUIC_FLOW QuicFlow; + XDP_PORT_SET PortSet; + XDP_IP_PORT_SET IpPortSet; +} XDP_MATCH_PATTERN; + +typedef enum _XDP_RULE_ACTION { + XDP_PROGRAM_ACTION_DROP, + XDP_PROGRAM_ACTION_PASS, + XDP_PROGRAM_ACTION_REDIRECT, + XDP_PROGRAM_ACTION_L2FWD, + // + // Reserved. + // + XDP_PROGRAM_ACTION_EBPF, +} XDP_RULE_ACTION; + +typedef enum _XDP_REDIRECT_TARGET_TYPE { + XDP_REDIRECT_TARGET_TYPE_XSK, +} XDP_REDIRECT_TARGET_TYPE; + +typedef struct _XDP_REDIRECT_PARAMS { + XDP_REDIRECT_TARGET_TYPE TargetType; + HANDLE Target; +} XDP_REDIRECT_PARAMS; + +typedef struct _XDP_EBPF_PARAMS { + HANDLE Target; +} XDP_EBPF_PARAMS; + +typedef struct _XDP_RULE { + XDP_MATCH_TYPE Match; + XDP_MATCH_PATTERN Pattern; + XDP_RULE_ACTION Action; + union { + XDP_REDIRECT_PARAMS Redirect; + XDP_EBPF_PARAMS Ebpf; + }; +} XDP_RULE; + +#pragma warning(pop) + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif diff --git a/src/xdp/include/xdp/queueinfo.h b/src/xdp/include/xdp/queueinfo.h new file mode 100644 index 00000000..3de8e008 --- /dev/null +++ b/src/xdp/include/xdp/queueinfo.h @@ -0,0 +1,32 @@ +// +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// + +#ifndef XDP_QUEUE_INFO_H +#define XDP_QUEUE_INFO_H + +#ifdef __cplusplus +extern "C" { +#endif + +typedef enum _XDP_QUEUE_TYPE { + XDP_QUEUE_TYPE_DEFAULT_RSS, +} XDP_QUEUE_TYPE; + +typedef struct _XDP_QUEUE_INFO { + XDP_OBJECT_HEADER Header; + XDP_QUEUE_TYPE QueueType; + UINT32 QueueId; +} XDP_QUEUE_INFO; + +#define XDP_QUEUE_INFO_REVISION_1 1 + +#define XDP_SIZEOF_QUEUE_INFO_REVISION_1 \ + RTL_SIZEOF_THROUGH_FIELD(XDP_QUEUE_INFO, QueueId) + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif diff --git a/src/xdp/include/xdp/rtl.h b/src/xdp/include/xdp/rtl.h new file mode 100644 index 00000000..8aabed12 --- /dev/null +++ b/src/xdp/include/xdp/rtl.h @@ -0,0 +1,71 @@ +// +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// + +#ifndef XDP_RTL_H +#define XDP_RTL_H + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef RTL_PTR_ADD +#define RTL_PTR_ADD(Pointer, Value) \ + ((VOID *)((ULONG_PTR)(Pointer) + (ULONG_PTR)(Value))) +#endif + +#ifndef RTL_PTR_SUBTRACT +#define RTL_PTR_SUBTRACT(Pointer, Value) \ + ((PVOID)((ULONG_PTR)(Pointer) - (ULONG_PTR)(Value))) +#endif + +#if (!defined(NTDDI_WIN10_CO) || (WDK_NTDDI_VERSION < NTDDI_WIN10_CO)) && \ + !defined(UINT32_VOLATILE_ACCESSORS) +#define UINT32_VOLATILE_ACCESSORS + +FORCEINLINE +UINT32 +ReadUInt32Acquire( + _In_ _Interlocked_operand_ UINT32 const volatile *Source + ) +{ + return (UINT32)ReadULongAcquire((PULONG)Source); +} + +FORCEINLINE +UINT32 +ReadUInt32NoFence( + _In_ _Interlocked_operand_ UINT32 const volatile *Source + ) +{ + return (UINT32)ReadULongNoFence((PULONG)Source); +} + +FORCEINLINE +VOID +WriteUInt32Release( + _Out_ _Interlocked_operand_ UINT32 volatile *Destination, + _In_ UINT32 Value + ) +{ + WriteULongRelease((PULONG)Destination, (ULONG)Value); +} + +FORCEINLINE +VOID +WriteUInt32NoFence( + _Out_ _Interlocked_operand_ UINT32 volatile *Destination, + _In_ UINT32 Value + ) +{ + WriteULongNoFence((PULONG)Destination, (ULONG)Value); +} + +#endif + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif diff --git a/src/xdp/include/xdp/rxqueueconfig.h b/src/xdp/include/xdp/rxqueueconfig.h new file mode 100644 index 00000000..7bf9f5b3 --- /dev/null +++ b/src/xdp/include/xdp/rxqueueconfig.h @@ -0,0 +1,121 @@ +// +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// + +#pragma once + +EXTERN_C_START + +#include +#include +#include +#include +#include + +DECLARE_HANDLE(XDP_RX_QUEUE_CONFIG_CREATE); +DECLARE_HANDLE(XDP_RX_QUEUE_CONFIG_ACTIVATE); + +CONST XDP_QUEUE_INFO * +XdpRxQueueGetTargetQueueInfo( + _In_ XDP_RX_QUEUE_CONFIG_CREATE RxQueueConfig + ); + +typedef struct _XDP_RX_CAPABILITIES { + XDP_OBJECT_HEADER Header; + BOOLEAN VirtualAddressSupported; + UINT16 ReceiveFrameCountHint; + UINT8 MaximumFragments; + BOOLEAN TxActionSupported; +} XDP_RX_CAPABILITIES; + +#define XDP_RX_CAPABILITIES_REVISION_1 1 + +#define XDP_SIZEOF_RX_CAPABILITIES_REVISION_1 \ + RTL_SIZEOF_THROUGH_FIELD(XDP_RX_CAPABILITIES, TxActionSupported) + +inline +VOID +XdpInitializeRxCapabilitiesDriverVa( + _Out_ XDP_RX_CAPABILITIES *Capabilities + ) +{ + RtlZeroMemory(Capabilities, sizeof(*Capabilities)); + Capabilities->Header.Revision = XDP_RX_CAPABILITIES_REVISION_1; + Capabilities->Header.Size = XDP_SIZEOF_RX_CAPABILITIES_REVISION_1; + Capabilities->VirtualAddressSupported = TRUE; +} + +typedef struct _XDP_RX_DESCRIPTOR_CONTEXTS { + XDP_OBJECT_HEADER Header; + UINT8 FrameContextSize; + UINT8 FrameContextAlignment; + UINT8 BufferContextSize; + UINT8 BufferContextAlignment; +} XDP_RX_DESCRIPTOR_CONTEXTS; + +#define XDP_RX_DESCRIPTOR_CONTEXTS_REVISION_1 1 + +#define XDP_SIZEOF_RX_DESCRIPTOR_CONTEXTS_REVISION_1 \ + RTL_SIZEOF_THROUGH_FIELD(XDP_RX_DESCRIPTOR_CONTEXTS, BufferContextAlignment) + +inline +VOID +XdpInitializeRxDescriptorContexts( + _Out_ XDP_RX_DESCRIPTOR_CONTEXTS *DescriptorContexts + ) +{ + RtlZeroMemory(DescriptorContexts, sizeof(*DescriptorContexts)); + DescriptorContexts->Header.Revision = XDP_RX_DESCRIPTOR_CONTEXTS_REVISION_1; + DescriptorContexts->Header.Size = XDP_SIZEOF_RX_DESCRIPTOR_CONTEXTS_REVISION_1; +} + +VOID +XdpRxQueueSetCapabilities( + _In_ XDP_RX_QUEUE_CONFIG_CREATE RxQueueConfig, + _In_ XDP_RX_CAPABILITIES *Capabilities + ); + +VOID +XdpRxQueueRegisterExtensionVersion( + _In_ XDP_RX_QUEUE_CONFIG_CREATE RxQueueConfig, + _In_ XDP_EXTENSION_INFO *ExtensionInfo + ); + +VOID +XdpRxQueueSetDescriptorContexts( + _In_ XDP_RX_QUEUE_CONFIG_CREATE RxQueueConfig, + _In_ XDP_RX_DESCRIPTOR_CONTEXTS *DescriptorContexts + ); + +VOID +XdpRxQueueSetPollInfo( + _In_ XDP_RX_QUEUE_CONFIG_CREATE RxQueueConfig, + _In_ XDP_POLL_INFO *PollInfo + ); + +XDP_RING * +XdpRxQueueGetFrameRing( + _In_ XDP_RX_QUEUE_CONFIG_ACTIVATE RxQueueConfig + ); + +XDP_RING * +XdpRxQueueGetFragmentRing( + _In_ XDP_RX_QUEUE_CONFIG_ACTIVATE RxQueueConfig + ); + +VOID +XdpRxQueueGetExtension( + _In_ XDP_RX_QUEUE_CONFIG_ACTIVATE RxQueueConfig, + _In_ XDP_EXTENSION_INFO *ExtensionInfo, + _Out_ XDP_EXTENSION *Extension + ); + +BOOLEAN +XdpRxQueueIsVirtualAddressEnabled( + _In_ XDP_RX_QUEUE_CONFIG_ACTIVATE RxQueueConfig + ); + +#include + +EXTERN_C_END diff --git a/src/xdp/include/xdp/txframecompletioncontext.h b/src/xdp/include/xdp/txframecompletioncontext.h new file mode 100644 index 00000000..ef02fc5f --- /dev/null +++ b/src/xdp/include/xdp/txframecompletioncontext.h @@ -0,0 +1,47 @@ +// +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// + +#pragma once + +EXTERN_C_START + +#pragma warning(push) +#pragma warning(default:4820) // warn if the compiler inserted padding + +typedef struct _XDP_TX_FRAME_COMPLETION_CONTEXT { + VOID *Context; +} XDP_TX_FRAME_COMPLETION_CONTEXT; + +C_ASSERT(sizeof(XDP_TX_FRAME_COMPLETION_CONTEXT) == sizeof(VOID *)); + +#pragma warning(pop) + +#define XDP_TX_FRAME_COMPLETION_CONTEXT_EXTENSION_NAME L"ms_tx_frame_completion_context" +#define XDP_TX_FRAME_COMPLETION_CONTEXT_EXTENSION_VERSION_1 1U + +#include +#include + +inline +XDP_TX_FRAME_COMPLETION_CONTEXT * +XdpGetFrameTxCompletionContextExtension( + _In_ XDP_FRAME *Frame, + _In_ XDP_EXTENSION *Extension + ) +{ + return (XDP_TX_FRAME_COMPLETION_CONTEXT *)XdpGetExtensionData(Frame, Extension); +} + +inline +XDP_TX_FRAME_COMPLETION_CONTEXT * +XdpGetTxCompletionContextExtension( + _In_ XDP_TX_FRAME_COMPLETION *Completion, + _In_ XDP_EXTENSION *Extension + ) +{ + return (XDP_TX_FRAME_COMPLETION_CONTEXT *)XdpGetExtensionData(Completion, Extension); +} + +EXTERN_C_END diff --git a/src/xdp/include/xdp/txqueueconfig.h b/src/xdp/include/xdp/txqueueconfig.h new file mode 100644 index 00000000..1fd85c7a --- /dev/null +++ b/src/xdp/include/xdp/txqueueconfig.h @@ -0,0 +1,177 @@ +// +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// + +#pragma once + +EXTERN_C_START + +#include +#include +#include +#include +#include +#include + +DECLARE_HANDLE(XDP_TX_QUEUE_CONFIG_CREATE); +DECLARE_HANDLE(XDP_TX_QUEUE_CONFIG_ACTIVATE); + +CONST XDP_QUEUE_INFO * +XdpTxQueueGetTargetQueueInfo( + _In_ XDP_TX_QUEUE_CONFIG_CREATE TxQueueConfig + ); + +typedef struct _XDP_TX_CAPABILITIES { + XDP_OBJECT_HEADER Header; + BOOLEAN VirtualAddressEnabled; + BOOLEAN MdlEnabled; + XDP_DMA_CAPABILITIES *DmaCapabilities; + UINT16 TransmitFrameCountHint; + UINT32 MaximumBufferSize; + UINT32 MaximumFrameSize; + UINT8 MaximumFragments; + BOOLEAN OutOfOrderCompletionEnabled; +} XDP_TX_CAPABILITIES; + + +#define XDP_TX_CAPABILITIES_REVISION_1 1 + +#define XDP_SIZEOF_TX_CAPABILITIES_REVISION_1 \ + RTL_SIZEOF_THROUGH_FIELD(XDP_TX_CAPABILITIES, OutOfOrderCompletionEnabled) + +// +// Reserved for system use. +// +inline +VOID +XdpInitializeTxCapabilities( + _Out_ XDP_TX_CAPABILITIES *Capabilities + ) +{ + RtlZeroMemory(Capabilities, sizeof(*Capabilities)); + Capabilities->Header.Revision = XDP_TX_CAPABILITIES_REVISION_1; + Capabilities->Header.Size = XDP_SIZEOF_TX_CAPABILITIES_REVISION_1; + Capabilities->MaximumBufferSize = MAXUINT32; + Capabilities->MaximumFrameSize = MAXUINT32; +} + +inline +VOID +XdpInitializeTxCapabilitiesSystemVa( + _Out_ XDP_TX_CAPABILITIES *Capabilities + ) +{ + XdpInitializeTxCapabilities(Capabilities); + Capabilities->VirtualAddressEnabled = TRUE; +} + +inline +VOID +XdpInitializeTxCapabilitiesSystemMdl( + _Out_ XDP_TX_CAPABILITIES *Capabilities + ) +{ + XdpInitializeTxCapabilities(Capabilities); + Capabilities->MdlEnabled = TRUE; +} + +inline +VOID +XdpInitializeTxCapabilitiesSystemDma( + _Out_ XDP_TX_CAPABILITIES *Capabilities, + _In_ XDP_DMA_CAPABILITIES *DmaCapabilities + ) +{ + XdpInitializeTxCapabilities(Capabilities); + Capabilities->DmaCapabilities = DmaCapabilities; +} + +typedef struct _XDP_TX_DESCRIPTOR_CONTEXTS { + XDP_OBJECT_HEADER Header; + UINT8 FrameContextSize; + UINT8 FrameContextAlignment; + UINT8 BufferContextSize; + UINT8 BufferContextAlignment; +} XDP_TX_DESCRIPTOR_CONTEXTS; + +#define XDP_TX_DESCRIPTOR_CONTEXTS_REVISION_1 1 + +#define XDP_SIZEOF_TX_DESCRIPTOR_CONTEXTS_REVISION_1 \ + RTL_SIZEOF_THROUGH_FIELD(XDP_TX_DESCRIPTOR_CONTEXTS, BufferContextAlignment) + +inline +VOID +XdpInitializeTxDescriptorContexts( + _Out_ XDP_TX_DESCRIPTOR_CONTEXTS *DescriptorContexts + ) +{ + RtlZeroMemory(DescriptorContexts, sizeof(*DescriptorContexts)); + DescriptorContexts->Header.Revision = XDP_TX_DESCRIPTOR_CONTEXTS_REVISION_1; + DescriptorContexts->Header.Size = XDP_SIZEOF_TX_DESCRIPTOR_CONTEXTS_REVISION_1; +} + +VOID +XdpTxQueueSetCapabilities( + _In_ XDP_TX_QUEUE_CONFIG_CREATE TxQueueConfig, + _In_ XDP_TX_CAPABILITIES *Capabilities + ); + +VOID +XdpTxQueueRegisterExtensionVersion( + _In_ XDP_TX_QUEUE_CONFIG_CREATE TxQueueConfig, + _In_ XDP_EXTENSION_INFO *ExtensionInfo + ); + +VOID +XdpTxQueueSetDescriptorContexts( + _In_ XDP_TX_QUEUE_CONFIG_CREATE TxQueueConfig, + _In_ XDP_TX_DESCRIPTOR_CONTEXTS *DescriptorContexts + ); + +VOID +XdpTxQueueSetPollInfo( + _In_ XDP_TX_QUEUE_CONFIG_CREATE TxQueueConfig, + _In_ XDP_POLL_INFO *PollInfo + ); + +XDP_RING * +XdpTxQueueGetFrameRing( + _In_ XDP_TX_QUEUE_CONFIG_ACTIVATE TxQueueConfig + ); + +XDP_RING * +XdpTxQueueGetFragmentRing( + _In_ XDP_TX_QUEUE_CONFIG_ACTIVATE TxQueueConfig + ); + +XDP_RING * +XdpTxQueueGetCompletionRing( + _In_ XDP_TX_QUEUE_CONFIG_ACTIVATE TxQueueConfig + ); + +VOID +XdpTxQueueGetExtension( + _In_ XDP_TX_QUEUE_CONFIG_ACTIVATE TxQueueConfig, + _In_ XDP_EXTENSION_INFO *ExtensionInfo, + _Out_ XDP_EXTENSION *Extension + ); + +BOOLEAN +XdpTxQueueIsFragmentationEnabled( + _In_ XDP_TX_QUEUE_CONFIG_ACTIVATE TxQueueConfig + ); + +BOOLEAN +XdpTxQueueIsTxCompletionContextEnabled( + _In_ XDP_TX_QUEUE_CONFIG_ACTIVATE TxQueueConfig + ); + +BOOLEAN +XdpTxQueueIsOutOfOrderCompletionEnabled( + _In_ XDP_TX_QUEUE_CONFIG_ACTIVATE TxQueueConfig + ); + +#include + +EXTERN_C_END diff --git a/src/xdp/include/xdpapi.h b/src/xdp/include/xdpapi.h new file mode 100644 index 00000000..efaf360f --- /dev/null +++ b/src/xdp/include/xdpapi.h @@ -0,0 +1,166 @@ +// +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// + +#ifndef XDPAPI_H +#define XDPAPI_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +#ifndef XDPAPI +#define XDPAPI __declspec(dllimport) +#endif + +typedef enum _XDP_CREATE_PROGRAM_FLAGS { + XDP_CREATE_PROGRAM_FLAG_NONE = 0x0, + XDP_CREATE_PROGRAM_FLAG_GENERIC = 0x1, + XDP_CREATE_PROGRAM_FLAG_NATIVE = 0x2, + XDP_CREATE_PROGRAM_FLAG_ALL_QUEUES = 0x4, +} XDP_CREATE_PROGRAM_FLAGS; + +DEFINE_ENUM_FLAG_OPERATORS(XDP_CREATE_PROGRAM_FLAGS); +C_ASSERT(sizeof(XDP_CREATE_PROGRAM_FLAGS) == sizeof(UINT32)); + +typedef +HRESULT +XDP_CREATE_PROGRAM_FN( + _In_ UINT32 InterfaceIndex, + _In_ CONST XDP_HOOK_ID *HookId, + _In_ UINT32 QueueId, + _In_ XDP_CREATE_PROGRAM_FLAGS Flags, + _In_reads_(RuleCount) CONST XDP_RULE *Rules, + _In_ UINT32 RuleCount, + _Out_ HANDLE *Program + ); + +typedef +HRESULT +XDP_INTERFACE_OPEN_FN( + _In_ UINT32 InterfaceIndex, + _Out_ HANDLE *InterfaceHandle + ); + +#include "afxdp.h" + +typedef struct _XDP_API_TABLE XDP_API_TABLE; + +// +// The only API version currently supported. Any change to the API is considered +// a breaking change and support for previous versions will be removed. +// +#define XDP_API_VERSION_1 1 + +typedef +HRESULT +XDP_OPEN_API_FN( + _In_ UINT32 XdpApiVersion, + _Out_ CONST XDP_API_TABLE **XdpApiTable + ); + +XDPAPI XDP_OPEN_API_FN XdpOpenApi; + +typedef +VOID +XDP_CLOSE_API_FN( + _In_ CONST XDP_API_TABLE *XdpApiTable + ); + +XDPAPI XDP_CLOSE_API_FN XdpCloseApi; + +typedef +VOID * +XDP_GET_ROUTINE_FN( + _In_z_ const CHAR *RoutineName + ); + +typedef struct _XDP_API_TABLE { + XDP_OPEN_API_FN *XdpOpenApi; + XDP_CLOSE_API_FN *XdpCloseApi; + XDP_GET_ROUTINE_FN *XdpGetRoutine; + XDP_CREATE_PROGRAM_FN *XdpCreateProgram; + XDP_INTERFACE_OPEN_FN *XdpInterfaceOpen; + XSK_CREATE_FN *XskCreate; + XSK_BIND_FN *XskBind; + XSK_ACTIVATE_FN *XskActivate; + XSK_NOTIFY_SOCKET_FN *XskNotifySocket; + XSK_NOTIFY_ASYNC_FN *XskNotifyAsync; + XSK_GET_NOTIFY_ASYNC_RESULT_FN *XskGetNotifyAsyncResult; + XSK_SET_SOCKOPT_FN *XskSetSockopt; + XSK_GET_SOCKOPT_FN *XskGetSockopt; + XSK_IOCTL_FN *XskIoctl; +} XDP_API_TABLE; + +typedef struct _XDP_LOAD_CONTEXT *XDP_LOAD_API_CONTEXT; + +#if !defined(_KERNEL_MODE) + +inline +HRESULT +XdpLoadApi( + _In_ UINT32 XdpApiVersion, + _Out_ XDP_LOAD_API_CONTEXT *XdpLoadApiContext, + _Out_ CONST XDP_API_TABLE **XdpApiTable + ) +{ + HRESULT Result; + HMODULE XdpHandle; + XDP_OPEN_API_FN *OpenApi; + + *XdpLoadApiContext = NULL; + *XdpApiTable = NULL; + + XdpHandle = LoadLibraryA("xdpapi.dll"); + if (XdpHandle == NULL) { + Result = E_NOINTERFACE; + goto Exit; + } + + OpenApi = (XDP_OPEN_API_FN *)GetProcAddress(XdpHandle, "XdpOpenApi"); + if (OpenApi == NULL) { + Result = E_NOINTERFACE; + goto Exit; + } + + Result = OpenApi(XdpApiVersion, XdpApiTable); + +Exit: + + if (SUCCEEDED(Result)) { + *XdpLoadApiContext = (XDP_LOAD_API_CONTEXT)XdpHandle; + } else { + if (XdpHandle != NULL) { + FreeLibrary(XdpHandle); + } + } + + return Result; +} + +inline +VOID +XdpUnloadApi( + _In_ XDP_LOAD_API_CONTEXT XdpLoadApiContext, + _In_ CONST XDP_API_TABLE *XdpApiTable + ) +{ + HMODULE XdpHandle = (HMODULE)XdpLoadApiContext; + + XdpApiTable->XdpCloseApi(XdpApiTable); + + FreeLibrary(XdpHandle); +} + +#endif // !defined(_KERNEL_MODE) + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif diff --git a/src/xdp/include/xdpapi_experimental.h b/src/xdp/include/xdpapi_experimental.h new file mode 100644 index 00000000..3672f4cf --- /dev/null +++ b/src/xdp/include/xdpapi_experimental.h @@ -0,0 +1,307 @@ +// +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// + +// +// This header declares experimental XDP interfaces. All definitions within +// this file are subject to breaking changes, including removal. +// + +#ifndef XDPAPI_EXPERIMENTAL_H +#define XDPAPI_EXPERIMENTAL_H + +#ifdef __cplusplus +extern "C" { +#endif + +// +// RSS offload. +// + +#define XDP_RSS_ + +#define XDP_RSS_HASH_TYPE_IPV4 0x001 +#define XDP_RSS_HASH_TYPE_TCP_IPV4 0x002 +#define XDP_RSS_HASH_TYPE_UDP_IPV4 0x004 +#define XDP_RSS_HASH_TYPE_IPV6 0x008 +#define XDP_RSS_HASH_TYPE_TCP_IPV6 0x010 +#define XDP_RSS_HASH_TYPE_UDP_IPV6 0x020 +#define XDP_RSS_HASH_TYPE_IPV6_EX 0x040 +#define XDP_RSS_HASH_TYPE_TCP_IPV6_EX 0x080 +#define XDP_RSS_HASH_TYPE_UDP_IPV6_EX 0x100 +#define XDP_RSS_VALID_HASH_TYPES ( \ + XDP_RSS_HASH_TYPE_IPV4 | \ + XDP_RSS_HASH_TYPE_TCP_IPV4 | \ + XDP_RSS_HASH_TYPE_UDP_IPV4 | \ + XDP_RSS_HASH_TYPE_IPV6 | \ + XDP_RSS_HASH_TYPE_TCP_IPV6 | \ + XDP_RSS_HASH_TYPE_UDP_IPV6 | \ + XDP_RSS_HASH_TYPE_IPV6_EX | \ + XDP_RSS_HASH_TYPE_TCP_IPV6_EX | \ + XDP_RSS_HASH_TYPE_UDP_IPV6_EX | \ + 0) + +typedef struct _XDP_RSS_CAPABILITIES { + XDP_OBJECT_HEADER Header; + UINT32 Flags; + + // + // Supported hash types. Contains OR'd XDP_RSS_HASH_TYPE_* flags, or 0 to + // indicate RSS is not supported. + // + UINT32 HashTypes; + + // + // Maximum hash secret key size, in bytes. + // + UINT32 HashSecretKeySize; + + // + // Number of hardware receive queues. + // + UINT32 NumberOfReceiveQueues; + + // + // Maximum number of indirection table entries. + // + UINT32 NumberOfIndirectionTableEntries; +} XDP_RSS_CAPABILITIES; + +#define XDP_RSS_CAPABILITIES_REVISION_1 1 + +#define XDP_SIZEOF_RSS_CAPABILITIES_REVISION_1 \ + RTL_SIZEOF_THROUGH_FIELD(XDP_RSS_CAPABILITIES, NumberOfReceiveQueues) + +#define XDP_RSS_CAPABILITIES_REVISION_2 2 + +#define XDP_SIZEOF_RSS_CAPABILITIES_REVISION_2 \ + RTL_SIZEOF_THROUGH_FIELD(XDP_RSS_CAPABILITIES, NumberOfIndirectionTableEntries) + +// +// Initializes an RSS capabilities object. +// +inline +VOID +XdpInitializeRssCapabilities( + _Out_ XDP_RSS_CAPABILITIES *RssCapabilities + ) +{ + RtlZeroMemory(RssCapabilities, sizeof(*RssCapabilities)); + RssCapabilities->Header.Revision = XDP_RSS_CAPABILITIES_REVISION_1; + RssCapabilities->Header.Size = XDP_SIZEOF_RSS_CAPABILITIES_REVISION_1; +} + +// +// Query RSS capabilities on an interface. If the input RssCapabilitiesSize is +// too small, HRESULT_FROM_WIN32(ERROR_INSUFFICIENT_BUFFER) will be returned. +// Call with a NULL RssCapabilities to get the length. +// +typedef +HRESULT +XDP_RSS_GET_CAPABILITIES_FN( + _In_ HANDLE InterfaceHandle, + _Out_opt_ XDP_RSS_CAPABILITIES *RssCapabilities, + _Inout_ UINT32 *RssCapabilitiesSize + ); + +#define XDP_RSS_GET_CAPABILITIES_FN_NAME "XdpRssGetCapabilitiesExperimental" + +// +// Upon set, indicates XDP_RSS_CONFIGURATION.HashType should not be ignored. +// +#define XDP_RSS_FLAG_SET_HASH_TYPE 0x0001 +// +// Upon set, indicates XDP_RSS_CONFIGURATION.HashSecretKeySize and +// XDP_RSS_CONFIGURATION.HashSecretKeyOffset should not be ignored. +// +#define XDP_RSS_FLAG_SET_HASH_SECRET_KEY 0x0002 +// +// Upon set, indicates XDP_RSS_CONFIGURATION.IndirectionTableSize and +// XDP_RSS_CONFIGURATION.IndirectionTableOffset should not be ignored. +// +#define XDP_RSS_FLAG_SET_INDIRECTION_TABLE 0x0004 +// +// Upon set, indicates RSS should be disabled. +// Upon get, indicates RSS is disabled. +// +#define XDP_RSS_FLAG_DISABLED 0x0008 +#define XDP_RSS_VALID_FLAGS ( \ + XDP_RSS_FLAG_SET_HASH_TYPE | \ + XDP_RSS_FLAG_SET_HASH_SECRET_KEY | \ + XDP_RSS_FLAG_SET_INDIRECTION_TABLE | \ + XDP_RSS_FLAG_DISABLED | \ + 0) + +typedef struct _XDP_RSS_CONFIGURATION { + XDP_OBJECT_HEADER Header; + + UINT32 Flags; + + // + // Packet hash type. + // Contains OR'd XDP_RSS_HASH_TYPE_* flags, or 0 to indicate RSS is disabled. + // + UINT32 HashType; + + // + // Number of bytes from the start of this struct to the start of the hash + // secret key. + // + UINT16 HashSecretKeyOffset; + + // + // Number of valid bytes in the hash secret key. Hash secret key + // representation is UCHAR[]. + // + UINT16 HashSecretKeySize; + + // + // Number of bytes from the start of this struct to the start of the + // indirection table. + // + UINT16 IndirectionTableOffset; + + // + // Number of valid bytes in the indirection table. Indirection table + // representation is PROCESSOR_NUMBER[]. + // + UINT16 IndirectionTableSize; +} XDP_RSS_CONFIGURATION; + +#define XDP_RSS_CONFIGURATION_REVISION_1 1 + +#define XDP_SIZEOF_RSS_CONFIGURATION_REVISION_1 \ + RTL_SIZEOF_THROUGH_FIELD(XDP_RSS_CONFIGURATION, IndirectionTableSize) + +// +// Initializes a RSS configuration object. +// +inline +VOID +XdpInitializeRssConfiguration( + _Out_writes_bytes_(RssConfigurationSize) XDP_RSS_CONFIGURATION *RssConfiguration, + _In_ UINT32 RssConfigurationSize + ) +{ + RtlZeroMemory(RssConfiguration, RssConfigurationSize); + RssConfiguration->Header.Revision = XDP_RSS_CONFIGURATION_REVISION_1; + RssConfiguration->Header.Size = XDP_SIZEOF_RSS_CONFIGURATION_REVISION_1; +} + +// +// Set RSS settings on an interface. Configured settings will remain valid until +// the handle is closed. Upon handle closure, RSS settings will revert back to +// their original state. +// +typedef +HRESULT +XDP_RSS_SET_FN( + _In_ HANDLE InterfaceHandle, + _In_ CONST XDP_RSS_CONFIGURATION *RssConfiguration, + _In_ UINT32 RssConfigurationSize + ); + +#define XDP_RSS_SET_FN_NAME "XdpRssSetExperimental" + +// +// Query RSS settings on an interface. If the input RssConfigurationSize is too +// small, HRESULT_FROM_WIN32(ERROR_INSUFFICIENT_BUFFER) will be returned. Call +// with a NULL RssConfiguration to get the length. +// +typedef +HRESULT +XDP_RSS_GET_FN( + _In_ HANDLE InterfaceHandle, + _Out_opt_ XDP_RSS_CONFIGURATION *RssConfiguration, + _Inout_ UINT32 *RssConfigurationSize + ); + +#define XDP_RSS_GET_FN_NAME "XdpRssGetExperimental" + +typedef enum _XDP_QUIC_OPERATION { + XDP_QUIC_OPERATION_ADD, // Add (or modify) a QUIC connection offload + XDP_QUIC_OPERATION_REMOVE, // Remove a QUIC connection offload +} XDP_QUIC_OPERATION; + +typedef enum _XDP_QUIC_DIRECTION { + XDP_QUIC_DIRECTION_TRANSMIT, // An offload for the transmit path + XDP_QUIC_DIRECTION_RECEIVE, // An offload for the receive path +} XDP_QUIC_DIRECTION; + +typedef enum _XDP_QUIC_DECRYPT_FAILURE_ACTION { + XDP_QUIC_DECRYPT_FAILURE_ACTION_DROP, // Drop the packet on decryption failure + XDP_QUIC_DECRYPT_FAILURE_ACTION_CONTINUE, // Continue and pass the packet up on decryption failure +} XDP_QUIC_DECRYPT_FAILURE_ACTION; + +typedef enum _XDP_QUIC_CIPHER_TYPE { + XDP_QUIC_CIPHER_TYPE_AEAD_AES_128_GCM, + XDP_QUIC_CIPHER_TYPE_AEAD_AES_256_GCM, + XDP_QUIC_CIPHER_TYPE_AEAD_CHACHA20_POLY1305, + XDP_QUIC_CIPHER_TYPE_AEAD_AES_128_CCM, +} XDP_QUIC_CIPHER_TYPE; + +typedef enum _XDP_QUIC_ADDRESS_FAMILY { + XDP_QUIC_ADDRESS_FAMILY_INET4, + XDP_QUIC_ADDRESS_FAMILY_INET6, +} XDP_QUIC_ADDRESS_FAMILY; + +typedef struct _XDP_QUIC_CONNECTION { + XDP_OBJECT_HEADER Header; + UINT32 Operation : 1; // XDP_QUIC_OPERATION + UINT32 Direction : 1; // XDP_QUIC_DIRECTION + UINT32 DecryptFailureAction : 1; // XDP_QUIC_DECRYPT_FAILURE_ACTION + UINT32 KeyPhase : 1; + UINT32 RESERVED : 12; // Must be set to 0. Don't read. + UINT32 CipherType : 16; // XDP_QUIC_CIPHER_TYPE + XDP_QUIC_ADDRESS_FAMILY AddressFamily; + UINT16 UdpPort; // Destination port. + UINT64 NextPacketNumber; + UINT8 ConnectionIdLength; + UINT8 Address[16]; // Destination IP address. + UINT8 ConnectionId[20]; // QUIC v1 and v2 max CID size + UINT8 PayloadKey[32]; // Length determined by CipherType + UINT8 HeaderKey[32]; // Length determined by CipherType + UINT8 PayloadIv[12]; + HRESULT Status; // The result of trying to offload this connection. +} XDP_QUIC_CONNECTION; + +#define XDP_QUIC_CONNECTION_REVISION_1 1 + +#define XDP_SIZEOF_QUIC_CONNECTION_REVISION_1 \ + RTL_SIZEOF_THROUGH_FIELD(XDP_QUIC_CONNECTION, Status) + +// +// Initializes a QEO configuration object. +// +inline +VOID +XdpInitializeQuicConnection( + _Out_writes_bytes_(XdpQuicConnectionSize) XDP_QUIC_CONNECTION *XdpQuicConnection, + _In_ UINT32 XdpQuicConnectionSize + ) +{ + RtlZeroMemory(XdpQuicConnection, XdpQuicConnectionSize); + XdpQuicConnection->Header.Revision = XDP_QUIC_CONNECTION_REVISION_1; + XdpQuicConnection->Header.Size = XDP_SIZEOF_QUIC_CONNECTION_REVISION_1; +} + +// +// Set QEO settings on an interface. Configured settings will remain valid until +// the handle is closed. Upon handle closure, QEO settings will revert back to +// their original state. +// +typedef HRESULT +XDP_QEO_SET_FN( + _In_ HANDLE InterfaceHandle, + _Inout_ XDP_QUIC_CONNECTION *QuicConnections, + _In_ UINT32 QuicConnectionsSize + ); + +#define XDP_QEO_SET_FN_NAME "XdpQeoSetExperimental" + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif diff --git a/src/xdp/include/xdpddi.h b/src/xdp/include/xdpddi.h new file mode 100644 index 00000000..6ffe9279 --- /dev/null +++ b/src/xdp/include/xdpddi.h @@ -0,0 +1,36 @@ +// +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// + +#pragma once + +// +// This top-level header includes all XDP headers for driver developers. +// + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include diff --git a/src/xdp/usage.md b/src/xdp/usage.md new file mode 100644 index 00000000..d8177b59 --- /dev/null +++ b/src/xdp/usage.md @@ -0,0 +1,133 @@ +# How to use XDP for Windows + +## Prerequisites + +- Windows Server 2019 or 2022 + +## Installation + +XDP for Windows consists of a usermode library (xdpapi.dll) and a driver (xdp.sys). + +If xdp.sys is not production-signed: + +```bat +CertUtil.exe -addstore Root CoreNetSignRoot.cer +CertUtil.exe -addstore TrustedPublisher CoreNetSignRoot.cer +bcdedit.exe /set testsigning on +[reboot] +``` + +Install: + +```bat +msiexec /i xdp-for-windows.msi /quiet +``` + +Uninstall: + +```bat +msiexec /x xdp-for-windows.msi /quiet +``` + +### Version Upgrade + +To upgrade versions of XDP, uninstall the old version and install the new version. If processes have XDP handles open (e.g. sockets, programs) those handles need to be closed for uninstallation to complete. + +## Logging + +XDP has detailed logging (via WPP) on its cold code paths and lightweight +logging (via manifest-based ETW) on its hot code paths. + +### Using log.ps1 + +The simplest way to capture and view XDP logs is to use the `log.ps1` script. +You'll need to copy the `tools` directory from this repo onto the target system. +All logging instructions require administrator privileges. + +To start XDP logging: + +```PowerShell +.\tools\log.ps1 -Start +``` + +To stop logging and convert the trace to plain text, use the following command. +This will create a binary ETL file and a plain text file under `artifacts\logs`. +To successfully convert WPP traces to plain text, the `-SymbolPath` to a directory +containing XDP symbols (.pdb files) must be provided. + +```PowerShell +.\tools\log.ps1 -Stop -Convert -SymbolPath Path\To\Symbols +``` + +The above command can be split into separate `-Stop` and `-Convert` actions when +the plain text file is not needed, or if it is more convenient to convert to +plain text on another system. + +### Advanced ETW + +These logs can be captured and formatted using any Windows ETW tool. The XDP +project itself uses [Windows Performance +Recorder](https://docs.microsoft.com/en-us/windows-hardware/test/wpt/windows-performance-recorder) +to configure ETW logging, so all XDP providers are included in +[xdptrace.wprp](..\tools\xdptrace.wprp) along with a variety of +scenario-specific profiles. + +| Type | GUID | +|------|----------------------------------------| +| ETW | `580BBDEA-B364-4369-B291-D3539E35D20B` | +| WPP | `D6143B5C-9FD6-44BA-BA02-FAD9EA0C263D` | + +### In-flight recorder + +There is also a continuously running WPP logging session writing to an in-kernel +circular buffer; the most recent log entries can be viewed at any time, +including in crash dumps, using the kernel debugger. + +``` +!rcdrkd.rcdrlogdump xdp +``` + +### Installer logging + +To collect XDP installer traces, append `/l*v filename.log` to the MSI command line. + +## Configuration + +XDP is in a passive state upon installation. XDP can be configured via a set of +usermode APIs exported from `xdpapi.dll`. + +### XDP Queues + +The number of XDP queues is determined by the number of RSS queues configured on +a network interface. The XDP queue IDs are assigned [0, N-1] for an interface +with N configured RSS queues. XDP programs and AF_XDP applications bind to RSS +queues using this queue ID space. + +### XDP access control + +Access to XDP is restricted to `SYSTEM` and the built-in administrators group by default. The `xdpcfg.exe` tool can be used to add or remove privileges. For example, to grant access to `SYSTEM`, built-in administrators, and the user or group represented by the `S-1-5-21-1626206346-3338949459-3778528156-1001` SID: + +```PowerShell +xdpcfg.exe SetDeviceSddl "D:P(A;;GA;;;SY)(A;;GA;;;BA)(A;;GA;;;S-1-5-21-1626206346-3338949459-3778528156-1001)" +``` + +The XDP driver must be restarted for these changes to take effect; the configuration is persistent across driver and machine restarts. + +## AF_XDP + +AF_XDP is the API for redirecting traffic to a usermode application. To use the API, +include the following headers: + +- afxdp.h (AF_XDP sockets API) +- xdpapi.h (XDP API) +- afxdp_helper.h (optional AF_XDP helpers) + +## Generic XDP + +A generic XDP implementation is provided by the XDP driver. Generic XDP inspects +the NBL data path of any NDIS interface without requiring third party driver +changes. + +## Native XDP + +Native XDP requires an updated NDIS driver.