|
|
/*++
Copyright (c) 1989-1999 Microsoft Corporation
Module Name:
fastio.c
Abstract:
This module contains routines for handling fast ("turbo") IO in AFD.
Author:
David Treadwell (davidtr) 12-Oct-1992
Revision History: VadimE 14-Jan-1998 Restructurred the code.
--*/
#include "afdp.h"
BOOLEANAfdFastConnectionReceive ( IN PAFD_ENDPOINT endpoint, IN PAFD_RECV_INFO recvInfo, IN ULONG recvLength, OUT PIO_STATUS_BLOCK IoStatus );
BOOLEANAfdFastDatagramReceive ( IN PAFD_ENDPOINT endpoint, IN PAFD_RECV_MESSAGE_INFO recvInfo, IN ULONG recvLength, OUT PIO_STATUS_BLOCK IoStatus );
BOOLEANAfdFastConnectionSend ( IN PAFD_ENDPOINT endpoint, IN PAFD_SEND_INFO sendInfo, IN ULONG sendLength, OUT PIO_STATUS_BLOCK IoStatus );
BOOLEANAfdFastDatagramSend ( IN PAFD_ENDPOINT endpoint, IN PAFD_SEND_DATAGRAM_INFO sendInfo, IN ULONG sendLength, OUT PIO_STATUS_BLOCK IoStatus );
NTSTATUSAfdRestartFastDatagramSend ( IN PDEVICE_OBJECT DeviceObject, IN PIRP Irp, IN PVOID Context );
#ifdef ALLOC_PRAGMA
#pragma alloc_text( PAGE, AfdFastIoDeviceControl )
#pragma alloc_text( PAGE, AfdFastIoRead )
#pragma alloc_text( PAGE, AfdFastIoWrite )
#pragma alloc_text( PAGEAFD, AfdFastDatagramSend )
#pragma alloc_text( PAGEAFD, AfdFastDatagramReceive )
#pragma alloc_text( PAGEAFD, AfdFastConnectionSend )
#pragma alloc_text( PAGEAFD, AfdFastConnectionReceive )
#pragma alloc_text( PAGEAFD, AfdRestartFastDatagramSend )
#pragma alloc_text( PAGEAFD, AfdShouldSendBlock )
#endif
#if AFD_PERF_DBG
BOOLEANAfdFastIoReadReal ( IN struct _FILE_OBJECT *FileObject, IN PLARGE_INTEGER FileOffset, IN ULONG Length, IN BOOLEAN Wait, IN ULONG LockKey, OUT PVOID Buffer, OUT PIO_STATUS_BLOCK IoStatus, IN struct _DEVICE_OBJECT *DeviceObject );
BOOLEANAfdFastIoRead ( IN struct _FILE_OBJECT *FileObject, IN PLARGE_INTEGER FileOffset, IN ULONG Length, IN BOOLEAN Wait, IN ULONG LockKey, OUT PVOID Buffer, OUT PIO_STATUS_BLOCK IoStatus, IN struct _DEVICE_OBJECT *DeviceObject ){ BOOLEAN success;
ASSERT( KeGetCurrentIrql( ) == LOW_LEVEL );
success = AfdFastIoReadReal ( FileObject, FileOffset, Length, Wait, LockKey, Buffer, IoStatus, DeviceObject );
ASSERT( KeGetCurrentIrql( ) == LOW_LEVEL ); if ( success ) { InterlockedIncrement (&AfdFastReadsSucceeded); ASSERT (IoStatus->Status == STATUS_SUCCESS || IoStatus->Status == STATUS_DEVICE_NOT_READY ); } else { InterlockedIncrement (&AfdFastReadsFailed); } return success;}
BOOLEANAfdFastIoReadReal ( IN struct _FILE_OBJECT *FileObject, IN PLARGE_INTEGER FileOffset, IN ULONG Length, IN BOOLEAN Wait, IN ULONG LockKey, OUT PVOID Buffer, OUT PIO_STATUS_BLOCK IoStatus, IN struct _DEVICE_OBJECT *DeviceObject )
#else // AFD_PERF_DBG
BOOLEANAfdFastIoRead ( IN struct _FILE_OBJECT *FileObject, IN PLARGE_INTEGER FileOffset, IN ULONG Length, IN BOOLEAN Wait, IN ULONG LockKey, OUT PVOID Buffer, OUT PIO_STATUS_BLOCK IoStatus, IN struct _DEVICE_OBJECT *DeviceObject )#endif // AFD_PERF_DBG
{
PAFD_ENDPOINT endpoint; WSABUF buf;
PAGED_CODE( );
//
// All we want to do is pass the request through to the TDI provider
// if possible. If not, we want to bail out of this code path back
// onto the main code path (with IRPs) with as little performance
// overhead as possible.
//
// Thus this routine only does general preliminary checks and input
// parameter validation. If it is determined that fast io path is
// likely to succeed, an operation specific routine is called
// to handle all the details.
//
endpoint = FileObject->FsContext; ASSERT( IS_AFD_ENDPOINT_TYPE( endpoint ) );
//
// If fast IO recv is disabled
// or the endpoint is shut down in any way
// or the endpoint isn't connected yet
// or the TDI provider for this endpoint supports bufferring,
// we do not want to do fast IO on it
//
if (endpoint->DisableFastIoRecv || endpoint->DisconnectMode != 0 || endpoint->State != AfdEndpointStateConnected || IS_TDI_BUFFERRING(endpoint)) { return FALSE; }
//
// Fake buffer array.
//
buf.buf = Buffer; buf.len = Length;
//
// Call routine based on endpoint type
//
if ( IS_DGRAM_ENDPOINT(endpoint) ) { //
// Fake input parameter strucuture
//
AFD_RECV_MESSAGE_INFO msgInfo;
msgInfo.dgi.BufferArray = &buf; msgInfo.dgi.BufferCount = 1; msgInfo.dgi.AfdFlags = AFD_OVERLAPPED; msgInfo.dgi.TdiFlags = TDI_RECEIVE_NORMAL; msgInfo.dgi.Address = NULL; msgInfo.dgi.AddressLength = 0; msgInfo.ControlBuffer = NULL; msgInfo.ControlLength = NULL; msgInfo.MsgFlags = NULL;
return AfdFastDatagramReceive( endpoint, &msgInfo, Length, IoStatus ); } else if (IS_VC_ENDPOINT(endpoint)) { //
// Fake input parameter strucuture
//
AFD_RECV_INFO recvInfo;
recvInfo.BufferArray = &buf; recvInfo.BufferCount = 1; recvInfo.AfdFlags = AFD_OVERLAPPED; recvInfo.TdiFlags = TDI_RECEIVE_NORMAL;
return AfdFastConnectionReceive ( endpoint, &recvInfo, Length, IoStatus); } else return FALSE;
} // AfdFastIoRead
#if AFD_PERF_DBG
BOOLEANAfdFastIoWriteReal ( IN struct _FILE_OBJECT *FileObject, IN PLARGE_INTEGER FileOffset, IN ULONG Length, IN BOOLEAN Wait, IN ULONG LockKey, OUT PVOID Buffer, OUT PIO_STATUS_BLOCK IoStatus, IN struct _DEVICE_OBJECT *DeviceObject );
BOOLEANAfdFastIoWrite ( IN struct _FILE_OBJECT *FileObject, IN PLARGE_INTEGER FileOffset, IN ULONG Length, IN BOOLEAN Wait, IN ULONG LockKey, OUT PVOID Buffer, OUT PIO_STATUS_BLOCK IoStatus, IN struct _DEVICE_OBJECT *DeviceObject ){ BOOLEAN success;
ASSERT( KeGetCurrentIrql( ) == LOW_LEVEL );
success = AfdFastIoWriteReal ( FileObject, FileOffset, Length, Wait, LockKey, Buffer, IoStatus, DeviceObject );
ASSERT( KeGetCurrentIrql( ) == LOW_LEVEL ); if ( success ) { InterlockedIncrement (&AfdFastWritesSucceeded); ASSERT (IoStatus->Status == STATUS_SUCCESS || IoStatus->Status == STATUS_DEVICE_NOT_READY); } else { InterlockedIncrement (&AfdFastWritesFailed); } return success;}
BOOLEANAfdFastIoWriteReal ( IN struct _FILE_OBJECT *FileObject, IN PLARGE_INTEGER FileOffset, IN ULONG Length, IN BOOLEAN Wait, IN ULONG LockKey, IN PVOID Buffer, OUT PIO_STATUS_BLOCK IoStatus, IN struct _DEVICE_OBJECT *DeviceObject )
#else // AFD_PERF_DBG
BOOLEANAfdFastIoWrite ( IN struct _FILE_OBJECT *FileObject, IN PLARGE_INTEGER FileOffset, IN ULONG Length, IN BOOLEAN Wait, IN ULONG LockKey, IN PVOID Buffer, OUT PIO_STATUS_BLOCK IoStatus, IN struct _DEVICE_OBJECT *DeviceObject )#endif // AFD_PERF_DBG
{
PAFD_ENDPOINT endpoint; WSABUF buf;
PAGED_CODE( );
//
// All we want to do is pass the request through to the TDI provider
// if possible. If not, we want to bail out of this code path back
// onto the main code path (with IRPs) with as little performance
// overhead as possible.
//
// Thus this routine only does general preliminary checks and input
// parameter validation. If it is determined that fast io path is
// likely to succeed, an operation specific routine is called
// to handle all the details.
//
endpoint = FileObject->FsContext; ASSERT( IS_AFD_ENDPOINT_TYPE( endpoint ) );
//
// If fast IO send is disabled
// or the endpoint is shut down in any way
// or the endpoint isn't connected yet
// or the TDI provider for this endpoint supports bufferring,
// we do not want to do fast IO on it
//
if (endpoint->DisableFastIoSend || endpoint->DisconnectMode != 0 || endpoint->State != AfdEndpointStateConnected || IS_TDI_BUFFERRING(endpoint) ) { return FALSE; }
//
// Fake buffer array.
//
buf.buf = Buffer; buf.len = Length;
//
// Call routine based on endpoint type
//
if ( IS_DGRAM_ENDPOINT(endpoint) ) { //
// Fake input parameter strucuture
//
AFD_SEND_DATAGRAM_INFO sendInfo;
sendInfo.BufferArray = &buf; sendInfo.BufferCount = 1; sendInfo.AfdFlags = AFD_OVERLAPPED; sendInfo.TdiConnInfo.RemoteAddress = NULL; sendInfo.TdiConnInfo.RemoteAddressLength = 0;
return AfdFastDatagramSend( endpoint, &sendInfo, Length, IoStatus ); } else if (IS_VC_ENDPOINT (endpoint)) { //
// Fake input parameter strucuture
//
AFD_SEND_INFO sendInfo;
sendInfo.BufferArray = &buf; sendInfo.BufferCount = 1; sendInfo.AfdFlags = AFD_OVERLAPPED; sendInfo.TdiFlags = 0;
return AfdFastConnectionSend ( endpoint, &sendInfo, Length, IoStatus); } else return FALSE;} // AfdFastIoWrite
#if AFD_PERF_DBG
BOOLEANAfdFastIoDeviceControlReal ( IN struct _FILE_OBJECT *FileObject, IN BOOLEAN Wait, IN PVOID InputBuffer OPTIONAL, IN ULONG InputBufferLength, OUT PVOID OutputBuffer OPTIONAL, IN ULONG OutputBufferLength, IN ULONG IoControlCode, OUT PIO_STATUS_BLOCK IoStatus, IN struct _DEVICE_OBJECT *DeviceObject );
BOOLEANAfdFastIoDeviceControl ( IN struct _FILE_OBJECT *FileObject, IN BOOLEAN Wait, IN PVOID InputBuffer OPTIONAL, IN ULONG InputBufferLength, OUT PVOID OutputBuffer OPTIONAL, IN ULONG OutputBufferLength, IN ULONG IoControlCode, OUT PIO_STATUS_BLOCK IoStatus, IN struct _DEVICE_OBJECT *DeviceObject ){ BOOLEAN success;
ASSERT( KeGetCurrentIrql( ) == LOW_LEVEL );
success = AfdFastIoDeviceControlReal ( FileObject, Wait, InputBuffer, InputBufferLength, OutputBuffer, OutputBufferLength, IoControlCode, IoStatus, DeviceObject );
ASSERT( KeGetCurrentIrql( ) == LOW_LEVEL );
switch ( IoControlCode ) {
case IOCTL_AFD_SEND:
if ( success ) { InterlockedIncrement (&AfdFastSendsSucceeded); } else { InterlockedIncrement (&AfdFastSendsFailed); } break;
case IOCTL_AFD_RECEIVE:
if ( success ) { InterlockedIncrement (&AfdFastReceivesSucceeded); } else { InterlockedIncrement (&AfdFastReceivesFailed); } break;
case IOCTL_AFD_SEND_DATAGRAM:
if ( success ) { InterlockedIncrement (&AfdFastSendDatagramsSucceeded); } else { InterlockedIncrement (&AfdFastSendDatagramsFailed); } break;
case IOCTL_AFD_RECEIVE_MESSAGE: case IOCTL_AFD_RECEIVE_DATAGRAM:
if ( success ) { InterlockedIncrement (&AfdFastReceiveDatagramsSucceeded); } else { InterlockedIncrement (&AfdFastReceiveDatagramsFailed); } break; case IOCTL_AFD_TRANSMIT_FILE:
if ( success ) { InterlockedIncrement (&AfdFastTfSucceeded); } else { InterlockedIncrement (&AfdFastTfFailed); } break; }
return success;
} // AfdFastIoDeviceControl
BOOLEANAfdFastIoDeviceControlReal ( IN struct _FILE_OBJECT *FileObject, IN BOOLEAN Wait, IN PVOID InputBuffer OPTIONAL, IN ULONG InputBufferLength, OUT PVOID OutputBuffer OPTIONAL, IN ULONG OutputBufferLength, IN ULONG IoControlCode, OUT PIO_STATUS_BLOCK IoStatus, IN struct _DEVICE_OBJECT *DeviceObject )#else
BOOLEANAfdFastIoDeviceControl ( IN struct _FILE_OBJECT *FileObject, IN BOOLEAN Wait, IN PVOID InputBuffer OPTIONAL, IN ULONG InputBufferLength, OUT PVOID OutputBuffer OPTIONAL, IN ULONG OutputBufferLength, IN ULONG IoControlCode, OUT PIO_STATUS_BLOCK IoStatus, IN struct _DEVICE_OBJECT *DeviceObject )#endif
{ PAFD_ENDPOINT endpoint; KPROCESSOR_MODE previousMode; BOOLEAN res; PAFD_IMMEDIATE_CALL proc; ULONG request;
#ifdef _WIN64
WSABUF localArray[8]; LPWSABUF pArray = localArray;#endif
PAGED_CODE( );
//
// All we want to do is pass the request through to the TDI provider
// if possible. If not, we want to bail out of this code path back
// onto the main code path (with IRPs) with as little performance
// overhead as possible.
//
// Thus this routine only does general preliminary checks and input
// parameter validation. If it is determined that fast io path is
// likely to succeed, an operation specific routine is called
// to handle all the details.
//
//
// First get the endpoint pointer and previous mode for input parameter
// validation.
//
endpoint = FileObject->FsContext; ASSERT( IS_AFD_ENDPOINT_TYPE( endpoint ) );
previousMode = ExGetPreviousMode ();
//
// Switch based on control code
//
switch (IoControlCode) { case IOCTL_AFD_RECEIVE: { union { AFD_RECV_INFO recvInfo; AFD_RECV_MESSAGE_INFO msgInfo; } u; ULONG recvLength;
#if !defined (_ALPHA_)
//
// Alpha compiler does not see that the following FIELD_OFFSET
// is in fact a constant expression.
//
//
// Check the validity of the union above.
//
C_ASSERT (FIELD_OFFSET (AFD_RECV_MESSAGE_INFO, dgi.BufferArray) == FIELD_OFFSET (AFD_RECV_INFO, BufferArray)); C_ASSERT (FIELD_OFFSET (AFD_RECV_MESSAGE_INFO, dgi.BufferCount) == FIELD_OFFSET (AFD_RECV_INFO, BufferCount)); C_ASSERT (FIELD_OFFSET (AFD_RECV_MESSAGE_INFO, dgi.AfdFlags) == FIELD_OFFSET (AFD_RECV_INFO, AfdFlags)); C_ASSERT (FIELD_OFFSET (AFD_RECV_MESSAGE_INFO, dgi.TdiFlags) == FIELD_OFFSET (AFD_RECV_INFO, TdiFlags));
//
#endif //!defined (_ALPHA_)
//
// If fast IO send is disabled
// or the endpoint is shut down in any way
// or the endpoint isn't connected yet
// or the TDI provider for this endpoint supports bufferring,
// we do not want to do fast IO on it
//
if (endpoint->DisableFastIoRecv || endpoint->DisconnectMode != 0 || endpoint->State != AfdEndpointStateConnected || IS_TDI_BUFFERRING(endpoint) ) { res = FALSE; break; }
try {
#ifdef _WIN64
if (IoIs32bitProcess (NULL)) { PAFD_RECV_INFO32 recvInfo32; LPWSABUF32 tempArray; ULONG i;
//
// If the input structure isn't large enough, return error.
//
if( InputBufferLength < sizeof(*recvInfo32) ) { // Fast io can't handle error returns
// if call is overlapped (completion port)
// IoStatus->Status = STATUS_INVALID_PARAMETER;
res = FALSE; break; }
//
// Validate the input structure if it comes from the user mode
// application
//
if (previousMode != KernelMode ) { ProbeForRead (InputBuffer, sizeof (*recvInfo32), PROBE_ALIGNMENT32(AFD_RECV_INFO32)); }
recvInfo32 = InputBuffer;
//
// Make local copies of the embeded pointer and parameters
// that we will be using more than once in case malicios
// application attempts to change them while we are
// validating
//
tempArray = recvInfo32->BufferArray; u.recvInfo.BufferCount = recvInfo32->BufferCount; u.recvInfo.AfdFlags = recvInfo32->AfdFlags; u.recvInfo.TdiFlags = recvInfo32->TdiFlags;
//
// If fast IO is not possible or this is not a normal receive.
// bail.
//
if( (u.recvInfo.AfdFlags & AFD_NO_FAST_IO) || u.recvInfo.TdiFlags != TDI_RECEIVE_NORMAL ) { res = FALSE; break; }
//
// Validate all the pointers that app gave to us and
// calculate the length of the send buffer.
// Buffers in the array will be validated in the
// process of copying
//
if ((tempArray == NULL) || (u.recvInfo.BufferCount == 0) || // Check for integer overflow (disabled by compiler)
(u.recvInfo.BufferCount>(MAXULONG/sizeof (WSABUF32))) ) { ExRaiseStatus(STATUS_INVALID_PARAMETER); }
if (previousMode != KernelMode) { ProbeForRead( tempArray, // Address
// Note check for overflow above (should actually be
// done here by the compiler generating code
// that causes exception on integer overflow)
u.recvInfo.BufferCount * sizeof (WSABUF32), // Length
PROBE_ALIGNMENT32(WSABUF32) // Alignment
); }
if (u.recvInfo.BufferCount>sizeof(localArray)/sizeof(localArray[0])) { try { pArray = AFD_ALLOCATE_POOL_WITH_QUOTA ( NonPagedPool, sizeof (WSABUF)*u.recvInfo.BufferCount, AFD_TEMPORARY_POOL_TAG); // AFD_ALLOCATE_POOL_WITH_QUOTA macro sets
// POOL_RAISE_IF_ALLOCATION_FAILURE flag
ASSERT (pArray!=NULL); } except (EXCEPTION_EXECUTE_HANDLER) { // Fast io can't handle error returns
// if call is overlapped (completion port)
// IoStatus->Status = GetExceptionCode ();
pArray = localArray; res = FALSE; break; } }
for (i=0; i<u.recvInfo.BufferCount; i++) { pArray[i].buf = tempArray[i].buf; pArray[i].len = tempArray[i].len; }
u.recvInfo.BufferArray = pArray;
} else#endif // _WIN64
{ //
// If the input structure isn't large enough, return error.
//
if( InputBufferLength < sizeof(u.recvInfo) ) { // Fast io can't handle error returns
// if call is overlapped (completion port)
// IoStatus->Status = STATUS_INVALID_PARAMETER;
res = FALSE; break; }
//
// Validate the input structure if it comes from the user mode
// application
//
if (previousMode != KernelMode ) { ProbeForRead (InputBuffer, sizeof (u.recvInfo), PROBE_ALIGNMENT(AFD_RECV_INFO)); }
//
// Make local copies of the embeded pointer and parameters
// that we will be using more than once in case malicios
// application attempts to change them while we are
// validating
//
u.recvInfo = *((PAFD_RECV_INFO)InputBuffer); //
// If fast IO is not possible or this is not a normal receive.
// bail.
//
if( (u.recvInfo.AfdFlags & AFD_NO_FAST_IO) || u.recvInfo.TdiFlags != TDI_RECEIVE_NORMAL ) { res = FALSE; break; }
//
// Validate all the pointers that app gave to us and
// calculate the length of the send buffer.
// Buffers in the array will be validated in the
// process of copying
//
if ((u.recvInfo.BufferArray == NULL) || (u.recvInfo.BufferCount == 0) || // Check for integer overflow (disabled by compiler)
(u.recvInfo.BufferCount>(MAXULONG/sizeof (WSABUF))) ) { ExRaiseStatus(STATUS_INVALID_PARAMETER); }
if (previousMode != KernelMode) { ProbeForRead( u.recvInfo.BufferArray, // Address
// Note check for overflow above (should actually be
// done here by the compiler generating code
// that causes exception on integer overflow)
u.recvInfo.BufferCount * sizeof (WSABUF), // Length
PROBE_ALIGNMENT(WSABUF) // Alignment
); } }
recvLength = AfdCalcBufferArrayByteLength( u.recvInfo.BufferArray, u.recvInfo.BufferCount );
} except( AFD_EXCEPTION_FILTER(NULL) ) {
// Fast io can't handle error returns
// if call is overlapped (completion port)
// IoStatus->Status = GetExceptionCode ();
res = FALSE; break; }
//
// Call routine based on endpoint type
//
if ( IS_DGRAM_ENDPOINT(endpoint) ) { u.msgInfo.dgi.Address = NULL; u.msgInfo.dgi.AddressLength = 0; u.msgInfo.ControlBuffer = NULL; u.msgInfo.ControlLength = NULL; u.msgInfo.MsgFlags = NULL;
res = AfdFastDatagramReceive( endpoint, &u.msgInfo, recvLength, IoStatus ); } else if (IS_VC_ENDPOINT (endpoint)) { res = AfdFastConnectionReceive ( endpoint, &u.recvInfo, recvLength, IoStatus); } else res = FALSE;
} break;
case IOCTL_AFD_RECEIVE_DATAGRAM: case IOCTL_AFD_RECEIVE_MESSAGE: { AFD_RECV_MESSAGE_INFO msgInfo; ULONG recvLength;
if (endpoint->DisableFastIoRecv || !IS_DGRAM_ENDPOINT(endpoint) || ((endpoint->State != AfdEndpointStateBound ) && (endpoint->State != AfdEndpointStateConnected)) ) { return FALSE; } if (IoControlCode==IOCTL_AFD_RECEIVE_MESSAGE) {#ifdef _WIN64
if (IoIs32bitProcess (NULL)) { PAFD_RECV_MESSAGE_INFO32 msgInfo32; //
// If the input structure isn't large enough, return error.
//
if( InputBufferLength < sizeof(*msgInfo32) ) { // Fast io can't handle error returns
// if call is overlapped (completion port)
// IoStatus->Status = STATUS_INVALID_PARAMETER;
res = FALSE; break; }
//
// Validate the input structure if it comes from the user mode
// application
//
if (previousMode != KernelMode ) { ProbeForRead (InputBuffer, sizeof (*msgInfo32), PROBE_ALIGNMENT32 (AFD_RECV_MESSAGE_INFO32)); }
msgInfo32 = InputBuffer;
//
// Make local copies of the embeded pointer and parameters
// that we will be using more than once in case malicios
// application attempts to change them while we are
// validating
//
msgInfo.ControlBuffer = msgInfo32->ControlBuffer; msgInfo.ControlLength = msgInfo32->ControlLength; msgInfo.MsgFlags = msgInfo32->MsgFlags; } else#endif // _WIN64
{
if( InputBufferLength < sizeof(msgInfo) ) { // Fast io can't handle error returns
// if call is overlapped (completion port)
// IoStatus->Status = STATUS_INVALID_PARAMETER;
res = FALSE; break; }
//
// Capture the input structure.
//
//
// Validate the input structure if it comes from the user mode
// application
//
if (previousMode != KernelMode ) { ProbeForRead (InputBuffer, sizeof (msgInfo), PROBE_ALIGNMENT (AFD_RECV_MESSAGE_INFO)); } msgInfo = *(PAFD_RECV_MESSAGE_INFO)InputBuffer; } if (previousMode != KernelMode ) {
ProbeForWrite (msgInfo.MsgFlags, sizeof (*msgInfo.MsgFlags), PROBE_ALIGNMENT (ULONG)); ProbeForWrite (msgInfo.ControlLength, sizeof (*msgInfo.ControlLength), PROBE_ALIGNMENT (ULONG)); //
// Checking of recvInfo->Address is postponed till
// we know the length of the address.
//
} } else { msgInfo.ControlBuffer = NULL; msgInfo.ControlLength = NULL; msgInfo.MsgFlags = NULL; }
//
// If the input structure isn't large enough, return error.
//
try {#ifdef _WIN64
if (IoIs32bitProcess (NULL)) { PAFD_RECV_DATAGRAM_INFO32 recvInfo32; LPWSABUF32 tempArray; ULONG i;
//
// If the input structure isn't large enough, return error.
//
if( InputBufferLength < sizeof(*recvInfo32) ) { // Fast io can't handle error returns
// if call is overlapped (completion port)
// IoStatus->Status = STATUS_INVALID_PARAMETER;
res = FALSE; break; }
//
// Validate the input structure if it comes from the user mode
// application
//
if (previousMode != KernelMode ) { ProbeForRead (InputBuffer, sizeof (*recvInfo32), PROBE_ALIGNMENT32 (AFD_RECV_DATAGRAM_INFO32)); }
recvInfo32 = InputBuffer;
//
// Make local copies of the embeded pointer and parameters
// that we will be using more than once in case malicios
// application attempts to change them while we are
// validating
//
tempArray = recvInfo32->BufferArray; msgInfo.dgi.BufferCount = recvInfo32->BufferCount; msgInfo.dgi.AfdFlags = recvInfo32->AfdFlags; msgInfo.dgi.TdiFlags = recvInfo32->TdiFlags; msgInfo.dgi.Address = recvInfo32->Address; msgInfo.dgi.AddressLength = recvInfo32->AddressLength;
//
// If fast IO is not possible or this is not a normal receive.
// bail.
//
if( (msgInfo.dgi.AfdFlags & AFD_NO_FAST_IO) || msgInfo.dgi.TdiFlags != TDI_RECEIVE_NORMAL ) { res = FALSE; break; }
//
// Validate all the pointers that app gave to us and
// calculate the length of the send buffer.
// Buffers in the array will be validated in the
// process of copying
//
if ((tempArray == NULL) || (msgInfo.dgi.BufferCount == 0) || // Check for integer overflow (disabled by compiler)
(msgInfo.dgi.BufferCount>(MAXULONG/sizeof (WSABUF32))) ) { ExRaiseStatus(STATUS_INVALID_PARAMETER); }
if (previousMode != KernelMode) { ProbeForRead( tempArray, // Address
// Note check for overflow above (should actually be
// done here by the compiler generating code
// that causes exception on integer overflow)
msgInfo.dgi.BufferCount * sizeof (WSABUF32), // Length
PROBE_ALIGNMENT (WSABUF32) // Alignment
); }
if (msgInfo.dgi.BufferCount>sizeof(localArray)/sizeof(localArray[0])) { try { pArray = AFD_ALLOCATE_POOL_WITH_QUOTA ( NonPagedPool, sizeof (WSABUF)*msgInfo.dgi.BufferCount, AFD_TEMPORARY_POOL_TAG); // AFD_ALLOCATE_POOL_WITH_QUOTA macro sets
// POOL_RAISE_IF_ALLOCATION_FAILURE flag
ASSERT (pArray!=NULL); } except (EXCEPTION_EXECUTE_HANDLER) { // Fast io can't handle error returns
// if call is overlapped (completion port)
// IoStatus->Status = GetExceptionCode ();
pArray = localArray; res = FALSE; break; } }
for (i=0; i<msgInfo.dgi.BufferCount; i++) { pArray[i].buf = tempArray[i].buf; pArray[i].len = tempArray[i].len; }
msgInfo.dgi.BufferArray = pArray;
} else#endif // _WIN64
{ if( InputBufferLength < sizeof(AFD_RECV_DATAGRAM_INFO) ) { // Fast io can't handle error returns
// if call is overlapped (completion port)
// IoStatus->Status = STATUS_INVALID_PARAMETER;
res = FALSE; break; }
//
// Capture the input structure.
//
//
// Validate the input structure if it comes from the user mode
// application
//
if (previousMode != KernelMode ) { ProbeForRead (InputBuffer, sizeof (AFD_RECV_DATAGRAM_INFO), PROBE_ALIGNMENT (AFD_RECV_DATAGRAM_INFO)); }
//
// Make local copies of the embeded pointer and parameters
// that we will be using more than once in case malicios
// application attempts to change them while we are
// validating
//
msgInfo.dgi = *(PAFD_RECV_DATAGRAM_INFO)InputBuffer;
//
// If fast IO is disabled or this is not a simple
// recv, fail
//
if( (msgInfo.dgi.AfdFlags & AFD_NO_FAST_IO) != 0 || msgInfo.dgi.TdiFlags != TDI_RECEIVE_NORMAL || ( (msgInfo.dgi.Address == NULL) ^ (msgInfo.dgi.AddressLength == NULL) ) ) { res = FALSE; break; }
//
// Validate all the pointers that app gave to us.
// and calculate total recv length.
// Buffers in the array will be validated in the
// process of copying
//
if ((msgInfo.dgi.BufferArray == NULL) || (msgInfo.dgi.BufferCount == 0) || // Check for integer overflow (disabled by compiler)
(msgInfo.dgi.BufferCount>(MAXULONG/sizeof (WSABUF))) ) { ExRaiseStatus(STATUS_INVALID_PARAMETER); }
if (previousMode != KernelMode) { ProbeForRead( msgInfo.dgi.BufferArray, // Address
// Note check for overflow above (should actually be
// done here by the compiler generating code
// that causes exception on integer overflow)
msgInfo.dgi.BufferCount * sizeof (WSABUF), // Length
PROBE_ALIGNMENT(WSABUF) // Alignment
); } }
recvLength = AfdCalcBufferArrayByteLength( msgInfo.dgi.BufferArray, msgInfo.dgi.BufferCount );
if (previousMode != KernelMode ) { if (msgInfo.dgi.AddressLength!=NULL) { ProbeForWrite (msgInfo.dgi.AddressLength, sizeof (*msgInfo.dgi.AddressLength), PROBE_ALIGNMENT (ULONG)); } //
// Checking of recvInfo->Address is postponed till
// we know the length of the address.
//
}
} except( AFD_EXCEPTION_FILTER(NULL) ) {
// Fast io can't handle error returns
// if call is overlapped (completion port)
// IoStatus->Status = GetExceptionCode ();
res = FALSE; break;
}
//
// Attempt to perform fast IO on the endpoint.
//
res = AfdFastDatagramReceive( endpoint, &msgInfo, recvLength, IoStatus );
} break;
case IOCTL_AFD_SEND: { union { AFD_SEND_INFO sendInfo; AFD_SEND_DATAGRAM_INFO sendInfoDg; } u; ULONG sendLength;
//
// Check the validity of the union above.
//
C_ASSERT (FIELD_OFFSET (AFD_SEND_DATAGRAM_INFO, BufferArray) == FIELD_OFFSET (AFD_SEND_INFO, BufferArray)); C_ASSERT (FIELD_OFFSET (AFD_SEND_DATAGRAM_INFO, BufferCount) == FIELD_OFFSET (AFD_SEND_INFO, BufferCount)); C_ASSERT (FIELD_OFFSET (AFD_SEND_DATAGRAM_INFO, AfdFlags) == FIELD_OFFSET (AFD_SEND_INFO, AfdFlags));
//
// If fast IO send is disabled
// or the endpoint is shut down in any way
// or the endpoint isn't connected yet
// or the TDI provider for this endpoint supports bufferring,
// we do not want to do fast IO on it
//
if (endpoint->DisableFastIoSend || endpoint->DisconnectMode != 0 || endpoint->State != AfdEndpointStateConnected || IS_TDI_BUFFERRING(endpoint) ) { return FALSE; }
try {
#ifdef _WIN64
if (IoIs32bitProcess (NULL)) { PAFD_SEND_INFO32 sendInfo32; LPWSABUF32 tempArray; ULONG i;
//
// If the input structure isn't large enough, return error.
//
if( InputBufferLength < sizeof(*sendInfo32) ) { // Fast io can't handle error returns
// if call is overlapped (completion port)
// IoStatus->Status = STATUS_INVALID_PARAMETER;
res = FALSE; break; }
//
// Validate the input structure if it comes from the user mode
// application
//
if (previousMode != KernelMode ) { ProbeForRead (InputBuffer, sizeof (*sendInfo32), PROBE_ALIGNMENT32 (AFD_SEND_INFO32)); }
sendInfo32 = InputBuffer;
//
// Make local copies of the embeded pointer and parameters
// that we will be using more than once in case malicios
// application attempts to change them while we are
// validating
//
tempArray = sendInfo32->BufferArray; u.sendInfo.BufferCount = sendInfo32->BufferCount; u.sendInfo.AfdFlags = sendInfo32->AfdFlags; u.sendInfo.TdiFlags = sendInfo32->TdiFlags;
//
// If fast IO is not possible or this is not a normal receive.
// bail.
//
if( (u.sendInfo.AfdFlags & AFD_NO_FAST_IO) || u.sendInfo.TdiFlags != 0 ) { res = FALSE; break; }
//
// Validate all the pointers that app gave to us and
// calculate the length of the send buffer.
// Buffers in the array will be validated in the
// process of copying
//
if ((tempArray == NULL) || (u.sendInfo.BufferCount == 0) || // Check for integer overflow (disabled by compiler)
(u.sendInfo.BufferCount>(MAXULONG/sizeof (WSABUF32))) ) { ExRaiseStatus(STATUS_INVALID_PARAMETER); }
if (previousMode != KernelMode) { ProbeForRead( tempArray, // Address
// Note check for overflow above (should actually be
// done here by the compiler generating code
// that causes exception on integer overflow)
u.sendInfo.BufferCount * sizeof (WSABUF32), // Length
PROBE_ALIGNMENT32(WSABUF32) // Alignment
); }
if (u.sendInfo.BufferCount>sizeof(localArray)/sizeof(localArray[0])) { try { pArray = AFD_ALLOCATE_POOL_WITH_QUOTA ( NonPagedPool, sizeof (WSABUF)*u.sendInfo.BufferCount, AFD_TEMPORARY_POOL_TAG); // AFD_ALLOCATE_POOL_WITH_QUOTA macro sets
// POOL_RAISE_IF_ALLOCATION_FAILURE flag
ASSERT (pArray!=NULL); } except (EXCEPTION_EXECUTE_HANDLER) { // Fast io can't handle error returns
// if call is overlapped (completion port)
// IoStatus->Status = GetExceptionCode ();
pArray = localArray; res = FALSE; break; } }
for (i=0; i<u.sendInfo.BufferCount; i++) { pArray[i].buf = tempArray[i].buf; pArray[i].len = tempArray[i].len; }
u.sendInfo.BufferArray = pArray;
} else#endif // _WIN64
{ //
// If the input structure isn't large enough, return error.
//
if( InputBufferLength < sizeof(u.sendInfo) ) { // Fast io can't handle error returns
// if call is overlapped (completion port)
// IoStatus->Status = STATUS_INVALID_PARAMETER;
res = FALSE; break; }
//
// Validate the input structure if it comes from the user mode
// application
//
if (previousMode != KernelMode) { ProbeForRead (InputBuffer, sizeof (u.sendInfo), PROBE_ALIGNMENT(AFD_SEND_INFO)); }
//
// Make local copies of the embeded pointer and parameters
// that we will be using more than once in case malicios
// application attempts to change them while we are
// validating
//
u.sendInfo = *((PAFD_SEND_INFO)InputBuffer);
if( (u.sendInfo.AfdFlags & AFD_NO_FAST_IO) != 0 || u.sendInfo.TdiFlags != 0 ) { res = FALSE; break; }
if ((u.sendInfo.BufferArray == NULL) || (u.sendInfo.BufferCount == 0) || // Check for integer overflow (disabled by compiler)
(u.sendInfo.BufferCount>(MAXULONG/sizeof (WSABUF))) ) { ExRaiseStatus(STATUS_INVALID_PARAMETER); }
if (previousMode != KernelMode) { ProbeForRead( u.sendInfo.BufferArray, // Address
// Note check for overflow above (should actually be
// done here by the compiler generating code
// that causes exception on integer overflow)
u.sendInfo.BufferCount * sizeof (WSABUF), // Length
PROBE_ALIGNMENT(WSABUF) // Alignment
); }
} sendLength = AfdCalcBufferArrayByteLength( u.sendInfo.BufferArray, u.sendInfo.BufferCount );
} except( AFD_EXCEPTION_FILTER(NULL) ) {
// Fast io can't handle error returns
// if call is overlapped (completion port)
// IoStatus->Status = GetExceptionCode ();
res = FALSE; break; }
if (IS_DGRAM_ENDPOINT (endpoint)) { u.sendInfoDg.TdiConnInfo.RemoteAddress = NULL; u.sendInfoDg.TdiConnInfo.RemoteAddressLength = 0; res = AfdFastDatagramSend ( endpoint, &u.sendInfoDg, sendLength, IoStatus); } else if (IS_VC_ENDPOINT (endpoint)) { res = AfdFastConnectionSend ( endpoint, &u.sendInfo, sendLength, IoStatus); } else res = FALSE; }
break; case IOCTL_AFD_SEND_DATAGRAM: { AFD_SEND_DATAGRAM_INFO sendInfo; ULONG sendLength;
if (endpoint->DisableFastIoSend || !IS_DGRAM_ENDPOINT(endpoint) || ((endpoint->State != AfdEndpointStateBound ) && (endpoint->State != AfdEndpointStateConnected)) ) { res = FALSE; break; }
try {
#ifdef _WIN64
if (IoIs32bitProcess (NULL)) { PAFD_SEND_DATAGRAM_INFO32 sendInfo32; LPWSABUF32 tempArray; ULONG i;
//
// If the input structure isn't large enough, return error.
//
if( InputBufferLength < sizeof(*sendInfo32) ) { // Fast io can't handle error returns
// if call is overlapped (completion port)
// IoStatus->Status = STATUS_INVALID_PARAMETER;
res = FALSE; break; }
//
// Validate the input structure if it comes from the user mode
// application
//
if (previousMode != KernelMode ) { ProbeForRead (InputBuffer, sizeof (*sendInfo32), PROBE_ALIGNMENT32(AFD_SEND_DATAGRAM_INFO32)); }
sendInfo32 = InputBuffer;
//
// Make local copies of the embeded pointer and parameters
// that we will be using more than once in case malicios
// application attempts to change them while we are
// validating
//
tempArray = sendInfo32->BufferArray; sendInfo.BufferCount = sendInfo32->BufferCount; sendInfo.AfdFlags = sendInfo32->AfdFlags; sendInfo.TdiConnInfo.RemoteAddress = sendInfo32->TdiConnInfo.RemoteAddress; sendInfo.TdiConnInfo.RemoteAddressLength = sendInfo32->TdiConnInfo.RemoteAddressLength;
//
// If fast IO is not possible bail.
//
if(sendInfo.AfdFlags & AFD_NO_FAST_IO) { res = FALSE; break; }
//
// Validate all the pointers that app gave to us and
// calculate the length of the send buffer.
// Buffers in the array will be validated in the
// process of copying
//
if ((tempArray == NULL) || (sendInfo.BufferCount == 0) || // Check for integer overflow (disabled by compiler)
(sendInfo.BufferCount>(MAXULONG/sizeof (WSABUF32))) ) { ExRaiseStatus(STATUS_INVALID_PARAMETER); }
if (previousMode != KernelMode) { ProbeForRead( tempArray, // Address
// Note check for overflow above (should actually be
// done here by the compiler generating code
// that causes exception on integer overflow)
sendInfo.BufferCount * sizeof (WSABUF32), // Length
PROBE_ALIGNMENT32(WSABUF32) // Alignment
); }
if (sendInfo.BufferCount>sizeof(localArray)/sizeof(localArray[0])) { try { pArray = AFD_ALLOCATE_POOL_WITH_QUOTA ( NonPagedPool, sizeof (WSABUF)*sendInfo.BufferCount, AFD_TEMPORARY_POOL_TAG); // AFD_ALLOCATE_POOL_WITH_QUOTA macro sets
// POOL_RAISE_IF_ALLOCATION_FAILURE flag
ASSERT (pArray!=NULL); } except (EXCEPTION_EXECUTE_HANDLER) { // Fast io can't handle error returns
// if call is overlapped (completion port)
// IoStatus->Status = GetExceptionCode ();
pArray = localArray; res = FALSE; break; } }
for (i=0; i<sendInfo.BufferCount; i++) { pArray[i].buf = tempArray[i].buf; pArray[i].len = tempArray[i].len; }
sendInfo.BufferArray = pArray;
} else#endif // _WIN64
{ //
// If the input structure isn't large enough, bail on fast IO.
//
if( InputBufferLength < sizeof(sendInfo) ) { // Fast io can't handle error returns
// if call is overlapped (completion port)
// IoStatus->Status = STATUS_INVALID_PARAMETER;
res = FALSE; break; }
//
// Validate the input structure if it comes from the user mode
// application
//
if (previousMode != KernelMode) { ProbeForRead (InputBuffer, sizeof (sendInfo), PROBE_ALIGNMENT(AFD_SEND_DATAGRAM_INFO));
}
//
// Make local copies of the embeded pointer and parameters
// that we will be using more than once in case malicios
// application attempts to change them while we are
// validating
//
sendInfo = *((PAFD_SEND_DATAGRAM_INFO)InputBuffer); //
// If fast IO is disabled, bail
//
if( (sendInfo.AfdFlags & AFD_NO_FAST_IO) != 0) { res = FALSE; break; }
//
// Validate all the pointers that app gave to us
// and calculate total send length
// Buffers in the array will be validated in the
// process of copying
//
if ((sendInfo.BufferArray == NULL) || (sendInfo.BufferCount == 0) || // Check for integer overflow (disabled by compiler)
(sendInfo.BufferCount>(MAXULONG/sizeof (WSABUF))) ) { ExRaiseStatus(STATUS_INVALID_PARAMETER); }
if (previousMode != KernelMode) { ProbeForRead( sendInfo.BufferArray, // Address
// Note check for overflow above (should actually be
// done here by the compiler generating code
// that causes exception on integer overflow)
sendInfo.BufferCount * sizeof (WSABUF), // Length
PROBE_ALIGNMENT(WSABUF) // Alignment
); } }
sendLength = AfdCalcBufferArrayByteLength( sendInfo.BufferArray, sendInfo.BufferCount );
if (previousMode != KernelMode ) { ProbeForRead ( sendInfo.TdiConnInfo.RemoteAddress, // Address
sendInfo.TdiConnInfo.RemoteAddressLength, // Length,
sizeof (UCHAR) // Aligment
); }
} except( AFD_EXCEPTION_FILTER(NULL) ) {
// Fast io can't handle error returns
// if call is overlapped (completion port)
// IoStatus->Status = GetExceptionCode ();
res = FALSE; break; } //
// Attempt to perform fast IO on the endpoint.
//
res = AfdFastDatagramSend( endpoint, &sendInfo, sendLength, IoStatus );
}
break;
case IOCTL_AFD_TRANSMIT_FILE: {
AFD_TRANSMIT_FILE_INFO userTransmitInfo; try {
#ifdef _WIN64
if (IoIs32bitProcess (NULL)) { PAFD_TRANSMIT_FILE_INFO32 userTransmitInfo32; if ( InputBufferLength < sizeof(AFD_TRANSMIT_FILE_INFO32) ) { return FALSE; }
if (previousMode != KernelMode) { ProbeForRead (InputBuffer, sizeof (*userTransmitInfo32), PROBE_ALIGNMENT32(AFD_TRANSMIT_FILE_INFO32));
userTransmitInfo32 = InputBuffer; userTransmitInfo.Offset = userTransmitInfo32->Offset; userTransmitInfo.WriteLength = userTransmitInfo32->WriteLength; userTransmitInfo.SendPacketLength = userTransmitInfo32->SendPacketLength; userTransmitInfo.FileHandle = userTransmitInfo32->FileHandle; userTransmitInfo.Head = userTransmitInfo32->Head; userTransmitInfo.HeadLength = userTransmitInfo32->HeadLength; userTransmitInfo.Tail = userTransmitInfo32->Tail; userTransmitInfo.TailLength = userTransmitInfo32->TailLength; userTransmitInfo.Flags = userTransmitInfo32->Flags;
if (userTransmitInfo.HeadLength>0) ProbeForRead (userTransmitInfo.Head, userTransmitInfo.HeadLength, sizeof (UCHAR)); if (userTransmitInfo.TailLength>0) ProbeForRead (userTransmitInfo.Tail, userTransmitInfo.TailLength, sizeof (UCHAR)); } else { ASSERT (FALSE); }
} else#endif // _WIN64
{ if ( InputBufferLength < sizeof(AFD_TRANSMIT_FILE_INFO) ) { return FALSE; }
if (previousMode != KernelMode) { ProbeForRead (InputBuffer, sizeof (userTransmitInfo), PROBE_ALIGNMENT(AFD_TRANSMIT_FILE_INFO)); userTransmitInfo = *((PAFD_TRANSMIT_FILE_INFO)InputBuffer); if (userTransmitInfo.HeadLength>0) ProbeForRead (userTransmitInfo.Head, userTransmitInfo.HeadLength, sizeof (UCHAR)); if (userTransmitInfo.TailLength>0) ProbeForRead (userTransmitInfo.Tail, userTransmitInfo.TailLength, sizeof (UCHAR)); } else { userTransmitInfo = *((PAFD_TRANSMIT_FILE_INFO)InputBuffer); } }
} except( AFD_EXCEPTION_FILTER(NULL) ) {
res = FALSE; break; }
res = AfdFastTransmitFile (endpoint, &userTransmitInfo, IoStatus);
}
return res;
default: request = _AFD_REQUEST(IoControlCode); if( request < AFD_NUM_IOCTLS && AfdIoctlTable[request] == IoControlCode && AfdImmediateCallDispatch[request]!=NULL) {
proc = AfdImmediateCallDispatch[request]; IoStatus->Status = (*proc) ( FileObject, IoControlCode, previousMode, InputBuffer, InputBufferLength, OutputBuffer, OutputBufferLength, &IoStatus->Information );
ASSERT (IoStatus->Status!=STATUS_PENDING); res = TRUE; } else { res = FALSE; } break; }
#ifdef _WIN64
if (pArray!=localArray) { AFD_FREE_POOL (pArray, AFD_TEMPORARY_POOL_TAG); }#endif
return res;}
BOOLEANAfdFastConnectionSend ( IN PAFD_ENDPOINT endpoint, IN PAFD_SEND_INFO sendInfo, IN ULONG sendLength, OUT PIO_STATUS_BLOCK IoStatus ){ PAFD_BUFFER afdBuffer; PAFD_CONNECTION connection; AFD_LOCK_QUEUE_HANDLE lockHandle; NTSTATUS status;
ASSERT( endpoint->Type == AfdBlockTypeVcConnecting || endpoint->Type == AfdBlockTypeVcBoth );
AfdAcquireSpinLock( &endpoint->SpinLock, &lockHandle );
connection = endpoint->Common.VcConnecting.Connection;
if (connection==NULL) { //
// connection might have been cleaned up by transmit file.
//
AfdReleaseSpinLock (&endpoint->SpinLock, &lockHandle); return FALSE; }
ASSERT( connection->Type == AfdBlockTypeConnection );
//
// If the connection has been aborted, then we don't want to try
// fast IO on it.
//
if ( connection->CleanupBegun || connection->AbortIndicated ) { AfdReleaseSpinLock (&endpoint->SpinLock, &lockHandle); return FALSE; }
//
// Determine whether we can do fast IO with this send. In order
// to perform fast IO, there must be no other sends pended on this
// connection and there must be enough space left for bufferring
// the requested amount of data.
//
if ( AfdShouldSendBlock( endpoint, connection, sendLength ) ) { AfdReleaseSpinLock (&endpoint->SpinLock, &lockHandle);
//
// If this is a nonblocking endpoint, fail the request here and
// save going through the regular path.
//
if ( endpoint->NonBlocking && !( sendInfo->AfdFlags & AFD_OVERLAPPED ) ) { // Fast io can't handle error returns
// if call is overlapped (completion port), but we know
// that it is not overlapped
IoStatus->Status = STATUS_DEVICE_NOT_READY; return TRUE; }
return FALSE; }
//
// Add a reference to the connection object since the send
// request will complete asynchronously.
//
REFERENCE_CONNECTION( connection ); AfdReleaseSpinLock (&endpoint->SpinLock, &lockHandle);
IF_DEBUG(FAST_IO) { KdPrintEx(( DPFLTR_WSOCKTRANSPORT_ID, DPFLTR_TRACE_LEVEL, "AfdFastConnectionSend: attempting fast IO on endp %p, conn %p\n", endpoint, connection)); }
//
// Next get an AFD buffer structure that contains an IRP and a
// buffer to hold the data.
//
afdBuffer = AfdGetBuffer( sendLength, 0, connection->OwningProcess );
if ( afdBuffer == NULL) { AfdAcquireSpinLock( &endpoint->SpinLock, &lockHandle ); connection->VcBufferredSendBytes -= sendLength; connection->VcBufferredSendCount -= 1; AfdReleaseSpinLock( &endpoint->SpinLock, &lockHandle );
DEREFERENCE_CONNECTION (connection);
return FALSE; }
//
// We have to rebuild the MDL in the AFD buffer structure to
// represent exactly the number of bytes we're going to be
// sending.
//
afdBuffer->Mdl->ByteCount = sendLength;
//
// Remember the connection in the AFD buffer structure. We need
// this in order to access the connection in the restart routine.
//
afdBuffer->Context = connection;
//
// Copy the user's data into the AFD buffer.
//
if( sendLength > 0 ) {
try {
AfdCopyBufferArrayToBuffer( afdBuffer->Buffer, sendLength, sendInfo->BufferArray, sendInfo->BufferCount );
} except( AFD_EXCEPTION_FILTER(NULL) ) {
afdBuffer->Mdl->ByteCount = afdBuffer->BufferLength; AfdReturnBuffer( &afdBuffer->Header, connection->OwningProcess ); AfdAcquireSpinLock( &endpoint->SpinLock, &lockHandle ); connection->VcBufferredSendBytes -= sendLength; connection->VcBufferredSendCount -= 1; AfdReleaseSpinLock( &endpoint->SpinLock, &lockHandle );
DEREFERENCE_CONNECTION (connection); // Fast io can't handle error returns
// if call is overlapped (completion port)
// IoStatus->Status = GetExceptionCode ();
return FALSE; } }
//
// Use the IRP in the AFD buffer structure to give to the TDI
// provider. Build the TDI send request.
//
TdiBuildSend( afdBuffer->Irp, connection->DeviceObject, connection->FileObject, AfdRestartBufferSend, afdBuffer, afdBuffer->Mdl, 0, sendLength );
//
// Call the transport to actually perform the send.
//
status = IoCallDriver( connection->DeviceObject, afdBuffer->Irp );
//
// Complete the user's IRP as appropriate. Note that we change the
// status code from what was returned by the TDI provider into
// STATUS_SUCCESS. This is because we don't want to complete
// the IRP with STATUS_PENDING etc.
//
if ( NT_SUCCESS(status) ) { IoStatus->Information = sendLength; IoStatus->Status = STATUS_SUCCESS; return TRUE; }
//
// The call failed for some reason. Fail fast IO.
//
return FALSE;}
BOOLEANAfdFastConnectionReceive ( IN PAFD_ENDPOINT endpoint, IN PAFD_RECV_INFO recvInfo, IN ULONG recvLength, OUT PIO_STATUS_BLOCK IoStatus ){ PLIST_ENTRY listEntry; ULONG totalOffset, partialLength; PAFD_BUFFER_HEADER afdBuffer, partialAfdBuffer=NULL; PAFD_CONNECTION connection; AFD_LOCK_QUEUE_HANDLE lockHandle; LIST_ENTRY bufferListHead; BOOLEAN retryReceive = FALSE; // Retry receive if additional data
// was indicated by the transport and buffered
// while we were copying current batch.
ASSERT( endpoint->Type == AfdBlockTypeVcConnecting || endpoint->Type == AfdBlockTypeVcBoth ); IoStatus->Status = STATUS_SUCCESS; IoStatus->Information = 0;
Retry:
AfdAcquireSpinLock( &endpoint->SpinLock, &lockHandle );
connection = endpoint->Common.VcConnecting.Connection; if (connection==NULL) { //
// connection might have been cleaned up by transmit file.
//
AfdReleaseSpinLock (&endpoint->SpinLock, &lockHandle); //
// If we have already copied something before retrying,
// return success, next receive will report the error.
//
return retryReceive; }
ASSERT( connection->Type == AfdBlockTypeConnection );
IF_DEBUG(FAST_IO) { KdPrintEx(( DPFLTR_WSOCKTRANSPORT_ID, DPFLTR_TRACE_LEVEL, "AfdFastConnectionReceive: attempting fast IO on endp %p, conn %p\n", endpoint, connection)); }
//
// Determine whether we'll be able to perform fast IO. In order
// to do fast IO, there must be some bufferred data on the
// connection, there must not be any pended receives on the
// connection, and there must not be any bufferred expedited
// data on the connection. This last requirement is for
// the sake of simplicity only.
//
if ( !IsListEmpty( &connection->VcReceiveIrpListHead ) || connection->VcBufferredExpeditedCount != 0 || connection->DisconnectIndicated || connection->AbortIndicated) { AfdReleaseSpinLock( &endpoint->SpinLock, &lockHandle ); //
// If we have already copied something before retrying,
// return success, next receive will report the error.
//
return retryReceive; }
if (connection->VcBufferredReceiveCount == 0) { ASSERT( IsListEmpty( &connection->VcReceiveBufferListHead ) );
//
// If this is a nonblocking endpoint, fail the request here and
// save going through the regular path.
if (!retryReceive && endpoint->NonBlocking && !(recvInfo->AfdFlags & AFD_OVERLAPPED)) { endpoint->EventsActive &= ~AFD_POLL_RECEIVE;
IF_DEBUG(EVENT_SELECT) { KdPrintEx(( DPFLTR_WSOCKTRANSPORT_ID, DPFLTR_TRACE_LEVEL, "AfdFastConnectionReceive: Endp %p, Active %lx\n", endpoint, endpoint->EventsActive )); }
AfdReleaseSpinLock( &endpoint->SpinLock, &lockHandle ); IoStatus->Status = STATUS_DEVICE_NOT_READY;
return TRUE; }
AfdReleaseSpinLock( &endpoint->SpinLock, &lockHandle ); //
// If we have already copied something before retrying,
// return success, next receive will report the error.
//
return retryReceive; }
ASSERT( !IsListEmpty( &connection->VcReceiveBufferListHead ) );
//
// Get a pointer to the first bufferred AFD buffer structure on
// the connection.
//
afdBuffer = CONTAINING_RECORD( connection->VcReceiveBufferListHead.Flink, AFD_BUFFER_HEADER, BufferListEntry );
ASSERT( !afdBuffer->ExpeditedData );
//
// For message endpoints if the buffer contains a partial message
// or doesn't fit into the buffer, bail out.
// We don't want the added complexity of handling
// partial messages in the fast path.
//
if ( IS_MESSAGE_ENDPOINT(endpoint) && (afdBuffer->PartialMessage || afdBuffer->DataLength>recvLength)) { //
// We shouldn't be retry-ing for message oriented endpoint
// since we only allow fast path if complete message is available.
//
ASSERT (retryReceive == FALSE);
AfdReleaseSpinLock( &endpoint->SpinLock, &lockHandle ); return FALSE; }
//
// Remeber current offset before we update
// information field (it is not 0 if we are
// re-trying).
//
totalOffset = (ULONG)IoStatus->Information;
InitializeListHead( &bufferListHead );
//
// Reference the connection object so it doen't go away
// until we return the buffer.
//
REFERENCE_CONNECTION (connection);
//
// Loop getting AFD buffers that will fill in the user's
// buffer with as much data as will fit, or else with a
// single buffer if this is not a stream endpoint. We don't
// actually do the copy within this loop because this loop
// must occur while holding a lock, and we cannot hold a
// lock while copying the data into the user's buffer
// because the user's buffer is not locked and we cannot
// take a page fault at raised IRQL.
//
while (IoStatus->Information<recvLength) { ASSERT( connection->VcBufferredReceiveBytes >= afdBuffer->DataLength ); ASSERT( connection->VcBufferredReceiveCount > 0 );
if (recvLength-IoStatus->Information>=afdBuffer->DataLength) { //
// If we can copy the whole buffer, remove it from the connection's list of
// buffers and place it on our local list of buffers.
//
RemoveEntryList( &afdBuffer->BufferListEntry ); InsertTailList( &bufferListHead, &afdBuffer->BufferListEntry ); //
// Update the count of bytes on the connection.
//
connection->VcBufferredReceiveBytes -= afdBuffer->DataLength; connection->VcBufferredReceiveCount -= 1; IoStatus->Information += afdBuffer->DataLength;
//
// If this is a stream endpoint and more buffers are available,
// try to fit the next one it as well..
//
if (!IS_MESSAGE_ENDPOINT(endpoint) && !IsListEmpty( &connection->VcReceiveBufferListHead ) ) {
afdBuffer = CONTAINING_RECORD( connection->VcReceiveBufferListHead.Flink, AFD_BUFFER_HEADER, BufferListEntry );
ASSERT( !afdBuffer->ExpeditedData ); continue; } } else { //
// Copy just a part of the buffer that fits and
// increment its reference count so it doesn't get
// destroyed until we done copying.
//
ASSERT (!IS_MESSAGE_ENDPOINT (endpoint));
partialLength = recvLength-(ULONG)IoStatus->Information; partialAfdBuffer = afdBuffer; partialAfdBuffer->DataLength -= partialLength; partialAfdBuffer->DataOffset += partialLength; InterlockedIncrement (&partialAfdBuffer->RefCount); connection->VcBufferredReceiveBytes -= partialLength; IoStatus->Information = recvLength; }
break; }
endpoint->EventsActive &= ~AFD_POLL_RECEIVE;
if( !IsListEmpty( &connection->VcReceiveBufferListHead )) {
AfdIndicateEventSelectEvent( endpoint, AFD_POLL_RECEIVE, STATUS_SUCCESS );
retryReceive = FALSE; } else { //
// We got all the data buffered. It is possible
// that while we are copying data, more gets indicated
// by the transport since we copy at passive level
// and indication occur at DPC (or even on another processor).
// We'll check again after copying, so we return as much data
// as possible to the application (to improve performance).
// For message oriented transports we can only
// deliver one message at a time and we shouldn't be on the fast path
// if we do not have a complete message.
// If application has EventSelect/select/AsyncSelect outstanding.
// we can't copy more data as well since it would receive a signal
// to come back and we would already consumed the data. We are not
// concerned with the case when application calls some form of select
// while we are in this routine because signaling is not guaranteed
// to be multithread safe (e.g. if select comes right before we take
// the spinlock in the beginning of this routine, application will
// get false signal as well).
//
retryReceive = IoStatus->Information<recvLength && !IS_MESSAGE_ENDPOINT (endpoint) && (endpoint->EventsEnabled & AFD_POLL_RECEIVE)==0 && !endpoint->PollCalled;
//
// Disable fast IO path to avoid performance penalty
// of unneccessarily going through it.
//
if (!endpoint->NonBlocking) endpoint->DisableFastIoRecv = TRUE; }
//
// If there is indicated but unreceived data in the TDI provider,
// and we have available buffer space, fire off an IRP to receive
// the data.
//
if ( connection->VcReceiveBytesInTransport > 0
&&
connection->VcBufferredReceiveBytes < connection->MaxBufferredReceiveBytes
) {
ULONG bytesToReceive; PAFD_BUFFER newAfdBuffer;
//
// Remember the count of data that we're going to receive,
// then reset the fields in the connection where we keep
// track of how much data is available in the transport.
// We reset it here before releasing the lock so that
// another thread doesn't try to receive the data at the
// same time as us.
//
if ( connection->VcReceiveBytesInTransport > AfdLargeBufferSize ) { bytesToReceive = connection->VcReceiveBytesInTransport; } else { bytesToReceive = AfdLargeBufferSize; }
//
// Get an AFD buffer structure to hold the data.
//
newAfdBuffer = AfdGetBuffer( bytesToReceive, 0, connection->OwningProcess ); if ( newAfdBuffer == NULL ) { //
// If we were unable to get a buffer, just remember
// that we still have data in transport
//
if (connection->VcBufferredReceiveBytes == 0 && !connection->OnLRList) { //
// Since we do not have any data buffered, application
// is not notified and will never call with recv.
// We will have to put this on low resource list
// and attempt to allocate memory and pull the data
// later.
//
connection->OnLRList = TRUE; REFERENCE_CONNECTION (connection); AfdLRListAddItem (&connection->LRListItem, AfdLRRepostReceive); } else { UPDATE_CONN (connection); } AfdReleaseSpinLock( &endpoint->SpinLock, &lockHandle );
} else {
connection->VcReceiveBytesInTransport = 0; ASSERT (InterlockedDecrement (&connection->VcReceiveIrpsInTransport)==-1);
//
// We need to remember the connection in the AFD buffer
// because we'll need to access it in the completion
// routine.
//
newAfdBuffer->Context = connection;
//
// Acquire connection reference to be released in completion routine
//
REFERENCE_CONNECTION (connection);
AfdReleaseSpinLock( &endpoint->SpinLock, &lockHandle );
//
// Finish building the receive IRP to give to the TDI provider.
//
TdiBuildReceive( newAfdBuffer->Irp, connection->DeviceObject, connection->FileObject, AfdRestartBufferReceive, newAfdBuffer, newAfdBuffer->Mdl, TDI_RECEIVE_NORMAL, (CLONG)bytesToReceive );
//
// Hand off the IRP to the TDI provider.
//
(VOID)IoCallDriver( connection->DeviceObject, newAfdBuffer->Irp ); }
} else {
AfdReleaseSpinLock( &endpoint->SpinLock, &lockHandle ); }
//
// We have in a local list all the data we'll use for this
// IO. Start copying data to the user buffer.
//
while ( !IsListEmpty( &bufferListHead ) ) {
//
// Take the first buffer from the list.
//
listEntry = RemoveHeadList( &bufferListHead ); afdBuffer = CONTAINING_RECORD( listEntry, AFD_BUFFER_HEADER, BufferListEntry ); DEBUG afdBuffer->BufferListEntry.Flink = NULL;
if( afdBuffer->DataLength > 0 ) {
ASSERTMSG ( "NIC Driver freed the packet before it was returned!!!", !afdBuffer->NdisPacket || (MmIsAddressValid (afdBuffer->Context) && MmIsAddressValid (MmGetSystemAddressForMdl (afdBuffer->Mdl))) ); try {
//
// Copy the data in the buffer to the user buffer.
//
AfdCopyMdlChainToBufferArray( recvInfo->BufferArray, totalOffset, recvInfo->BufferCount, afdBuffer->Mdl, afdBuffer->DataOffset, afdBuffer->DataLength );
} except( AFD_EXCEPTION_FILTER(NULL) ) {
//
// If an exception is hit, there is the possibility of
// data corruption. However, it is nearly impossible to
// avoid this in all cases, so just throw out the
// remainder of the data that we would have copied to
// the user buffer.
//
if (afdBuffer->RefCount==1 || // Can't change once off the list
InterlockedDecrement (&afdBuffer->RefCount)==0) { AfdReturnBuffer( afdBuffer, connection->OwningProcess ); }
while ( !IsListEmpty( &bufferListHead ) ) { listEntry = RemoveHeadList( &bufferListHead ); afdBuffer = CONTAINING_RECORD( listEntry, AFD_BUFFER_HEADER, BufferListEntry ); DEBUG afdBuffer->BufferListEntry.Flink = NULL; if (afdBuffer->RefCount==1 || // Can't change once off the list
InterlockedDecrement (&afdBuffer->RefCount)==0) { AfdReturnBuffer( afdBuffer, connection->OwningProcess ); } }
//
// We'll have to abort since there is a possibility of data corruption.
// Shame on application for giving us bogus buffers.
// This also releases the reference that we needed to return the buffer.
//
AfdAbortConnection (connection);
// Fast io can't handle error returns
// if call is overlapped (completion port)
// IoStatus->Status = GetExceptionCode ();
return FALSE; }
totalOffset += afdBuffer->DataLength; }
//
// We're done with the AFD buffer.
//
if (afdBuffer->RefCount==1 || // Can't change once off the list
InterlockedDecrement (&afdBuffer->RefCount)==0) { AfdReturnBuffer( afdBuffer, connection->OwningProcess ); } }
//
// Copy any partial buffers
//
if (partialAfdBuffer) { ASSERT (partialLength>0); ASSERTMSG ( "NIC Driver freed the packet before it was returned!!!", !partialAfdBuffer->NdisPacket || (MmIsAddressValid (partialAfdBuffer->Context) && MmIsAddressValid (MmGetSystemAddressForMdl (partialAfdBuffer->Mdl))) ); try {
//
// Copy the data in the buffer to the user buffer.
//
AfdCopyMdlChainToBufferArray( recvInfo->BufferArray, totalOffset, recvInfo->BufferCount, partialAfdBuffer->Mdl, partialAfdBuffer->DataOffset-partialLength, partialLength );
} except( AFD_EXCEPTION_FILTER(NULL) ) { if (InterlockedDecrement (&partialAfdBuffer->RefCount)==0) { ASSERT (partialAfdBuffer->BufferListEntry.Flink == NULL); AfdReturnBuffer( partialAfdBuffer, connection->OwningProcess ); } //
// We'll have to abort since there is a possibility of data corruption.
// Shame on application for giving us bogus buffers.
// This also releases the reference that we needed to return the buffer.
//
AfdAbortConnection (connection);
// Fast io can't handle error returns
// if call is overlapped (completion port)
// IoStatus->Status = GetExceptionCode ();
return FALSE; }
if (InterlockedDecrement (&partialAfdBuffer->RefCount)==0) { ASSERT (partialAfdBuffer->BufferListEntry.Flink == NULL); AfdReturnBuffer( partialAfdBuffer, connection->OwningProcess ); }
totalOffset += partialLength; }
ASSERT (IoStatus->Information==totalOffset);
//
// If more data is available, we need to retry and attempt to completely
// fill application's buffer.
//
if (retryReceive && (endpoint->EventsActive & AFD_POLL_RECEIVE)) { ASSERT (IoStatus->Information<recvLength && !IS_MESSAGE_ENDPOINT (endpoint)); DEREFERENCE_CONNECTION2 (connection, "Fast retry receive 0x%lX bytes", (ULONG)IoStatus->Information); goto Retry; } else { //
// Release the reference needed to return the buffer(s).
//
DEREFERENCE_CONNECTION2 (connection, "Fast receive 0x%lX bytes", (ULONG)IoStatus->Information); }
ASSERT( IoStatus->Information <= recvLength ); ASSERT (IoStatus->Status == STATUS_SUCCESS); return TRUE;}
BOOLEANAfdFastDatagramSend ( IN PAFD_ENDPOINT endpoint, IN PAFD_SEND_DATAGRAM_INFO sendInfo, IN ULONG sendLength, OUT PIO_STATUS_BLOCK IoStatus ){ PAFD_BUFFER afdBuffer = NULL; NTSTATUS status; AFD_LOCK_QUEUE_HANDLE lockHandle;
//
// If this is a send for more than the threshold number of
// bytes, don't use the fast path. We don't allow larger sends
// in the fast path because of the extra data copy it entails,
// which is more expensive for large buffers. For smaller
// buffers, however, the cost of the copy is small compared to
// the IO system overhead of the slow path.
//
// We also copy and return for non-blocking endpoints regardless
// of the size. That's what we are supposed to do according
// to the spec.
//
if ( !endpoint->NonBlocking && sendLength > AfdFastSendDatagramThreshold ) { return FALSE; }
//
// If we already buffered to many sends, go the long way.
//
AfdAcquireSpinLock (&endpoint->SpinLock, &lockHandle); if ( endpoint->DgBufferredSendBytes >= endpoint->Common.Datagram.MaxBufferredSendBytes && endpoint->DgBufferredSendBytes>0) {
if ( endpoint->NonBlocking && !( sendInfo->AfdFlags & AFD_OVERLAPPED ) ) { endpoint->EventsActive &= ~AFD_POLL_SEND; endpoint->EnableSendEvent = TRUE;
IF_DEBUG(EVENT_SELECT) { KdPrintEx(( DPFLTR_WSOCKTRANSPORT_ID, DPFLTR_TRACE_LEVEL, "AfdFastIoDeviceControl: Endp %p, Active %lX\n", endpoint, endpoint->EventsActive )); } } AfdReleaseSpinLock (&endpoint->SpinLock, &lockHandle); //
// If this is a nonblocking endpoint, fail the request here and
// save going through the regular path.(check for non-blocking is
// below, otherwise status code is ignored).
//
status = STATUS_DEVICE_NOT_READY; goto errorset; }
endpoint->DgBufferredSendBytes += sendLength;
AfdReleaseSpinLock (&endpoint->SpinLock, &lockHandle);
IF_DEBUG(FAST_IO) { KdPrintEx(( DPFLTR_WSOCKTRANSPORT_ID, DPFLTR_TRACE_LEVEL, "AfdFastDatagramSend: attempting fast IO on endp %p\n", endpoint)); }
//
// Get an AFD buffer to use for the request. We'll copy the
// user's data to the AFD buffer then submit the IRP in the AFD
// buffer to the TDI provider.
if ((sendInfo->TdiConnInfo.RemoteAddressLength==0) && !IS_TDI_DGRAM_CONNECTION(endpoint)) { retry: try { //
// Get an AFD buffer to use for the request. We'll copy the
// user to the AFD buffer then submit the IRP in the AFD
// buffer to the TDI provider.
//
afdBuffer = AfdGetBufferRaiseOnFailure( sendLength, endpoint->Common.Datagram.RemoteAddressLength, endpoint->OwningProcess ); } except (AFD_EXCEPTION_FILTER (&status)) { goto exit; }
AfdAcquireSpinLock( &endpoint->SpinLock, &lockHandle );
//
// If the endpoint is not connected, fail.
//
if ( endpoint->State != AfdEndpointStateConnected ) { AfdReleaseSpinLock( &endpoint->SpinLock, &lockHandle ); AfdReturnBuffer (&afdBuffer->Header, endpoint->OwningProcess); status = STATUS_INVALID_CONNECTION; goto exit; }
if (afdBuffer->AllocatedAddressLength < endpoint->Common.Datagram.RemoteAddressLength ) { //
// Apparently connection address length has changed
// on us while we were allocating the buffer.
// This is extremely unlikely (even if endpoint got
// connected to a different address, the length is unlikely
// to change), but we must handle this, just try again.
//
AfdReleaseSpinLock( &endpoint->SpinLock, &lockHandle ); AfdReturnBuffer (&afdBuffer->Header, endpoint->OwningProcess); goto retry; } //
// Copy the address to the AFD buffer.
//
RtlCopyMemory( afdBuffer->TdiInfo.RemoteAddress, endpoint->Common.Datagram.RemoteAddress, endpoint->Common.Datagram.RemoteAddressLength );
afdBuffer->TdiInfo.RemoteAddressLength = endpoint->Common.Datagram.RemoteAddressLength;
AfdReleaseSpinLock( &endpoint->SpinLock, &lockHandle ); } else { try { afdBuffer = AfdGetBufferRaiseOnFailure( sendLength, sendInfo->TdiConnInfo.RemoteAddressLength, endpoint->OwningProcess); //
// Copy address if necessary.
//
if (sendInfo->TdiConnInfo.RemoteAddressLength!=0) { RtlCopyMemory( afdBuffer->TdiInfo.RemoteAddress, sendInfo->TdiConnInfo.RemoteAddress, sendInfo->TdiConnInfo.RemoteAddressLength );
//
// Validate internal consistency of the transport address structure.
// Note that we HAVE to do this after copying since the malicious
// application can change the content of the buffer on us any time
// and our check will be bypassed.
//
if ((((PTRANSPORT_ADDRESS)afdBuffer->TdiInfo.RemoteAddress)->TAAddressCount!=1) || (LONG)sendInfo->TdiConnInfo.RemoteAddressLength< FIELD_OFFSET (TRANSPORT_ADDRESS, Address[0].Address[((PTRANSPORT_ADDRESS)afdBuffer->TdiInfo.RemoteAddress)->Address[0].AddressLength])) { ExRaiseStatus (STATUS_INVALID_PARAMETER); } } } except( AFD_EXCEPTION_FILTER(&status) ) { if (afdBuffer!=NULL) { AfdReturnBuffer( &afdBuffer->Header, endpoint->OwningProcess ); } goto exit; }
afdBuffer->TdiInfo.RemoteAddressLength = sendInfo->TdiConnInfo.RemoteAddressLength; }
//
// Copy the output buffer to the AFD buffer.
//
try {
AfdCopyBufferArrayToBuffer( afdBuffer->Buffer, sendLength, sendInfo->BufferArray, sendInfo->BufferCount );
//
// Store the length of the data and the address we're going to
// send.
//
afdBuffer->DataLength = sendLength;
} except( AFD_EXCEPTION_FILTER(&status) ) {
AfdReturnBuffer( &afdBuffer->Header, endpoint->OwningProcess ); goto exit; }
if (IS_TDI_DGRAM_CONNECTION(endpoint) && (afdBuffer->TdiInfo.RemoteAddressLength==0)) { TdiBuildSend( afdBuffer->Irp, endpoint->AddressDeviceObject, endpoint->AddressFileObject, AfdRestartFastDatagramSend, afdBuffer, afdBuffer->Irp->MdlAddress, 0, sendLength ); } else { //
// Set up the input TDI information to point to the destination
// address.
//
afdBuffer->TdiInfo.Options = NULL; afdBuffer->TdiInfo.OptionsLength = 0; afdBuffer->TdiInfo.UserData = NULL; afdBuffer->TdiInfo.UserDataLength = 0;
//
// Initialize the IRP in the AFD buffer to do a fast datagram send.
//
TdiBuildSendDatagram( afdBuffer->Irp, endpoint->AddressDeviceObject, endpoint->AddressFileObject, AfdRestartFastDatagramSend, afdBuffer, afdBuffer->Irp->MdlAddress, sendLength, &afdBuffer->TdiInfo ); }
//
// Change the MDL in the AFD buffer to specify only the number
// of bytes we're actually sending. This is a requirement of TDI--
// the MDL chain cannot describe a longer buffer than the send
// request.
//
afdBuffer->Mdl->ByteCount = sendLength;
//
// Reference the endpoint so that it does not go away until the send
// completes. This is necessary to ensure that a send which takes a
// very long time and lasts longer than the process will not cause a
// crash when the send datragram finally completes.
//
REFERENCE_ENDPOINT2( endpoint, "AfdFastDatagramSend, length", sendLength );
//
// Set the context to NULL initially so that if the IRP is completed
// by the stack before IoCallDriver returns, the completion routine
// does not free the buffer (and IRP in it) and we can figure out
// what the final status of the operation was and report it to the
// application
//
afdBuffer->Context = NULL;
//
// Give the IRP to the TDI provider. If the request fails
// immediately, then fail fast IO. If the request fails later on,
// there's nothing we can do about it.
//
status = IoCallDriver( endpoint->AddressDeviceObject, afdBuffer->Irp );
//
// Check if completion routine has already been called and we
// can figure out what the final status is
//
if (InterlockedCompareExchangePointer ( &afdBuffer->Context, endpoint, NULL)!=NULL) { BOOLEAN indicateSendEvent; //
// Completion routine has been called, pick the final status
// and dereference the endpoint and free the buffer
//
status = afdBuffer->Irp->IoStatus.Status;
AfdAcquireSpinLock (&endpoint->SpinLock, &lockHandle); endpoint->DgBufferredSendBytes -= sendLength; if (endpoint->DgBufferredSendBytes < endpoint->Common.Datagram.MaxBufferredSendBytes || endpoint->DgBufferredSendBytes==0) { indicateSendEvent = TRUE; AfdIndicateEventSelectEvent (endpoint, AFD_POLL_SEND, STATUS_SUCCESS); } else { indicateSendEvent = FALSE; } AfdReleaseSpinLock (&endpoint->SpinLock, &lockHandle); if (indicateSendEvent) { AfdIndicatePollEvent (endpoint, AFD_POLL_SEND, STATUS_SUCCESS); }
AfdReturnBuffer (&afdBuffer->Header, endpoint->OwningProcess);
DEREFERENCE_ENDPOINT2 (endpoint, "AfdFastDatagramSend-inline completion, status", status ); } //else Completion routine has not been called, we set the pointer
// to the endpoint in the buffer context, so it can derefernce it
// and knows to free the buffer
//
if ( NT_SUCCESS(status) ) { IoStatus->Information = sendLength; IoStatus->Status = STATUS_SUCCESS; return TRUE; } else { goto errorset; }
exit: AfdAcquireSpinLock (&endpoint->SpinLock, &lockHandle); endpoint->DgBufferredSendBytes -= sendLength; AfdReleaseSpinLock (&endpoint->SpinLock, &lockHandle);
errorset: // Fast io can't handle error returns
// if call is overlapped (completion port),
if ( endpoint->NonBlocking && !( sendInfo->AfdFlags & AFD_OVERLAPPED ) ) { // We know that it is not overlapped
IoStatus->Status = status; return TRUE; } else { return FALSE; }} // AfdFastDatagramSend
NTSTATUSAfdRestartFastDatagramSend ( IN PDEVICE_OBJECT DeviceObject, IN PIRP Irp, IN PVOID Context ){ PAFD_BUFFER afdBuffer; PAFD_ENDPOINT endpoint; ULONG sendLength; AFD_LOCK_QUEUE_HANDLE lockHandle;
afdBuffer = Context; ASSERT (IS_VALID_AFD_BUFFER (afdBuffer));
//
// Reset the AFD buffer structure.
//
ASSERT( afdBuffer->Irp == Irp );
sendLength = afdBuffer->Mdl->ByteCount; ASSERT (afdBuffer->DataLength==sendLength); afdBuffer->Mdl->ByteCount = afdBuffer->BufferLength;
//
// If call succeeded, transport should have sent the number of bytes requested
//
ASSERT (Irp->IoStatus.Status!=STATUS_SUCCESS || Irp->IoStatus.Information==sendLength); //
// Find the endpoint used for this request if
// the IoCallDriver call has completed already
//
endpoint = InterlockedCompareExchangePointer (&afdBuffer->Context, (PVOID)-1, NULL); if (endpoint!=NULL) { BOOLEAN indicateSendEvent;#if REFERENCE_DEBUG
NTSTATUS status;#endif
//
// IoCallDriver has completed, free the buffer and
// dereference endpoint here
//
ASSERT( IS_DGRAM_ENDPOINT(endpoint) );
AfdAcquireSpinLock (&endpoint->SpinLock, &lockHandle); endpoint->DgBufferredSendBytes -= sendLength; if (endpoint->DgBufferredSendBytes < endpoint->Common.Datagram.MaxBufferredSendBytes || endpoint->DgBufferredSendBytes==0) { AfdIndicateEventSelectEvent (endpoint, AFD_POLL_SEND, STATUS_SUCCESS); indicateSendEvent = TRUE; } else indicateSendEvent = FALSE; AfdReleaseSpinLock (&endpoint->SpinLock, &lockHandle);
if (indicateSendEvent) { AfdIndicatePollEvent (endpoint, AFD_POLL_SEND, STATUS_SUCCESS); } //
// Get rid of the reference we put on the endpoint when we started
// this I/O.
//
#if REFERENCE_DEBUG
status = Irp->IoStatus.Status;#endif
AfdReturnBuffer( &afdBuffer->Header, endpoint->OwningProcess );
DEREFERENCE_ENDPOINT2 (endpoint, "AfdRestartFastDatagramSend, status", status );
} // else IoCallDriver is not done yet, it will free the buffer
// and endpoint when done (it will look at final status and
// report it to the application).
//
// Tell the IO system to stop processing this IRP.
//
return STATUS_MORE_PROCESSING_REQUIRED;
} // AfdRestartFastSendDatagram
BOOLEANAfdFastDatagramReceive ( IN PAFD_ENDPOINT endpoint, IN PAFD_RECV_MESSAGE_INFO msgInfo, IN ULONG recvLength, OUT PIO_STATUS_BLOCK IoStatus ){ AFD_LOCK_QUEUE_HANDLE lockHandle; PLIST_ENTRY listEntry; PAFD_BUFFER_HEADER afdBuffer; PTRANSPORT_ADDRESS tdiAddress; ULONG length;
IF_DEBUG(FAST_IO) { KdPrintEx(( DPFLTR_WSOCKTRANSPORT_ID, DPFLTR_TRACE_LEVEL, "AfdFastDatagramReceive: attempting fast IO on endp %p\n", endpoint)); }
AfdAcquireSpinLock( &endpoint->SpinLock, &lockHandle );
//
// If there are no datagrams available to be received, don't
// bother with the fast path.
//
if ( !ARE_DATAGRAMS_ON_ENDPOINT( endpoint ) ) {
//
// If this is a nonblocking endpoint, fail the request here and
// save going through the regular path.
//
if ( endpoint->NonBlocking && !( msgInfo->dgi.AfdFlags & AFD_OVERLAPPED ) ) { endpoint->EventsActive &= ~AFD_POLL_RECEIVE;
IF_DEBUG(EVENT_SELECT) { KdPrintEx(( DPFLTR_WSOCKTRANSPORT_ID, DPFLTR_TRACE_LEVEL, "AfdFastDatagramReceive: Endp %p, Active %lX\n", endpoint, endpoint->EventsActive )); }
// Fast io can't handle error returns
// if call is overlapped (completion port), but we know here
// that call is not overlapped
IoStatus->Status = STATUS_DEVICE_NOT_READY; AfdReleaseSpinLock( &endpoint->SpinLock, &lockHandle ); return TRUE; }
AfdReleaseSpinLock( &endpoint->SpinLock, &lockHandle ); return FALSE; }
//
// There is at least one datagram bufferred on the endpoint. Use it
// for this receive.
//
listEntry = RemoveHeadList( &endpoint->ReceiveDatagramBufferListHead ); afdBuffer = CONTAINING_RECORD( listEntry, AFD_BUFFER_HEADER, BufferListEntry );
//
// If the datagram is too large or it is an error indication
// fail fast IO.
//
if ( (afdBuffer->DataLength > recvLength) || !NT_SUCCESS (afdBuffer->Status)) { InsertHeadList( &endpoint->ReceiveDatagramBufferListHead, &afdBuffer->BufferListEntry ); AfdReleaseSpinLock( &endpoint->SpinLock, &lockHandle ); return FALSE; }
//
// Update counts of bufferred datagrams and bytes on the endpoint.
//
endpoint->DgBufferredReceiveCount--; endpoint->DgBufferredReceiveBytes -= afdBuffer->DataLength;
//
// Release the lock and copy the datagram into the user buffer. We
// can't continue to hold the lock, because it is not legal to take
// an exception at raised IRQL. Releasing the lock may result in a
// misordered datagram if there is an exception in copying to the
// user's buffer, but that is the application's fault for giving us a bogus
// pointer. Besides, datagram order is not guaranteed.
//
AfdReleaseSpinLock( &endpoint->SpinLock, &lockHandle );
try {
if (afdBuffer->DataLength>0) { AfdCopyMdlChainToBufferArray( msgInfo->dgi.BufferArray, 0, msgInfo->dgi.BufferCount, afdBuffer->Mdl, 0, afdBuffer->DataLength ); }
//
// If we need to return the source address, copy it to the
// user's output buffer.
//
if ( msgInfo->dgi.Address != NULL ) {
tdiAddress = afdBuffer->TdiInfo.RemoteAddress;
length = tdiAddress->Address[0].AddressLength + sizeof(u_short); // sa_family
if( *msgInfo->dgi.AddressLength < length ) {
ExRaiseAccessViolation();
}
if (ExGetPreviousMode ()!=KernelMode) { ProbeForWrite (msgInfo->dgi.Address, length, sizeof (UCHAR)); }
RtlCopyMemory( msgInfo->dgi.Address, &tdiAddress->Address[0].AddressType, length );
*msgInfo->dgi.AddressLength = length; }
if (msgInfo->ControlLength!=NULL) { if (afdBuffer->DatagramFlags & TDI_RECEIVE_CONTROL_INFO && afdBuffer->DataOffset>0) { PAFD_BUFFER buf = CONTAINING_RECORD (afdBuffer, AFD_BUFFER, Header); ASSERT (msgInfo->MsgFlags!=NULL); ASSERT (buf->BufferLength != AfdBufferTagSize); length = buf->DataOffset;#ifdef _WIN64
if (IoIs32bitProcess (NULL)) { length = AfdComputeCMSGLength32 ( (PUCHAR)buf->Buffer+afdBuffer->DataLength, length);
if (length>*msgInfo->ControlLength) { ExRaiseAccessViolation (); } if (ExGetPreviousMode ()!=KernelMode) { ProbeForWrite (msgInfo->ControlBuffer, length, sizeof (UCHAR)); } AfdCopyCMSGBuffer32 ( msgInfo->ControlBuffer, (PUCHAR)buf->Buffer+afdBuffer->DataLength, length); } else#endif // _WIN64
{ if (length>*msgInfo->ControlLength) { ExRaiseAccessViolation (); }
if (ExGetPreviousMode ()!=KernelMode) { ProbeForWrite (msgInfo->ControlBuffer, length, sizeof (UCHAR)); }
RtlCopyMemory( msgInfo->ControlBuffer, (PUCHAR)buf->Buffer+afdBuffer->DataLength, length ); }
} else { length = 0; }
*msgInfo->ControlLength = length; }
if (msgInfo->MsgFlags!=NULL) { ULONG flags = 0; if (afdBuffer->DatagramFlags & TDI_RECEIVE_BROADCAST) flags |= MSG_BCAST; if (afdBuffer->DatagramFlags & TDI_RECEIVE_MULTICAST) flags |= MSG_MCAST; *msgInfo->MsgFlags = flags; }
IoStatus->Information = afdBuffer->DataLength; IoStatus->Status = STATUS_SUCCESS;
} except( AFD_EXCEPTION_FILTER(NULL) ) {
//
// Put the buffer back on the endpoint's list.
//
AfdAcquireSpinLock( &endpoint->SpinLock, &lockHandle );
InsertHeadList( &endpoint->ReceiveDatagramBufferListHead, &afdBuffer->BufferListEntry );
endpoint->DgBufferredReceiveCount++; endpoint->DgBufferredReceiveBytes += afdBuffer->DataLength;
AfdReleaseSpinLock( &endpoint->SpinLock, &lockHandle );
// Fast io can't handle error returns
// if call is overlapped (completion port)
// IoStatus->Status = GetExceptionCode ();
return FALSE; }
//
// Clear the receive data active bit. If there's more data
// available, set the corresponding event.
//
AfdAcquireSpinLock( &endpoint->SpinLock, &lockHandle );
endpoint->EventsActive &= ~AFD_POLL_RECEIVE;
if( ARE_DATAGRAMS_ON_ENDPOINT( endpoint ) ) {
AfdIndicateEventSelectEvent( endpoint, AFD_POLL_RECEIVE, STATUS_SUCCESS );
} else { //
// Disable fast IO path to avoid performance penalty
// of going through it.
//
if (!endpoint->NonBlocking) endpoint->DisableFastIoRecv = TRUE; }
AfdReleaseSpinLock( &endpoint->SpinLock, &lockHandle );
//
// The fast IO worked! Clean up and return to the user.
//
AfdReturnBuffer( afdBuffer, endpoint->OwningProcess );
ASSERT (IoStatus->Status == STATUS_SUCCESS); return TRUE;
} // AfdFastDatagramReceive
BOOLEANAfdShouldSendBlock ( IN PAFD_ENDPOINT Endpoint, IN PAFD_CONNECTION Connection, IN ULONG SendLength )
/*++
Routine Description:
Determines whether a nonblocking send can be performed on the connection, and if the send is possible, updates the connection's send tracking information.
Arguments:
Endpoint - the AFD endpoint for the send.
Connection - the AFD connection for the send.
SendLength - the number of bytes that the caller wants to send.
Return Value:
TRUE if the there is not too much data on the endpoint to perform the send; FALSE otherwise.
Note: This routine assumes that endpoint spinlock is held when calling it.
--*/
{
//
// Determine whether we can do fast IO with this send. In order
// to perform fast IO, there must be no other sends pended on this
// connection and there must be enough space left for bufferring
// the requested amount of data.
//
if ( !IsListEmpty( &Connection->VcSendIrpListHead )
||
Connection->VcBufferredSendBytes >= Connection->MaxBufferredSendBytes ) {
//
// If this is a nonblocking endpoint, fail the request here and
// save going through the regular path.
//
if ( Endpoint->NonBlocking ) { Endpoint->EventsActive &= ~AFD_POLL_SEND; Endpoint->EnableSendEvent = TRUE;
IF_DEBUG(EVENT_SELECT) { KdPrintEx(( DPFLTR_WSOCKTRANSPORT_ID, DPFLTR_TRACE_LEVEL, "AfdFastIoDeviceControl: Endp %p, Active %lX\n", Endpoint, Endpoint->EventsActive )); } }
return TRUE; }
//
// Update count of send bytes pending on the connection.
//
Connection->VcBufferredSendBytes += SendLength; Connection->VcBufferredSendCount += 1;
//
// Indicate to the caller that it is OK to proceed with the send.
//
return FALSE;
} // AfdShouldSendBlock
|
