/*++

Copyright (c) 1989  Microsoft Corporation

Module Name:

    buffer.c

Abstract:

    This module contains routines for handling non-bufferring TDI
    providers.  The AFD interface assumes that bufferring will be done
    below AFD; if the TDI provider doesn't buffer, then AFD must.

Author:

    David Treadwell (davidtr)    21-Feb-1992

Revision History:

--*/

#include "afdp.h"

PAFD_BUFFER
AfdInitializeBuffer (
    IN PVOID MemBlock,
    IN ULONG BufferDataSize,
    IN ULONG AddressSize,
    IN CCHAR StackSize
    );

VOID
AfdInitializeBufferTag (
    IN PAFD_BUFFER_TAG AfdBufferTag,
    IN ULONG AddressSize
    );

PAFD_BUFFER
AfdGetBufferSlow (
    IN ULONG BufferDataSize,
    IN ULONG AddressSize,
    IN PEPROCESS Process,
    IN CCHAR StackSize
    );


#if DBG
VOID
AfdFreeBufferReal (
    PVOID   AfdBuffer
    );
#endif

#ifdef ALLOC_PRAGMA
#pragma alloc_text( PAGEAFD, AfdAllocateBuffer )
#pragma alloc_text( PAGEAFD, AfdFreeBuffer )
#if DBG
#pragma alloc_text( PAGEAFD, AfdFreeBufferReal )
#endif
#pragma alloc_text( PAGEAFD, AfdCalculateBufferSize )
#pragma alloc_text( PAGEAFD, AfdInitializeBuffer )
#pragma alloc_text( PAGEAFD, AfdGetBufferFast )
#ifdef _AFD_VARIABLE_STACK_
#pragma alloc_text( PAGEAFD, AfdGetBufferWithMaxStackSize )
#endif //_AFD_VARIABLE_STACK_
#pragma alloc_text( PAGEAFD, AfdGetBufferSlow )
#pragma alloc_text( PAGEAFD, AfdReturnBuffer )
#pragma alloc_text( PAGEAFD, AfdAllocateBufferTag )
#pragma alloc_text( PAGEAFD, AfdFreeBufferTag )
#pragma alloc_text( PAGEAFD, AfdAllocateRemoteAddress )
#pragma alloc_text( PAGEAFD, AfdFreeRemoteAddress )
#pragma alloc_text( PAGEAFD, AfdInitializeBufferTag )
#pragma alloc_text( PAGEAFD, AfdGetBufferTag )
#pragma alloc_text( INIT, AfdInitializeBufferManager)
#endif


PVOID
AfdAllocateBuffer (
    IN POOL_TYPE PoolType,
    IN SIZE_T NumberOfBytes,
    IN ULONG Tag
    )

/*++

Routine Description:

    Used by the lookaside list allocation function to allocate a new
    AFD buffer structure.  The returned structure will be fully
    initialized.

Arguments:

    PoolType - passed to ExAllocatePoolWithTag.

    NumberOfBytes - the number of bytes required for the data buffer
        portion of the AFD buffer.

    Tag - passed to ExAllocatePoolWithTag.

Return Value:

    PVOID - a fully initialized PAFD_BUFFER, or NULL if the allocation
        attempt fails.

--*/

{
    ULONG       dataLength;
    PVOID       memBlock;

    //
    // Get nonpaged pool for the buffer.
    //

    memBlock = AFD_ALLOCATE_POOL( PoolType, NumberOfBytes, Tag );
    if ( memBlock == NULL ) {
        return NULL;
    }

    if (NumberOfBytes==AfdLookasideLists->SmallBufferList.L.Size) {
        dataLength = AfdSmallBufferSize;
    }
    else if (NumberOfBytes==AfdLookasideLists->MediumBufferList.L.Size) {
        dataLength = AfdMediumBufferSize;
    }
    else if (NumberOfBytes==AfdLookasideLists->LargeBufferList.L.Size) {
        dataLength = AfdLargeBufferSize;
    }
    else {
        ASSERT (!"Unknown buffer length");
        return NULL;
    }
    //
    // Initialize the buffer and return a pointer to it.
    //
#if DBG
    {
        PAFD_BUFFER afdBuffer = AfdInitializeBuffer( memBlock, 
                                                    dataLength, 
                                                    AfdStandardAddressLength, 
                                                    AfdTdiStackSize );
        ASSERT ((PCHAR)afdBuffer+sizeof (AFD_BUFFER)<=(PCHAR)memBlock+NumberOfBytes &&
                    (PCHAR)afdBuffer->Buffer+dataLength<=(PCHAR)memBlock+NumberOfBytes &&
                    (PCHAR)afdBuffer->Irp+IoSizeOfIrp(AfdTdiStackSize)<=(PCHAR)memBlock+NumberOfBytes &&
                    (PCHAR)afdBuffer->Mdl+MmSizeOfMdl(afdBuffer->Buffer, dataLength)<=(PCHAR)memBlock+NumberOfBytes &&
                    (PCHAR)afdBuffer->TdiInfo.RemoteAddress+AfdStandardAddressLength<=(PCHAR)memBlock+NumberOfBytes);
        return afdBuffer;
    }
#else
    return AfdInitializeBuffer( memBlock, dataLength, AfdStandardAddressLength, AfdTdiStackSize );
#endif


} // AfdAllocateBuffer


VOID
NTAPI
AfdFreeBuffer (
    PVOID   AfdBuffer
    )
{
    ASSERT( ((PAFD_BUFFER)AfdBuffer)->BufferLength == AfdSmallBufferSize ||
            ((PAFD_BUFFER)AfdBuffer)->BufferLength == AfdMediumBufferSize ||
            ((PAFD_BUFFER)AfdBuffer)->BufferLength == AfdLargeBufferSize );
#if DBG
    AfdFreeBufferReal (AfdBuffer);
}

VOID
NTAPI
AfdFreeBufferReal (
    PVOID   AfdBuffer
    )
{
#endif
    {
        PAFD_BUFFER hdr = AfdBuffer;
        switch (hdr->Placement) {
        case AFD_PLACEMENT_BUFFER:
            AfdBuffer = hdr->Buffer;
            break;
        case AFD_PLACEMENT_HDR:
            AfdBuffer = hdr;
            break;
        case AFD_PLACEMENT_MDL:
            AfdBuffer = hdr->Mdl;
            break;
        case AFD_PLACEMENT_IRP:
            AfdBuffer = hdr->Irp;
            break;
        default:
            ASSERT (!"Unknown placement!");
            __assume (0);
        }
        if (hdr->AlignmentAdjusted) {
            //
            // The original memory block was adjusted to meet alignment
            // requirement of AFD buffers.
            // The amount of adjustment should be stored in the space
            // used for adjustment (right below the first piece).
            //
            ASSERT ((*(((PSIZE_T)AfdBuffer)-1))>0 &&
                        (*(((PSIZE_T)AfdBuffer)-1))<AfdBufferAlignment);
            AfdBuffer = (PUCHAR)AfdBuffer - (*(((PSIZE_T)AfdBuffer)-1));
        }
        AFD_FREE_POOL (AfdBuffer, AFD_DATA_BUFFER_POOL_TAG);
    }
}


PVOID
AfdAllocateBufferTag (
    IN POOL_TYPE PoolType,
    IN SIZE_T NumberOfBytes,
    IN ULONG Tag
    )

/*++

Routine Description:

    Used by the lookaside list allocation function to allocate a new
    AFD buffer tag structure.  The returned structure will be fully
    initialized.

Arguments:

    PoolType - passed to ExAllocatePoolWithTag.

    NumberOfBytes - the number of bytes required for the data buffer
        portion of the AFD buffer tag (0).

    Tag - passed to ExAllocatePoolWithTag.

Return Value:

    PVOID - a fully initialized PAFD_BUFFER_TAG, or NULL if the allocation
        attempt fails.

--*/

{
    PAFD_BUFFER_TAG afdBufferTag;

    //
    // The requested length must be the same as buffer tag size
    //

    ASSERT(NumberOfBytes == sizeof (AFD_BUFFER_TAG) );

    //
    // Get nonpaged pool for the buffer tag.
    //

    afdBufferTag = AFD_ALLOCATE_POOL( PoolType, NumberOfBytes, Tag );
    if ( afdBufferTag == NULL ) {
        return NULL;
    }

    //
    // Initialize the buffer tag and return a pointer to it.
    //

    AfdInitializeBufferTag( afdBufferTag, 0 );

    return afdBufferTag;


} // AfdAllocateBufferTag

VOID
NTAPI
AfdFreeBufferTag (
    PVOID   AfdBufferTag
    )
{
    AFD_FREE_POOL (AfdBufferTag, AFD_DATA_BUFFER_POOL_TAG);
}


PVOID
AfdAllocateRemoteAddress (
    IN POOL_TYPE PoolType,
    IN SIZE_T NumberOfBytes,
    IN ULONG Tag
    )

/*++

Routine Description:

    Used by the lookaside list allocation function to allocate a new
    remote address structure.  The returned structure will be fully
    initialized.

Arguments:

    PoolType - passed to ExAllocatePoolWithTag.

    NumberOfBytes - the number of bytes required for the data buffer
        portion of the AFD buffer tag (0).

    Tag - passed to ExAllocatePoolWithTag.

Return Value:

    PVOID - a fully initialized remote address, or NULL if the allocation
        attempt fails.

--*/

{
    //
    // The requested length must be the same as standard address size
    //

    ASSERT(NumberOfBytes == AfdStandardAddressLength );

    //
    // Get nonpaged pool for the remote address.
    //

    return AFD_ALLOCATE_POOL( PoolType, NumberOfBytes, Tag );


} // AfdAllocateRemoteAddress

VOID
NTAPI
AfdFreeRemoteAddress (
    PVOID   AfdBufferTag
    )
{
    AFD_FREE_POOL (AfdBufferTag, AFD_REMOTE_ADDRESS_POOL_TAG);
}


ULONG
AfdCalculateBufferSize (
    IN ULONG BufferDataSize,
    IN ULONG AddressSize,
    IN CCHAR StackSize
    )

/*++

Routine Description:

    Determines the size of an AFD buffer structure given the amount of
    data that the buffer contains.

Arguments:

    BufferDataSize - data length of the buffer.

    AddressSize - length of address structure for the buffer.

Return Value:

    Number of bytes needed for an AFD_BUFFER structure for data of
    this size.

--*/

{
    ULONG irpSize;
    ULONG mdlSize;
    ULONG hdrSize;
    ULONG size;

    //
    // Determine the sizes of the various components of an AFD_BUFFER
    // structure.
    //

    hdrSize = sizeof (AFD_BUFFER);
    irpSize = IoSizeOfIrp( StackSize );
    //
    // For mdl size calculation we rely on ex guarantee that buffer will be
    // aligned on the page boundary (for allocations >= PAGE_SIZE)
    // or will not spawn pages (for allocations < PAGE_SIZE).
    //
    mdlSize = (CLONG)MmSizeOfMdl( NULL, BufferDataSize );

    size = ALIGN_UP_A(hdrSize,AFD_MINIMUM_BUFFER_ALIGNMENT) +
                ALIGN_UP_A(irpSize,AFD_MINIMUM_BUFFER_ALIGNMENT) +
                ALIGN_UP_A(mdlSize,AFD_MINIMUM_BUFFER_ALIGNMENT) +
                ALIGN_UP_A(BufferDataSize,AFD_MINIMUM_BUFFER_ALIGNMENT) +
                AddressSize;
    if (size>=PAGE_SIZE) {
        return size;
    }
    else {
        if (StackSize==AfdTdiStackSize) {
            size += AfdAlignmentOverhead;
        }
        else {
            size += AfdBufferAlignment-AFD_MINIMUM_BUFFER_ALIGNMENT;
        }

        if (size>=PAGE_SIZE) {
            //
            // PAGE_SIZE allocations are aligned, ignore any extra overhead
            //
            return PAGE_SIZE;
        }
        else {
            return size;
        }
    }
} // AfdCalculateBufferSize


PAFD_BUFFER
FASTCALL
AfdGetBufferFast (
    IN ULONG BufferDataSize,
    IN ULONG  AddressSize,
    IN PEPROCESS Process
    )

/*++

Routine Description:

    Obtains a buffer of the appropriate size for the caller.  Uses
    the preallocated buffers if possible, or else allocates a new buffer
    structure if required.

Arguments:

    BufferDataSize - the size of the data buffer that goes along with the
        buffer structure.

    AddressSize - size of the address field required for the buffer.

Return Value:

    PAFD_BUFFER - a pointer to an AFD_BUFFER structure, or NULL if one
        was not available or could not be allocated.

--*/

{
    PAFD_BUFFER afdBuffer;
    NTSTATUS    status;

    //
    // If possible, allocate the buffer from one of the lookaside lists.
    //

    if ( AddressSize <= AfdStandardAddressLength &&
             BufferDataSize <= AfdLargeBufferSize ) {
        PNPAGED_LOOKASIDE_LIST lookasideList;

        if ( BufferDataSize <= AfdSmallBufferSize ) {

            lookasideList = &AfdLookasideLists->SmallBufferList;

        } else if ( BufferDataSize <= AfdMediumBufferSize ) {

            lookasideList = &AfdLookasideLists->MediumBufferList;

        } else {

            lookasideList = &AfdLookasideLists->LargeBufferList;
        }

        afdBuffer = ExAllocateFromNPagedLookasideList( lookasideList );
        if ( afdBuffer != NULL) {

            if (!afdBuffer->Lookaside) {
                status = PsChargeProcessPoolQuota (
                                (PEPROCESS)((ULONG_PTR)Process & (~AFDB_RAISE_ON_FAILURE)),
                                NonPagedPool,
                                lookasideList->L.Size);

                if (!NT_SUCCESS (status)) {
                    AfdFreeBuffer (afdBuffer);
                    goto ExitQuotaFailure;
                }

                AfdRecordQuotaHistory(
                    process,
                    (LONG)lookasideList->L.Size,
                    "BuffAlloc   ",
                    afdBuffer
                    );
                AfdRecordPoolQuotaCharged(lookasideList->L.Size);
            }

#if DBG
            RtlGetCallersAddress(
                &afdBuffer->Caller,
                &afdBuffer->CallersCaller
                );
#endif
            return afdBuffer;
        }
    }
    else {
        afdBuffer = AfdGetBufferSlow (BufferDataSize, AddressSize, Process, AfdTdiStackSize);
#if DBG
        if (afdBuffer!=NULL) {
            RtlGetCallersAddress(
                &afdBuffer->Caller,
                &afdBuffer->CallersCaller
                );
        }
#endif
        return afdBuffer;
    }

    status = STATUS_INSUFFICIENT_RESOURCES;

ExitQuotaFailure:
    if ((ULONG_PTR)Process & AFDB_RAISE_ON_FAILURE) {
        ExRaiseStatus (status);
    }

    return NULL;
} // AfdGetBuffer

#ifdef _AFD_VARIABLE_STACK_
PAFD_BUFFER
FASTCALL
AfdGetBufferWithMaxStackSize (
    IN ULONG BufferDataSize,
    IN ULONG AddressSize,
    IN PEPROCESS Process
    )
{
    PAFD_BUFFER afdBuffer;

    afdBuffer = AfdGetBufferSlow (BufferDataSize, AddressSize, Process, AfdMaxStackSize);
#if DBG
    if (afdBuffer!=NULL) {
            RtlGetCallersAddress(
                &afdBuffer->Caller,
                &afdBuffer->CallersCaller
                );
    }
#endif
    return afdBuffer;
}
#endif // _AFD_VARIABLE_STACK_

PAFD_BUFFER
AfdGetBufferSlow (
    IN ULONG BufferDataSize,
    IN ULONG AddressSize,
    IN PEPROCESS Process,
    IN CCHAR StackSize
    )
{
    NTSTATUS    status;

    if (AddressSize<=0xFFFF) {
        PVOID       memBlock;
        LONG        sz;
        ULONG       bufferSize;

        //
        // Couldn't find an appropriate buffer that was preallocated.
        // Allocate one manually.  If the buffer size requested was
        // zero bytes, give them four bytes.  This is because some of
        // the routines like MmSizeOfMdl() cannot handle getting passed
        // in a length of zero.
        //
        // !!! It would be good to ROUND_TO_PAGES for this allocation
        //     if appropriate, then use entire buffer size.
        //

        if ( BufferDataSize == 0 ) {
            BufferDataSize = sizeof(ULONG);
        }

        bufferSize = AfdCalculateBufferSize( BufferDataSize, AddressSize, StackSize );

        //
        // Check for overflow (stack size comes from trusted source - no need to check
        // for overflow).
        //
        if (bufferSize>=BufferDataSize && bufferSize>=AddressSize) {

            memBlock = AFD_ALLOCATE_POOL(
                            NonPagedPool,
                            bufferSize,
                            AFD_DATA_BUFFER_POOL_TAG
                            );

            if ( memBlock != NULL) {
                status = PsChargeProcessPoolQuota (
                                (PEPROCESS)((ULONG_PTR)Process & (~AFDB_RAISE_ON_FAILURE)),
                                NonPagedPool,
                                sz = BufferDataSize
                                    +AfdBufferOverhead
                                    +AddressSize
                                    -AfdStandardAddressLength
                                    +BufferDataSize<PAGE_SIZE
                                        ? min (AfdAlignmentOverhead, PAGE_SIZE-BufferDataSize)
                                        : 0);
                if (NT_SUCCESS (status)) {
                    PAFD_BUFFER afdBuffer;

                    //
                    // Initialize the AFD buffer structure and return it.
                    //

                    afdBuffer = AfdInitializeBuffer( memBlock,
                                                    BufferDataSize,
                                                    AddressSize, 
                                                    StackSize);

                    ASSERT ((PCHAR)afdBuffer+sizeof (AFD_BUFFER)<=(PCHAR)memBlock+bufferSize &&
                                (PCHAR)afdBuffer->Buffer+BufferDataSize<=(PCHAR)memBlock+bufferSize &&
                                (PCHAR)afdBuffer->Irp+IoSizeOfIrp(StackSize)<=(PCHAR)memBlock+bufferSize &&
                                (PCHAR)afdBuffer->Mdl+MmSizeOfMdl(afdBuffer->Buffer, BufferDataSize)<=(PCHAR)memBlock+bufferSize &&
                                (PCHAR)afdBuffer->TdiInfo.RemoteAddress+AddressSize<=(PCHAR)memBlock+bufferSize);

                    AfdRecordPoolQuotaCharged(sz);
                
                    AfdRecordQuotaHistory(
                        process,
                        sz,
                        "BuffAlloc   ",
                        afdBuffer
                        );

                    return afdBuffer;
                }
                else {
                    AFD_FREE_POOL (memBlock, AFD_DATA_BUFFER_POOL_TAG);
                    goto ExitQuotaFailure;
                }
            } // memblock==NULL
        } // overflow
    }
    else {
        // TDI does not support addresses > USHORT
        ASSERT (FALSE);
    }

    //
    // This is default status code.
    // Quota failures jump directly to the
    // label below to raise the status returned by
    // the quota charging code if requested by the caller..
    //
    status = STATUS_INSUFFICIENT_RESOURCES;

ExitQuotaFailure:
    if ((ULONG_PTR)Process & AFDB_RAISE_ON_FAILURE) {
        ExRaiseStatus (status);
    }
    return NULL;
}


PAFD_BUFFER_TAG
AfdGetBufferTag (
    IN ULONG AddressSize,
    IN PEPROCESS Process
    )

/*++

Routine Description:

    Obtains a buffer for tagging TDSU received via chained indication.  Uses
    the preallocated buffers if possible, or else allocates a new buffer
    structure if required.

Arguments:

    AddressSize - size of the address field required for the buffer.

Return Value:

    PAFD_BUFFER_TAG - a pointer to an AFD_BUFFER_TAG structure, or NULL if one
        was not available or could not be allocated.

--*/

{
    PAFD_BUFFER_TAG afdBufferTag;
    ULONG           bufferSize;
    NTSTATUS        status;

    if ( AddressSize <= AfdStandardAddressLength) {
        if (AddressSize>0)
            AddressSize = AfdStandardAddressLength;
        afdBufferTag = ExAllocateFromNPagedLookasideList( 
                                &AfdLookasideLists->BufferTagList );
        if ( afdBufferTag != NULL &&
                ( AddressSize==0 || 
                    (afdBufferTag->TdiInfo.RemoteAddress = 
                                ExAllocateFromNPagedLookasideList( 
                                &AfdLookasideLists->RemoteAddrList ))!=NULL ) ) {

            afdBufferTag->AllocatedAddressLength = (USHORT)AddressSize;
            if (!afdBufferTag->Lookaside) {
                status = PsChargeProcessPoolQuota (
                                (PEPROCESS)((ULONG_PTR)Process & (~AFDB_RAISE_ON_FAILURE)),
                                NonPagedPool,
                                sizeof (AFD_BUFFER_TAG)+AddressSize);
                if (!NT_SUCCESS (status)) {
                    if ((afdBufferTag->TdiInfo.RemoteAddress!=NULL) &&
                            (afdBufferTag->TdiInfo.RemoteAddress != (PVOID)(afdBufferTag+1))) {
                        ExFreeToNPagedLookasideList( &AfdLookasideLists->RemoteAddrList, 
                                                            afdBufferTag->TdiInfo.RemoteAddress );
                    }
                    AFD_FREE_POOL (afdBufferTag, AFD_DATA_BUFFER_POOL_TAG);
                    goto ExitQuotaFailure;
                }

                AfdRecordQuotaHistory(
                    process,
                    (LONG)(sizeof (AFD_BUFFER_TAG)+AddressSize),
                    "BuffAlloc   ",
                    afdBufferTag
                    );
                AfdRecordPoolQuotaCharged(sizeof (AFD_BUFFER_TAG)+AddressSize);
            }
#if DBG
            RtlGetCallersAddress(
                &afdBufferTag->Caller,
                &afdBufferTag->CallersCaller
                );
#endif
            return afdBufferTag;
        } // afdBufferTag==NULL || RemoteAddress==NULL
    }
    else if (AddressSize<=0xFFFF) {
        bufferSize = sizeof (AFD_BUFFER_TAG) + AddressSize;

        afdBufferTag = AFD_ALLOCATE_POOL(
                        NonPagedPool,
                        bufferSize,
                        AFD_DATA_BUFFER_POOL_TAG
                        );

        if (afdBufferTag!=NULL) {
            status = PsChargeProcessPoolQuota (
                                (PEPROCESS)((ULONG_PTR)Process & (~AFDB_RAISE_ON_FAILURE)),
                                NonPagedPool,
                                bufferSize);
            if (NT_SUCCESS (status)) {

                //
                // Initialize the AFD buffer structure and return it.
                //

                AfdInitializeBufferTag (afdBufferTag, AddressSize);
                AfdRecordQuotaHistory(
                    process,
                    (LONG)bufferSize,
                    "BuffAlloc   ",
                    afdBufferTag
                    );

                AfdRecordPoolQuotaCharged(bufferSize);
#if DBG
                RtlGetCallersAddress(
                    &afdBufferTag->Caller,
                    &afdBufferTag->CallersCaller
                    );
#endif
                return afdBufferTag;
            }
            else {
                AFD_FREE_POOL (afdBufferTag, AFD_DATA_BUFFER_POOL_TAG);
                goto ExitQuotaFailure;
            }
        }
    }
    else {
        // TDI does not support addresses > USHORT
        ASSERT (FALSE);
    }

    //
    // This is default status code.
    // Quota failures jump directly to the
    // label below to raise the status returned by
    // the quota charging code if requested by the caller..
    //

    status = STATUS_INSUFFICIENT_RESOURCES;

ExitQuotaFailure:

    if ((ULONG_PTR)Process & AFDB_RAISE_ON_FAILURE) {
        ExRaiseStatus (status);
    }

    return NULL;
}


VOID
AfdReturnBuffer (
    IN PAFD_BUFFER_HEADER AfdBufferHeader,
    IN PEPROCESS Process
    )

/*++

Routine Description:

    Returns an AFD buffer to the appropriate global list, or frees
    it if necessary.

Arguments:

    AfdBufferHeader - points to the AFD_BUFFER_HEADER structure to return or free.

Return Value:

    None.

--*/

{

    
    if (AfdBufferHeader->BufferLength!=AfdBufferTagSize) {
        PNPAGED_LOOKASIDE_LIST lookasideList;
        PAFD_BUFFER AfdBuffer = CONTAINING_RECORD (AfdBufferHeader, AFD_BUFFER, Header);

        ASSERT (IS_VALID_AFD_BUFFER (AfdBuffer));
        
        //
        // Most of the AFD buffer must be zeroed when returning the buffer.
        //

        ASSERT( !AfdBuffer->ExpeditedData );
        ASSERT( AfdBuffer->Mdl->ByteCount == AfdBuffer->BufferLength );
        ASSERT( AfdBuffer->Mdl->Next == NULL );

        //
        // If appropriate, return the buffer to one of the AFD buffer
        // lookaside lists.
        //
#ifdef _AFD_VARIABLE_STACK_
        if (AfdBuffer->Irp->StackCount==AfdTdiStackSize &&
                    AfdBuffer->AllocatedAddressLength == AfdStandardAddressLength &&
                    AfdBuffer->BufferLength <= AfdLargeBufferSize) {
#else // _AFD_VARIABLE_STACK_
        ASSERT (AfdBuffer->Irp->StackCount==AfdTdiStackSize);
        if (AfdBuffer->AllocatedAddressLength == AfdStandardAddressLength &&
                    AfdBuffer->BufferLength <= AfdLargeBufferSize) {
#endif // _AFD_VARIABLE_STACK_

            if (AfdBuffer->BufferLength==AfdSmallBufferSize) {
                lookasideList = &AfdLookasideLists->SmallBufferList;
            } else if (AfdBuffer->BufferLength == AfdMediumBufferSize) {
                lookasideList = &AfdLookasideLists->MediumBufferList;
            } else { 
                ASSERT (AfdBuffer->BufferLength==AfdLargeBufferSize);
                lookasideList = &AfdLookasideLists->LargeBufferList;
            }

            if (!AfdBuffer->Lookaside) {
                PsReturnPoolQuota (Process, NonPagedPool, lookasideList->L.Size);
                AfdRecordQuotaHistory(
                    Process,
                    -(LONG)lookasideList->L.Size,
                    "BuffDealloc ",
                    AfdBuffer
                    );
                AfdRecordPoolQuotaReturned(
                    lookasideList->L.Size
                    );
                AfdBuffer->Lookaside = TRUE;
            }
            ExFreeToNPagedLookasideList( lookasideList, AfdBuffer );

            return;

        }

        // The buffer was not from a lookaside list allocation, so just free
        // the pool we used for it.
        //

#ifdef _AFD_VARIABLE_STACK_
        ASSERT (AfdBuffer->Irp->StackCount>=AfdTdiStackSize &&
                    AfdBuffer->Irp->StackCount<=AfdMaxStackSize);
#endif // _AFD_VARIABLE_STACK_
        ASSERT (AfdBuffer->Lookaside==FALSE);
        {
            LONG    sz;
            PsReturnPoolQuota (Process,
                                  NonPagedPool,
                                  sz=AfdBuffer->BufferLength
                                        +AfdBufferOverhead
                                        +AfdBuffer->AllocatedAddressLength
                                        -AfdStandardAddressLength
                                        +AfdBuffer->BufferLength<PAGE_SIZE 
                                            ? min (AfdAlignmentOverhead, PAGE_SIZE-AfdBuffer->BufferLength)
                                            : 0);
            AfdRecordQuotaHistory(
                Process,
                -(LONG)sz,
                "BuffDealloc ",
                AfdBuffer
                );
            AfdRecordPoolQuotaReturned(
                sz
                );
        }
#if DBG
        AfdFreeBufferReal (AfdBuffer);
#else
        AfdFreeBuffer (AfdBuffer);
#endif

        return;
    }
    else {
        PAFD_BUFFER_TAG AfdBufferTag = CONTAINING_RECORD (AfdBufferHeader, AFD_BUFFER_TAG, Header);

        ASSERT( !AfdBufferTag->ExpeditedData );

        if (AfdBufferTag->NdisPacket) {
            AfdBufferTag->NdisPacket = FALSE;
            TdiReturnChainedReceives (&AfdBufferTag->Context, 1);
        }

        if (AfdBufferTag->TdiInfo.RemoteAddress != (PVOID)(AfdBufferTag+1)) {
            if (AfdBufferTag->TdiInfo.RemoteAddress!=NULL) {
                ASSERT (AfdBufferTag->AllocatedAddressLength==AfdStandardAddressLength);
                ExFreeToNPagedLookasideList( &AfdLookasideLists->RemoteAddrList, 
                                                    AfdBufferTag->TdiInfo.RemoteAddress );
                AfdBufferTag->TdiInfo.RemoteAddress = NULL;
            }
            else {
                ASSERT (AfdBufferTag->AllocatedAddressLength==0);
            }

            if (!AfdBufferTag->Lookaside) {
                LONG    sz;
                PsReturnPoolQuota (
                                Process,
                                NonPagedPool,
                                sz=sizeof (AFD_BUFFER_TAG) 
                                        + AfdBufferTag->AllocatedAddressLength);
                AfdRecordQuotaHistory(
                    Process,
                    -(LONG)sz,
                    "BuffDealloc ",
                    AfdBufferTag
                    );
                AfdRecordPoolQuotaReturned(
                    sz
                    );
                AfdBufferTag->Lookaside = TRUE;
            }
            ExFreeToNPagedLookasideList( &AfdLookasideLists->BufferTagList, AfdBufferTag );
        }
        else {
            LONG    sz;
            ASSERT (AfdBufferTag->AllocatedAddressLength>AfdStandardAddressLength);
            ASSERT (AfdBufferTag->Lookaside == FALSE);
            PsReturnPoolQuota (
                                Process,
                                NonPagedPool,
                                sz = sizeof (AFD_BUFFER_TAG) 
                                        + AfdBufferTag->AllocatedAddressLength);
            AfdRecordQuotaHistory(
                Process,
                -(LONG)sz,
                "BuffDealloc ",
                AfdBufferTag
                );
            AfdRecordPoolQuotaReturned(
                sz
                );
            AFD_FREE_POOL(
                AfdBufferTag,
                AFD_DATA_BUFFER_POOL_TAG
                );
        }
    }

} // AfdReturnBuffer





PAFD_BUFFER
AfdInitializeBuffer (
    IN PVOID MemoryBlock,
    IN ULONG BufferDataSize,
    IN ULONG AddressSize,
    IN CCHAR StackSize
    )

/*++

Routine Description:

    Initializes an AFD buffer.  Sets up fields in the actual AFD_BUFFER
    structure and initializes the IRP and MDL associated with the
    buffer.  This routine assumes that the caller has properly allocated
    sufficient space for all this.

Arguments:

    AfdBuffer - points to the AFD_BUFFER structure to initialize.

    BufferDataSize - the size of the data buffer that goes along with the
        buffer structure.

    AddressSize - the size of data allocated for the address buffer.

    ListHead - the global list this buffer belongs to, or NULL if it
        doesn't belong on any list.  This routine does NOT place the
        buffer structure on the list.

Return Value:

    None.

--*/

{
    USHORT  irpSize;
    SIZE_T  mdlSize;
    SIZE_T  hdrSize;
    PAFD_BUFFER hdr;
    PMDL    mdl;
    PIRP    irp;
    PVOID   buf;
    PVOID   addr;
    UCHAR   placement;
    SIZE_T  alignment;
#ifdef AFD_CHECK_ALIGNMENT
    PLONG  alignmentCounters = (PLONG)&AfdAlignmentTable[AfdAlignmentTableSize];
#endif
    irpSize = IoSizeOfIrp( StackSize );
    mdlSize = (ULONG)MmSizeOfMdl( NULL, BufferDataSize );
    hdrSize = sizeof (AFD_BUFFER);

    //
    // Compute the index into (mis)alignment table to determine
    // what placement of the buffer block elements (e.g. hdr, IRP, MDL,
    // and data buffer itself) we need to choose to compensate and
    // align data buffer on AfdBufferAlignment boundary.
    //
    ASSERT ((PtrToUlong(MemoryBlock)%AFD_MINIMUM_BUFFER_ALIGNMENT)==0);
#ifdef _AFD_VARIABLE_STACK_
    if (PAGE_ALIGN (MemoryBlock)==MemoryBlock || StackSize!=AfdTdiStackSize) {
        ASSERT (StackSize>=AfdTdiStackSize && StackSize<=AfdMaxStackSize);
#else // _AFD_VARIABLE_STACK
    if (PAGE_ALIGN (MemoryBlock)==MemoryBlock) {
#endif // _AFD_VARIABLE_STACK
        //
        // For page-aligned blocks (which are >= page size),
        // we always place the buffer first.
        // Same thing for big IRP stacks - allocate more memory to
        // compensate for alignment.
        //
        placement = AFD_PLACEMENT_BUFFER;
    }
    else {
        placement = AfdAlignmentTable[
                (PtrToUlong(MemoryBlock)&(AfdBufferAlignment-1))/AFD_MINIMUM_BUFFER_ALIGNMENT];
    }

#ifdef AFD_CHECK_ALIGNMENT
    InterlockedIncrement (&alignmentCounters[
            (PtrToUlong(MemoryBlock)&(AfdBufferAlignment-1))/AFD_MINIMUM_BUFFER_ALIGNMENT]);
#endif

    switch (placement) {
    case AFD_PLACEMENT_BUFFER:
        //
        // Perfect case: the memory is aready aligned as we need it.
        //
        buf = ALIGN_UP_A_POINTER(MemoryBlock, AfdBufferAlignment);
        alignment = (PUCHAR)buf-(PUCHAR)MemoryBlock;
#ifdef _AFD_VARIABLE_STACK_
        ASSERT (alignment<=AfdAlignmentOverhead || (StackSize!=AfdTdiStackSize));
#else // _AFD_VARIABLE_STACK_
        ASSERT (alignment<=AfdAlignmentOverhead);
#endif // _AFD_VARIABLE_STACK_
        hdr = ALIGN_UP_TO_TYPE_POINTER((PCHAR)buf+BufferDataSize, AFD_BUFFER);
        irp = ALIGN_UP_TO_TYPE_POINTER((PCHAR)hdr+hdrSize, IRP);
        mdl = ALIGN_UP_TO_TYPE_POINTER((PCHAR)irp+irpSize, MDL);
        addr = (PCHAR)mdl+mdlSize;
        break;

        //
        // Other cases, we use hdr, mdl, and IRP to try to compensate
        // and have the data buffer aligned at the AfdBufferAlignment
        // boundary.
        //
    case AFD_PLACEMENT_HDR:
        hdr = ALIGN_UP_TO_TYPE_POINTER(MemoryBlock, AFD_BUFFER);
        alignment = (PUCHAR)hdr-(PUCHAR)MemoryBlock;
        ASSERT (alignment<=AfdAlignmentOverhead);
        buf = ALIGN_UP_A_POINTER((PCHAR)hdr+hdrSize, AfdBufferAlignment);
        irp = ALIGN_UP_TO_TYPE_POINTER((PCHAR)buf+BufferDataSize, IRP);
        mdl = ALIGN_UP_TO_TYPE_POINTER((PCHAR)irp+irpSize, MDL);
        addr = (PCHAR)mdl+mdlSize;
        break;

    case AFD_PLACEMENT_MDL:
        mdl = ALIGN_UP_TO_TYPE_POINTER(MemoryBlock, MDL);
        alignment = (PUCHAR)mdl-(PUCHAR)MemoryBlock;
        ASSERT (alignment<=AfdAlignmentOverhead);
        buf = ALIGN_UP_A_POINTER((PCHAR)mdl+mdlSize, AfdBufferAlignment);
        hdr = ALIGN_UP_TO_TYPE_POINTER((PCHAR)buf+BufferDataSize, AFD_BUFFER);
        irp = ALIGN_UP_TO_TYPE_POINTER((PCHAR)hdr+hdrSize, IRP);
        addr = (PCHAR)irp+irpSize;
        break;
    case AFD_PLACEMENT_IRP:
        irp = ALIGN_UP_TO_TYPE_POINTER(MemoryBlock, IRP);
        alignment = (PUCHAR)irp-(PUCHAR)MemoryBlock;
        ASSERT (alignment<=AfdAlignmentOverhead);
        buf = ALIGN_UP_A_POINTER((PCHAR)irp+irpSize, AfdBufferAlignment);
        hdr = ALIGN_UP_TO_TYPE_POINTER((PCHAR)buf+BufferDataSize, AFD_BUFFER);
        mdl = ALIGN_UP_TO_TYPE_POINTER((PCHAR)hdr+hdrSize, MDL);
        addr = (PCHAR)mdl+mdlSize;
        break;
    case AFD_PLACEMENT_HDR_IRP:
        hdr = ALIGN_UP_TO_TYPE_POINTER(MemoryBlock, AFD_BUFFER);
        alignment = (PUCHAR)hdr-(PUCHAR)MemoryBlock;
        ASSERT (alignment<=AfdAlignmentOverhead);
        irp = ALIGN_UP_TO_TYPE_POINTER((PCHAR)hdr+hdrSize, IRP);
        buf = ALIGN_UP_A_POINTER((PCHAR)irp+irpSize, AfdBufferAlignment);
        mdl = ALIGN_UP_TO_TYPE_POINTER((PCHAR)buf+BufferDataSize, MDL);
        addr = (PCHAR)mdl+mdlSize;
        break;
    case AFD_PLACEMENT_HDR_MDL:
        hdr = ALIGN_UP_TO_TYPE_POINTER(MemoryBlock, AFD_BUFFER);
        alignment = (PUCHAR)hdr-(PUCHAR)MemoryBlock;
        ASSERT (alignment<=AfdAlignmentOverhead);
        mdl = ALIGN_UP_TO_TYPE_POINTER((PCHAR)hdr+hdrSize, MDL);
        buf = ALIGN_UP_A_POINTER((PCHAR)mdl+mdlSize, AfdBufferAlignment);
        irp = ALIGN_UP_TO_TYPE_POINTER((PCHAR)buf+BufferDataSize, IRP);
        addr = (PCHAR)irp+irpSize;
        break;
    case AFD_PLACEMENT_IRP_MDL:
        irp = ALIGN_UP_TO_TYPE_POINTER(MemoryBlock, IRP);
        alignment = (PUCHAR)irp-(PUCHAR)MemoryBlock;
        ASSERT (alignment<=AfdAlignmentOverhead);
        mdl = ALIGN_UP_TO_TYPE_POINTER((PCHAR)irp+irpSize, MDL);
        buf = ALIGN_UP_A_POINTER((PCHAR)mdl+mdlSize, AfdBufferAlignment);
        hdr = ALIGN_UP_TO_TYPE_POINTER((PCHAR)buf+BufferDataSize, AFD_BUFFER);
        addr = (PCHAR)hdr+hdrSize;
        break;
    case AFD_PLACEMENT_HDR_IRP_MDL:
        hdr = ALIGN_UP_TO_TYPE_POINTER(MemoryBlock, AFD_BUFFER);
        alignment = (PUCHAR)hdr-(PUCHAR)MemoryBlock;
        ASSERT (alignment<=AfdAlignmentOverhead);
        irp = ALIGN_UP_TO_TYPE_POINTER((PCHAR)hdr+hdrSize, IRP);
        mdl = ALIGN_UP_TO_TYPE_POINTER((PCHAR)irp+irpSize, MDL);
        buf = ALIGN_UP_A_POINTER((PCHAR)mdl+mdlSize, AfdBufferAlignment);
        addr = (PCHAR)buf+BufferDataSize;
        break;
    default:
        ASSERT (!"Unknown placement!");
        __assume (0);
    }

    
    //
    // Initialize the AfdBuffer - most fields need to be 0.
    //

    RtlZeroMemory( hdr, sizeof(*hdr) );

    //
    // Setup buffer
    //
    hdr->Buffer = buf;
    hdr->BufferLength = BufferDataSize;
    
    //
    // We just need to store first two bits of placement
    // so we know which part comes first to free it properly.
    //
    hdr->Placement = placement & 3;

    //
    // If we have to align the memory block to meet the requirement
    // store this information right below the first piece.
    //
    if (alignment!=0) {
        C_ASSERT (AFD_MINIMUM_BUFFER_ALIGNMENT>=sizeof (SIZE_T));
        C_ASSERT ((AFD_MINIMUM_BUFFER_ALIGNMENT & (sizeof(SIZE_T)-1))==0);
        ASSERT (alignment>=sizeof (SIZE_T));
        hdr->AlignmentAdjusted = TRUE;
        *(((PSIZE_T)(((PUCHAR)MemoryBlock)+alignment))-1) = alignment;
    }


    //
    // Initialize the IRP pointer.
    //

    hdr->Irp = irp;
    IoInitializeIrp( hdr->Irp, irpSize, StackSize );
    hdr->Irp->MdlAddress = mdl;

    //
    // Set up the MDL pointer.
    //

    hdr->Mdl = mdl;
    MmInitializeMdl( hdr->Mdl, buf, BufferDataSize );
    MmBuildMdlForNonPagedPool( hdr->Mdl );
    
    //
    // Set up the address buffer pointer.
    //

    if (AddressSize>0) {
        hdr->TdiInfo.RemoteAddress = ALIGN_UP_TO_TYPE_POINTER(addr, TRANSPORT_ADDRESS);;
        hdr->AllocatedAddressLength = (USHORT)AddressSize;
    }


#if DBG
    hdr->BufferListEntry.Flink = UIntToPtr( 0xE0E1E2E3 );
    hdr->BufferListEntry.Blink = UIntToPtr( 0xE4E5E6E7 );
#endif

    return hdr;

} // AfdInitializeBuffer


VOID
AfdInitializeBufferTag (
    IN PAFD_BUFFER_TAG AfdBufferTag,
    IN CLONG           AddressSize
    )

/*++

Routine Description:

    Initializes an AFD buffer.  Sets up fields in the actual AFD_BUFFER
    structure and initializes the IRP and MDL associated with the
    buffer.  This routine assumes that the caller has properly allocated
    sufficient space for all this.

Arguments:

    AfdBuffer - points to the AFD_BUFFER structure to initialize.

    BufferDataSize - the size of the data buffer that goes along with the
        buffer structure.

    AddressSize - the size of data allocated for the address buffer.

    ListHead - the global list this buffer belongs to, or NULL if it
        doesn't belong on any list.  This routine does NOT place the
        buffer structure on the list.

Return Value:

    None.

--*/

{
    AfdBufferTag->Mdl = NULL;
    AfdBufferTag->BufferLength = AfdBufferTagSize;
    AfdBufferTag->TdiInfo.RemoteAddress = AddressSize ? AfdBufferTag+1 : NULL;
    AfdBufferTag->AllocatedAddressLength = (USHORT)AddressSize;
    AfdBufferTag->Flags = 0;

#if DBG
    AfdBufferTag->BufferListEntry.Flink = UIntToPtr( 0xE0E1E2E3 );
    AfdBufferTag->BufferListEntry.Blink = UIntToPtr( 0xE4E5E6E7 );
    AfdBufferTag->Caller = NULL;
    AfdBufferTag->CallersCaller = NULL;
#endif
}


VOID
AfdInitializeBufferManager (
    VOID
    )
{
    SIZE_T  irpSize = ALIGN_UP_A(IoSizeOfIrp (AfdTdiStackSize), AFD_MINIMUM_BUFFER_ALIGNMENT);
    SIZE_T  hdrSize = ALIGN_UP_A(sizeof (AFD_BUFFER), AFD_MINIMUM_BUFFER_ALIGNMENT);
    SIZE_T  mdlSize = ALIGN_UP_A(MmSizeOfMdl (NULL, PAGE_SIZE),AFD_MINIMUM_BUFFER_ALIGNMENT);
    UCHAR   placement;
    ULONG   i;
    ULONG   currentOverhead;

    //
    // Initialize the alignment table.
    // This table is used to determine what kind of element
    // placement to use in AFD_BUFFER depending on the alignment
    // of the memory block returned by the executive pool manager.
    // The goal is to align the data buffer on the cache line
    // boundary.  However, since executive only guarantees alignment of
    // it's blocks at the CPU alignment requirements, we need to 
    // adjust and potentially waste up to CACHE_LIST_SIZE-CPU_ALIGNMENT_SIZE.
    // With some machines having cache line alignment at 128 such memory
    // waste is prohibitive (small buffers with default size of 128 will double
    // in size).
    // The table below allows us to rearrange pieces in AFD_BUFFER structure,
    // namely, the header, IRP, MDL, and data buffer, so that pieces with
    // lower alignment requirement can be used to consume the space needed
    // to adjust the memory block to the cache line boundary.
    
    //
    // AfdAlignmentTable has an entry for each possible case of memory block
    // misaligned against cache line size.  For example, in typical X86 system
    // case executive pool manager always returns blocks aligned on 8 byte bounday,
    // while cache lines are typically 64 bytes long, so memory manager can 
    // theoretically return blocks misaligned against cache by:
    //        8, 16, 24, 32, 40, 48, 56.
    // For each of these cases we will try to adjust the alignment by using
    // any possible combination of header, IRP, and MDL.  There will be some
    // cases that cannot be adjusted exactly, and we will have to pad.
    //

    //
    // First initialize the table assuming the data buffer is placed first.
    //
    RtlFillMemory (AfdAlignmentTable, AfdAlignmentTableSize, AFD_PLACEMENT_BUFFER);
#ifdef AFD_CHECK_ALIGNMENT
    RtlZeroMemory (&AfdAlignmentTable[AfdAlignmentTableSize],
                        AfdAlignmentTableSize*sizeof(LONG));
#endif
    //
    // Now identify the entries that can be padded with some combination of
    // header, IRP, and MDL:
    //      extract the bits that can be used for padding
    //      reverse to get corresponding memory block alignments
    //      divide by the step of the alignment table
    //      make sure we won't go past table size (last entry => 0 entry).
    //
#define AfdInitAlignmentTableRow(_size,_plcmnt)                     \
    AfdAlignmentTable[                                              \
            ((AfdBufferAlignment-(_size&(AfdBufferAlignment-1)))    \
                /AFD_MINIMUM_BUFFER_ALIGNMENT)                      \
                &(AfdAlignmentTableSize-1)] = _plcmnt

    //
    // We let placements beginning with header override others,
    // since it is more natural and easier to debug (header has references
    // to other pieces).
    //

    AfdInitAlignmentTableRow(mdlSize,AFD_PLACEMENT_MDL);
    AfdInitAlignmentTableRow(irpSize,AFD_PLACEMENT_IRP);
    AfdInitAlignmentTableRow((irpSize+mdlSize),AFD_PLACEMENT_IRP_MDL);
    AfdInitAlignmentTableRow((hdrSize+mdlSize),AFD_PLACEMENT_HDR_MDL);
    AfdInitAlignmentTableRow((hdrSize+irpSize),AFD_PLACEMENT_HDR_IRP);
    AfdInitAlignmentTableRow((hdrSize+irpSize+mdlSize),AFD_PLACEMENT_HDR_IRP_MDL);
    AfdInitAlignmentTableRow(hdrSize,AFD_PLACEMENT_HDR);

    //
    // Now scan the table from top to bottom and fill entries that do not have
    // exact match using the above combinations. Use the closest entry above and
    // in the process compute how much do we need to pad in addition to padding
    // achieved via placement.
    //
    AfdAlignmentOverhead = 0;
    currentOverhead = 0;
    //
    // By default use the placement of aligned block.
    //
    placement = AfdAlignmentTable[0];
    for (i=AfdAlignmentTableSize-1; i>0; i--) {
        if (AfdAlignmentTable[i]==AFD_PLACEMENT_BUFFER) {
            AfdAlignmentTable[i] = placement;
            currentOverhead += AFD_MINIMUM_BUFFER_ALIGNMENT;
        }
        else {
            placement = AfdAlignmentTable[i];
            if (AfdAlignmentOverhead<currentOverhead) {
                AfdAlignmentOverhead = currentOverhead;
            }
            currentOverhead = 0;
        }
    }
    if (AfdAlignmentOverhead<currentOverhead) {
        AfdAlignmentOverhead = currentOverhead;
    }


    KdPrintEx(( DPFLTR_WSOCKTRANSPORT_ID, DPFLTR_INFO_LEVEL,
                "AfdInitializeBufferManager: Alignment requirements: MM-%d, cache-%d, overhead-%d\n",
                        AFD_MINIMUM_BUFFER_ALIGNMENT,
                        AfdBufferAlignment,
                        AfdAlignmentOverhead));
    {
        CLONG   oldBufferLengthForOnePage = AfdBufferLengthForOnePage;

        AfdBufferOverhead = AfdCalculateBufferSize(
                                        PAGE_SIZE,
                                        AfdStandardAddressLength,
                                        AfdTdiStackSize) - PAGE_SIZE;
        AfdBufferLengthForOnePage = ALIGN_DOWN_A(
                                        PAGE_SIZE-AfdBufferOverhead,
                                        AFD_MINIMUM_BUFFER_ALIGNMENT);
        if (AfdLargeBufferSize==oldBufferLengthForOnePage) {
            AfdLargeBufferSize = AfdBufferLengthForOnePage;
        }
    }
}