windows-server-2003/net/tcpip/driver/common/nt/mdlpool.c

/*++

Copyright (c) 1999-2000  Microsoft Corporation

Module Name:

    mdlpool.c

Abstract:

    This file contains the implementation of an MDL buffer pool.

Author:

    Shaun Cox (shaunco) 21-Oct-1999

--*/

#include "ntddk.h"
#include "mdlpool.h"

typedef PVOID LPVOID;
#include "align.h"              // Macros: ROUND_UP_POINTER, POINTER_IS_ALIGNED

#define SHOW_DEBUG_OUTPUT 0

#define SCAVENGE_PERIOD_IN_SECONDS          30
#define MINIMUM_PAGE_LIFETIME_IN_SECONDS    20
#define USED_PAGES_SCAVENGE_THRESHOLD       64

#if defined (_WIN64)
#define MAX_CACHE_LINE_SIZE 128
#define BLOCK_TYPE  SLIST_HEADER
#else
#define MAX_CACHE_LINE_SIZE 64
#define BLOCK_TYPE  PVOID
#endif

// The following structures are used in the single allocation that
// a pool handle points to.
//      PoolHandle ---> [POOL_HEADER + CPU_POOL_HEADER for cpu 0 +
//                                     CPU_POOL_HEADER for cpu 1 + ...
//                                     CPU_POOL_HEADER for cpu N]
//

// POOL_HEADER is the data common to all CPUs for a given pool.
//
typedef struct _POOL_HEADER
{
// cache-line -----
    struct _POOL_HEADER_BASE
    {
        ULONG Tag;
        USHORT BufferSize;
        USHORT MdlsPerPage;
        PVOID Allocation;
    };
    UCHAR Alignment[MAX_CACHE_LINE_SIZE
            - (sizeof(struct _POOL_HEADER_BASE) % MAX_CACHE_LINE_SIZE)];
} POOL_HEADER, *PPOOL_HEADER;

C_ASSERT(sizeof(POOL_HEADER) % MAX_CACHE_LINE_SIZE == 0);


// CPU_POOL_HEADER is the data specific to a CPU for a given pool.
//
typedef struct _CPU_POOL_HEADER
{
// cache-line -----
    struct _CPU_POOL_HEADER_BASE
    {
        // The doubly-linked list of pages that make up this processor's pool.
        // These pages have one or more free MDLs available.
        //
        LIST_ENTRY PageList;

        // The doubly-linked list of pages that are fully in use.  This list
        // is separate from the above list so that we do not spend time walking
        // a very long list during MdpAllocate when many pages are fully used.
        //
        LIST_ENTRY UsedPageList;

        // The next scheduled time (in units of KeQueryTickCount()) for
        // scavenging this pool.  The next scavenge will happen no earlier
        // that this.
        //
        LARGE_INTEGER NextScavengeTick;

        // Count of pages on the used page list.
        // If this becomes greater than USED_PAGES_SCAVENGE_THRESHOLD
        // and we know we missed a page move during a prior MdpFree,
        // we will scavenge during the next MdpAllocate.
        //
        USHORT PagesOnUsedPageList;

        // Set to TRUE during MdpFree if could not move a previously used
        // page back to the normal list because the free was done by a
        // non-owning processor.  Set to FALSE during MdpScavenge.
        //
        BOOLEAN MissedPageMove;

        // The number of the processor that owns this pool.
        //
        UCHAR OwnerCpu;

        ULONG TotalMdlsAllocated;
        ULONG TotalMdlsFreed;
        ULONG PeakMdlsInUse;
        ULONG TotalPagesAllocated;
        ULONG TotalPagesFreed;
        ULONG PeakPagesInUse;
    };
    UCHAR Alignment[MAX_CACHE_LINE_SIZE
            - (sizeof(struct _CPU_POOL_HEADER_BASE) % MAX_CACHE_LINE_SIZE)];
} CPU_POOL_HEADER, *PCPU_POOL_HEADER;

C_ASSERT(sizeof(CPU_POOL_HEADER) % MAX_CACHE_LINE_SIZE == 0);


// PAGE_HEADER is the data at the beginning of each allocated pool page
// that describes the current state of the MDLs on the page.
//
typedef struct _PAGE_HEADER
{
// cache-line -----
    // Back pointer to the owning cpu pool.
    //
    PCPU_POOL_HEADER Pool;

    // Linkage entry for the list of pages managed by the cpu pool.
    //
    LIST_ENTRY PageLink;

    // Number of MDLs built so far on this page.  MDLs are built on
    // demand.  When this number reaches Pool->MdlsPerPage, all MDLs on this
    // page have been built.
    //
    USHORT MdlsBuilt;

    // Boolean indicator of whether or not this page is on the cpu pool's
    // used-page list.  This is checked during MdpFree to see if the page
    // should be moved back to the normal page list.
    // (it is a USHORT, instead of BOOLEAN, for proper padding)
    //
    USHORT OnUsedPageList;

    // List of free MDLs on this page.
    //
    SLIST_HEADER FreeList;

    // The value of KeQueryTickCount (normalized to units of seconds)
    // which represents the time after which this page can be freed back
    // to the system's pool.  This time is only used once the depth of
    // FreeList is Pool->MdlsPerPage.  (i.e. this time is only used if
    // the page is completely unused.)
    //
    LARGE_INTEGER LastUsedTick;

} PAGE_HEADER, *PPAGE_HEADER;


// MDLs that we build are always limited to one page and they never
// describe buffers that span a page boundry.
//
#define MDLSIZE sizeof(MDL) + sizeof(PFN_NUMBER)


// Get a pointer to the overall pool given a pointer to one of
// the per-processor pools within it.
//
__inline
PPOOL_HEADER
PoolFromCpuPool(
    IN PCPU_POOL_HEADER CpuPool
    )
{
    return (PPOOL_HEADER)(CpuPool - CpuPool->OwnerCpu) - 1;
}

__inline
VOID
ConvertSecondsToTicks(
    IN ULONG Seconds,
    OUT PLARGE_INTEGER Ticks
    )
{
    // If the following assert fires, you need to cast Seconds below to
    // ULONGLONG so that 64 bit multiplication and division are used.
    // The current code assumes less that 430 seconds so that the
    // 32 multiplication below won't overflow.
    //
    ASSERT(Seconds < 430);

    Ticks->HighPart = 0;
    Ticks->LowPart = (Seconds * 10*1000*1000) / KeQueryTimeIncrement();
}

// Build the next MDL on the specified pool page.
// This can only be called if not all of the MDLs have been built yet.
//
PMDL
MdppBuildNextMdl(
    IN const POOL_HEADER* Pool,
    IN OUT PPAGE_HEADER Page
    )
{
    PMDL Mdl;
    ULONG BlockSize = ALIGN_UP(MDLSIZE + Pool->BufferSize, BLOCK_TYPE);

    ASSERT(Page->MdlsBuilt < Pool->MdlsPerPage);
    ASSERT((PAGE_SIZE - sizeof(PAGE_HEADER)) / BlockSize == Pool->MdlsPerPage);

    Mdl = (PMDL)((PCHAR)(Page + 1) + (Page->MdlsBuilt * BlockSize));
    ASSERT(PAGE_ALIGN(Mdl) == Page);

    MmInitializeMdl(Mdl, (PCHAR)Mdl + MDLSIZE, Pool->BufferSize);
    MmBuildMdlForNonPagedPool(Mdl);

    ASSERT(MDLSIZE == Mdl->Size);
    ASSERT(MmGetMdlBaseVa(Mdl) == Page);
    ASSERT(MmGetMdlByteCount(Mdl) == Pool->BufferSize);

    Page->MdlsBuilt++;

    return Mdl;
}

// Allocate a new pool page and insert it at the head of the specified
// CPU pool.  Build the first MDL on the new page and return a pointer
// to it.
//
PMDL
MdppAllocateNewPageAndBuildOneMdl(
    IN const POOL_HEADER* Pool,
    IN PCPU_POOL_HEADER CpuPool
    )
{
    PPAGE_HEADER Page;
    PMDL Mdl = NULL;
    ULONG PagesInUse;

    ASSERT(Pool);

    Page = ExAllocatePoolWithTagPriority(NonPagedPool, PAGE_SIZE, Pool->Tag,
                                         NormalPoolPriority);
    if (Page)
    {
        ASSERT(Page == PAGE_ALIGN(Page));

        RtlZeroMemory(Page, sizeof(PAGE_HEADER));
        Page->Pool = CpuPool;
        ExInitializeSListHead(&Page->FreeList);

        // Insert the page at the head of the cpu's pool.
        //
        InsertHeadList(&CpuPool->PageList, &Page->PageLink);
        CpuPool->TotalPagesAllocated++;

        // Update the pool's statistics.
        //
        PagesInUse = CpuPool->TotalPagesAllocated - CpuPool->TotalPagesFreed;
        if (PagesInUse > CpuPool->PeakPagesInUse)
        {
            CpuPool->PeakPagesInUse = PagesInUse;
        }

        Mdl = MdppBuildNextMdl(Pool, Page);
        ASSERT(Mdl);

#if SHOW_DEBUG_OUTPUT
        DbgPrint(
            "[%d] %c%c%c%c page allocated : Pages(a%4d,u%4d,p%4d), Mdls(a%6d,u%6d,p%6d)\n",
            CpuPool->OwnerCpu,
            Pool->Tag, Pool->Tag >> 8, Pool->Tag >> 16, Pool->Tag >> 24,
            CpuPool->TotalPagesAllocated,
            CpuPool->TotalPagesAllocated - CpuPool->TotalPagesFreed,
            CpuPool->PeakPagesInUse,
            CpuPool->TotalMdlsAllocated,
            CpuPool->TotalMdlsAllocated - CpuPool->TotalMdlsFreed,
            CpuPool->PeakMdlsInUse);
#endif
    }

    return Mdl;
}

// Free the specified pool page back to the system's pool.
//
VOID
MdppFreePage(
    IN PCPU_POOL_HEADER CpuPool,
    IN PPAGE_HEADER Page
    )
{
#if SHOW_DEBUG_OUTPUT
    ULONG Tag;
#endif

    ASSERT(Page == PAGE_ALIGN(Page));
    ASSERT(Page->Pool == CpuPool);

    ExFreePool (Page);
    CpuPool->TotalPagesFreed++;

    ASSERT(CpuPool->TotalPagesFreed <= CpuPool->TotalPagesAllocated);

#if SHOW_DEBUG_OUTPUT
    Tag = PoolFromCpuPool(CpuPool)->Tag;

    DbgPrint(
        "[%d] %c%c%c%c page freed     : Pages(a%4d,u%4d,p%4d), Mdls(a%6d,u%6d,p%6d)\n",
        CpuPool->OwnerCpu,
        Tag, Tag >> 8, Tag >> 16, Tag >> 24,
        CpuPool->TotalPagesAllocated,
        CpuPool->TotalPagesAllocated - CpuPool->TotalPagesFreed,
        CpuPool->PeakPagesInUse,
        CpuPool->TotalMdlsAllocated,
        CpuPool->TotalMdlsAllocated - CpuPool->TotalMdlsFreed,
        CpuPool->PeakMdlsInUse);
#endif
}


// Free the specified pool page list back to the system's pool.
//
VOID
MdppFreeList(
    IN PCPU_POOL_HEADER CpuPool,
    IN PLIST_ENTRY Head
    )
{
    PPOOL_HEADER Pool;
    PPAGE_HEADER Page;
    PLIST_ENTRY Scan;
    PLIST_ENTRY Next;
    BOOLEAN UsedPageList;

    Pool = PoolFromCpuPool(CpuPool);
    UsedPageList = (Head == &CpuPool->UsedPageList);

    for (Scan = Head->Flink; Scan != Head; Scan = Next)
    {
        Page = CONTAINING_RECORD(Scan, PAGE_HEADER, PageLink);
        ASSERT(Page == PAGE_ALIGN(Page));
        ASSERT(CpuPool == Page->Pool);
        ASSERT(UsedPageList ? Page->OnUsedPageList : !Page->OnUsedPageList);
        
        ASSERT(Page->MdlsBuilt <= Pool->MdlsPerPage);
        ASSERT(Page->MdlsBuilt == ExQueryDepthSList(&Page->FreeList));
        
        // Step to the next link before we free this page.
        //
        Next = Scan->Flink;
        
        RemoveEntryList(Scan);
        MdppFreePage(CpuPool, Page);
    }
}


// Reclaim the memory consumed by completely unused pool pages belonging
// to the specified per-processor pool.
//
// Caller IRQL: [DISPATCH_LEVEL]
//
VOID
MdppScavengePool(
    IN OUT PCPU_POOL_HEADER CpuPool
    )
{
    PPOOL_HEADER Pool;
    PPAGE_HEADER Page;
    PLIST_ENTRY Scan;
    PLIST_ENTRY Next;
    LARGE_INTEGER Ticks;
    LARGE_INTEGER TicksDelta;

    // We must not only be at DISPATCH_LEVEL (or higher), we must also
    // be called on the processor that owns the specified pool.
    //
    ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);
    ASSERT(KeGetCurrentProcessorNumber() == CpuPool->OwnerCpu);

    Pool = PoolFromCpuPool(CpuPool);

    KeQueryTickCount(&Ticks);

    // Compute the next tick value which represents the earliest time
    // that we will scavenge this pool again.
    //
    ConvertSecondsToTicks(SCAVENGE_PERIOD_IN_SECONDS, &TicksDelta);
    CpuPool->NextScavengeTick.QuadPart = Ticks.QuadPart + TicksDelta.QuadPart;

    // Compute the tick value which represents the last point at which
    // its okay to free a page.
    //
    ConvertSecondsToTicks(MINIMUM_PAGE_LIFETIME_IN_SECONDS, &TicksDelta);
    Ticks.QuadPart = Ticks.QuadPart - TicksDelta.QuadPart;

    for (Scan = CpuPool->PageList.Flink;
         Scan != &CpuPool->PageList;
         Scan = Next)
    {
        Page = CONTAINING_RECORD(Scan, PAGE_HEADER, PageLink);
        ASSERT(Page == PAGE_ALIGN(Page));
        ASSERT(CpuPool == Page->Pool);
        ASSERT(!Page->OnUsedPageList);

        // Step to the next link before we possibly unlink this page.
        //
        Next = Scan->Flink;

        if ((Pool->MdlsPerPage == ExQueryDepthSList(&Page->FreeList)) &&
            (Ticks.QuadPart > Page->LastUsedTick.QuadPart))
        {
            RemoveEntryList(Scan);

            MdppFreePage(CpuPool, Page);
        }
    }

    // Scan the used pages to see if they can be moved back to the normal
    // list.  This can happen if too many frees by non-owning processors
    // are done.  In that case, the pages get orphaned on the used-page
    // list after all of their MDLs have been freed to the page.  Un-orphan
    // them here.
    //
    for (Scan = CpuPool->UsedPageList.Flink;
         Scan != &CpuPool->UsedPageList;
         Scan = Next)
    {
        Page = CONTAINING_RECORD(Scan, PAGE_HEADER, PageLink);
        ASSERT(Page == PAGE_ALIGN(Page));
        ASSERT(CpuPool == Page->Pool);
        ASSERT(Page->OnUsedPageList);

        // Step to the next link before we possibly unlink this page.
        //
        Next = Scan->Flink;

        if (0 != ExQueryDepthSList(&Page->FreeList))
        {
            RemoveEntryList(Scan);
            Page->OnUsedPageList = FALSE;
            InsertTailList(&CpuPool->PageList, Scan);
            CpuPool->PagesOnUsedPageList--;

#if SHOW_DEBUG_OUTPUT
            DbgPrint(
                "[%d] %c%c%c%c page moved off of used-page list during scavenge\n",
                CpuPool->OwnerCpu,
                Pool->Tag, Pool->Tag >> 8, Pool->Tag >> 16, Pool->Tag >> 24);
#endif
        }
    }

    // Reset our indicator of a missed page move now that we've scavenged.
    //
    CpuPool->MissedPageMove = FALSE;
}


// Creates a pool of MDLs built over non-paged pool.  Each MDL describes
// a buffer that is BufferSize bytes long.  If NULL is not returned,
// MdpDestroyPool should be called at a later time to reclaim the
// resources used by the pool.
//
// Arguments:
//  BufferSize - The size, in bytes, of the buffer that each MDL
//      should describe.
//  Tag - The pool tag to be used internally for calls to
//      ExAllocatePoolWithTag.  This allows callers to track
//      memory consumption for different pools.
//
//  Returns the handle used to identify the pool.
//
// Caller IRQL: [PASSIVE_LEVEL, DISPATCH_LEVEL]
//
HANDLE
MdpCreatePool(
    IN USHORT BufferSize,
    IN ULONG Tag
    )
{
    SIZE_T Size;
    PVOID Allocation;
    PPOOL_HEADER Pool = NULL;
    PCPU_POOL_HEADER CpuPool;
    USHORT BlockSize;
    CCHAR NumberCpus = KeNumberProcessors;
    CCHAR i;

    ASSERT(BufferSize);

    // Compute the size of our pool header allocation.
    // Add padding to ensure that the pool header can begin on a cache line.
    //
    Size = sizeof(POOL_HEADER) + (sizeof(CPU_POOL_HEADER) * NumberCpus) +
        (MAX_CACHE_LINE_SIZE - MEMORY_ALLOCATION_ALIGNMENT);

    // Allocate the pool header.
    //
    Allocation = ExAllocatePoolWithTag(NonPagedPool, Size, ' pdM');
    if (Allocation)
    {
        ASSERT(POINTER_IS_ALIGNED(Allocation, MEMORY_ALLOCATION_ALIGNMENT));
        RtlZeroMemory(Allocation, Size);
        
        Pool = ROUND_UP_POINTER(Allocation, MAX_CACHE_LINE_SIZE);
        
        BlockSize = (USHORT)ALIGN_UP(MDLSIZE + BufferSize, BLOCK_TYPE);

        // Initialize the pool header fields.
        //
        Pool->Tag = Tag;
        Pool->BufferSize = BufferSize;
        Pool->MdlsPerPage = (PAGE_SIZE - sizeof(PAGE_HEADER)) / BlockSize;
        Pool->Allocation = Allocation;
        
        // Initialize the per-cpu pool headers.
        //
        CpuPool = (PCPU_POOL_HEADER)(Pool + 1);

        for (i = 0; i < NumberCpus; i++)
        {
            InitializeListHead(&CpuPool[i].PageList);
            InitializeListHead(&CpuPool[i].UsedPageList);
            CpuPool[i].OwnerCpu = i;
        }
    }

    return Pool;
}

// Destroys a pool of MDLs previously created by a call to MdpCreatePool.
//
// Arguments:
//  Pool - Handle which identifies the pool being destroyed.
//
// Caller IRQL: [PASSIVE_LEVEL, DISPATCH_LEVEL]
//
VOID
MdpDestroyPool(
    IN HANDLE PoolHandle
    )
{
    PPOOL_HEADER Pool;
    PCPU_POOL_HEADER CpuPool;
    CCHAR NumberCpus = KeNumberProcessors;
    CCHAR i;

    ASSERT(PoolHandle);

    Pool = (PPOOL_HEADER)PoolHandle;
    if (!Pool)
    {
        return;
    }

    for (i = 0, CpuPool = (PCPU_POOL_HEADER)(Pool + 1);
         i < NumberCpus;
         i++, CpuPool++)
    {
        ASSERT(CpuPool->OwnerCpu == (ULONG)i);

        MdppFreeList(CpuPool, &CpuPool->PageList);
        MdppFreeList(CpuPool, &CpuPool->UsedPageList);

        ASSERT(CpuPool->TotalPagesAllocated == CpuPool->TotalPagesFreed);
        ASSERT(CpuPool->TotalMdlsAllocated == CpuPool->TotalMdlsFreed);
    }

    ASSERT(Pool == ROUND_UP_POINTER(Pool->Allocation, MAX_CACHE_LINE_SIZE));
    ExFreePool(Pool->Allocation);
}


// Returns an MDL allocated from a pool.  NULL is returned if the
// request could not be granted.
//
// Arguments:
//  PoolHandle - Handle which identifies the pool being allocated from.
//  Buffer - Address to receive the pointer to the underlying mapped buffer
//      described by the MDL.
//
// Caller IRQL: [PASSIVE_LEVEL, DISPATCH_LEVEL]
//
PMDL
MdpAllocate(
    IN HANDLE PoolHandle,
    OUT PVOID* Buffer
    )
{
    KIRQL OldIrql;
    PMDL Mdl;

    OldIrql = KeRaiseIrqlToDpcLevel();

    Mdl = MdpAllocateAtDpcLevel(PoolHandle, Buffer);

    KeLowerIrql(OldIrql);

    return Mdl;
}

// Returns an MDL allocated from a pool.  NULL is returned if the
// request could not be granted.
//
// Arguments:
//  PoolHandle - Handle which identifies the pool being allocated from.
//  Buffer - Address to receive the pointer to the underlying mapped buffer
//      described by the MDL.
//
// Caller IRQL: [DISPATCH_LEVEL]
//
PMDL
MdpAllocateAtDpcLevel(
    IN HANDLE PoolHandle,
    OUT PVOID* Buffer
    )
{
    PPOOL_HEADER Pool;
    PCPU_POOL_HEADER CpuPool;
    PPAGE_HEADER Page;
    PSLIST_ENTRY MdlLink;
    PMDL Mdl = NULL;
    ULONG Cpu;
    LARGE_INTEGER Ticks;

#if DBG
    ASSERT(PoolHandle);
    ASSERT(Buffer);
    ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);
#endif
    *Buffer = NULL;

    Pool = (PPOOL_HEADER)PoolHandle;
    Cpu = KeGetCurrentProcessorNumber();
    CpuPool = (PCPU_POOL_HEADER)(Pool + 1) + Cpu;

    // If we know we've had frees by non-owning processors and there
    // are more than USED_PAGES_SCAVENGE_THRESHOLD pages on the used
    // page list, it is time to scavenge.  This is common in situations
    // where the buffer size is very large causing there to be just a few
    // MDLs per page.  Pages get used up quickly and if non-owning frees
    // are prevalent, the used page list can get very big even in
    // the normal scavenge period.
    //
    if (CpuPool->MissedPageMove &&
        (CpuPool->PagesOnUsedPageList > USED_PAGES_SCAVENGE_THRESHOLD))
    {
#if SHOW_DEBUG_OUTPUT
        DbgPrint(
            "[%d] %c%c%c%c Scavenging because of excessive used pages.\n",
            CpuPool->OwnerCpu,
            Pool->Tag, Pool->Tag >> 8, Pool->Tag >> 16, Pool->Tag >> 24);
#endif
        MdppScavengePool(CpuPool);
    }
    else
    {
        // See if the minimum time has passed since we last scavenged
        // the pool.  If it has, we'll scavenge again.  Normally, scavenging
        // should only be performed when we free.  However, for the case when
        // the caller constantly frees on a non-owning processor, we'll
        // take this chance to do the scavenging.
        //
        KeQueryTickCount(&Ticks);
        if (Ticks.QuadPart > CpuPool->NextScavengeTick.QuadPart)
        {
            MdppScavengePool(CpuPool);
        }
    }

    if (!IsListEmpty(&CpuPool->PageList))
    {
        Page = CONTAINING_RECORD(CpuPool->PageList.Flink, PAGE_HEADER, PageLink);
        ASSERT(Page == PAGE_ALIGN(Page));
        ASSERT(CpuPool == Page->Pool);
        ASSERT(!Page->OnUsedPageList);

        MdlLink = InterlockedPopEntrySList(&Page->FreeList);
        if (MdlLink)
        {
            Mdl = CONTAINING_RECORD(MdlLink, MDL, Next);
        }
        else
        {
            // If there were no MDLs on this page's free list, it had better
            // mean we haven't yet built all of the MDLs on the page.
            // (Otherwise, what is a fully used page doing on the page list
            // and not on the used-page list?)
            //
            ASSERT(Page->MdlsBuilt < Pool->MdlsPerPage);

            Mdl = MdppBuildNextMdl(Pool, Page);
            ASSERT(Mdl);
        }

        if ((Page != PAGE_ALIGN(Page)) || (CpuPool != Page->Pool) ||
            Page->OnUsedPageList || (PAGE_ALIGN(Mdl) != Page))
        {
            KeBugCheckEx(BAD_POOL_CALLER, 2, (ULONG_PTR)Mdl,
                (ULONG_PTR)Page, (ULONG_PTR)CpuPool);
        }

        // Got an MDL.  Now check to see if it was the last one on a fully
        // built page.  If so, move the page to the used-page list.
        //
        if ((0 == ExQueryDepthSList(&Page->FreeList)) &&
            (Page->MdlsBuilt == Pool->MdlsPerPage))
        {
            PLIST_ENTRY PageLink;
            PageLink = RemoveHeadList(&CpuPool->PageList);
            InsertTailList(&CpuPool->UsedPageList, PageLink);
            Page->OnUsedPageList = TRUE;
            CpuPool->PagesOnUsedPageList++;

            ASSERT(Page == CONTAINING_RECORD(PageLink, PAGE_HEADER, PageLink));

#if SHOW_DEBUG_OUTPUT
            DbgPrint(
                "[%d] %c%c%c%c page moved to used-page list\n",
                CpuPool->OwnerCpu,
                Pool->Tag, Pool->Tag >> 8, Pool->Tag >> 16, Pool->Tag >> 24);
#endif
        }

        ASSERT(Mdl);
        goto GotAnMdl;
    }
    else
    {
        // The page list is empty so we have to allocate and add a new page.
        //
        Mdl = MdppAllocateNewPageAndBuildOneMdl(Pool, CpuPool);
    }

    // If we are returning an MDL, update the statistics.
    //
    if (Mdl)
    {
        ULONG MdlsInUse;
GotAnMdl:

        CpuPool->TotalMdlsAllocated++;

        MdlsInUse = CpuPool->TotalMdlsAllocated - CpuPool->TotalMdlsFreed;
        if (MdlsInUse > CpuPool->PeakMdlsInUse)
        {
            CpuPool->PeakMdlsInUse = MdlsInUse;
        }

        // Don't give anyone ideas about where this might point.  I don't
        // want anyone trashing my pool because they thought this field
        // was valid for some reason.
        //
        Mdl->Next = NULL;

        // Reset the length of the buffer described by the MDL.  This is
        // a convienence to callers who sometimes adjust this length while
        // using the MDL, but who expect it to be reset on subsequent MDL
        // allocations.
        //
        Mdl->ByteCount = Pool->BufferSize;

        ASSERT(Mdl->MdlFlags & MDL_SOURCE_IS_NONPAGED_POOL);
        *Buffer = Mdl->MappedSystemVa;
    }

    return Mdl;
}

// Free an MDL to the pool from which it was allocated.
//
// Arguments:
//  Mdl - An Mdl returned from a prior call to MdpAllocate.
//
// Caller IRQL: [PASSIVE_LEVEL, DISPATCH_LEVEL]
//
VOID
MdpFree(
    IN PMDL Mdl
    )
{
    PPAGE_HEADER Page;
    PCPU_POOL_HEADER CpuPool;
    PPOOL_HEADER Pool;
    LARGE_INTEGER Ticks;
    LOGICAL PageIsOnUsedPageList;
    LOGICAL Scavenge = FALSE;

    ASSERT(Mdl);

    // Get the address of the page that this MDL maps.  This is where
    // our page header is stored.
    //
    Page = PAGE_ALIGN(Mdl);

    // Follow the back pointer in the page header to locate the owning
    // cpu's pool.
    //
    CpuPool = Page->Pool;

    // Locate the pool header.
    //
    Pool = PoolFromCpuPool(CpuPool);

//#if DBG
    // If someone changed the MDL to point to there own buffer,
    // or otherwise corrupted it, we'll stop here and let them know.
    //
    if ((MmGetMdlBaseVa(Mdl) != Page) ||
        (MDLSIZE != Mdl->Size) ||
        ((ULONG_PTR)Mdl->MappedSystemVa != (ULONG_PTR)Mdl + MDLSIZE) ||
        (MmGetMdlVirtualAddress(Mdl) != Mdl->MappedSystemVa))
    {
        KeBugCheckEx(BAD_POOL_CALLER, 3, (ULONG_PTR)Mdl,
            (ULONG_PTR)CpuPool, (ULONG_PTR)Pool);
    }
//#endif

    // See if the minimum time has passed since we last scavenged
    // the pool.  If it has, we'll scavenge again.
    //
    KeQueryTickCount(&Ticks);
    if (Ticks.QuadPart > CpuPool->NextScavengeTick.QuadPart)
    {
        Scavenge = TRUE;
    }

    // Note the tick that this page was last used.  If this is the last MDL to
    // be returned to this page, this sets the minimum time that this page will
    // continue to live unless it gets re-used.
    //
    Page->LastUsedTick.QuadPart = Ticks.QuadPart;

    // If this page is on the used-page list, we'll put it back on the normal
    // page list (only after pushing the MDL back on the page's free list)
    // if, after raising IRQL, we are on the processor that owns this
    // pool.
    //
    PageIsOnUsedPageList = Page->OnUsedPageList;


    InterlockedIncrement(&CpuPool->TotalMdlsFreed);

    // Now return the MDL to the page's free list.
    //
    InterlockedPushEntrySList(&Page->FreeList, (PSLIST_ENTRY)&Mdl->Next);

    //
    // Warning: Now that the MDL is back on the page, one cannot *reliably*
    // dereference anything through 'Page' anymore.  It may have just been
    // scavenged by its owning processor and subsequently freed.  This is a
    // particularly rare condition given that MINIMUM_PAGE_LIFETIME_IN_SECONDS
    // is 20s, so we choose to live with it.  The alternative would be to walk
    // the UsedPageList whenever PageIsOnUsedPageList is true, making the
    // MdpFree operation potentially expensive.  We saved off the value of
    // Page->OnUsedPageList before returning the MDL so we would not risk
    // touching Page to get this value only to find that it was false.
    //

    // If we need to move the page from the used-page list to the normal
    // page list, or if we need to scavenge, we need to be at DISPATCH_LEVEL
    // and be executing on the processor that owns this pool.
    // Find out if the CPU we are executing on right now owns this pool.
    // Note that if we are running at PASSIVE_LEVEL, the current CPU may
    // change over the duration of this function call, so this value is
    // not absolute over the life of the function.
    //
    if ((PageIsOnUsedPageList || Scavenge) &&
        (KeGetCurrentProcessorNumber() == CpuPool->OwnerCpu))
    {
        KIRQL OldIrql;

        OldIrql = KeRaiseIrqlToDpcLevel();

        // Now that we are at DISPATCH_LEVEL, perform the work if we are still
        // executing on the processor that owns the pool.
        //
        if (KeGetCurrentProcessorNumber() == CpuPool->OwnerCpu)
        {
            // If the page is still on the used-page list (meaning another
            // MdpFree didn't just sneak by) and still has a free MDL (for
            // instance, an MdpAllocate might sneak in on its owning processor,
            // scavenge the page, and allocate the MDL we just freed; thereby
            // putting it back on the used-page list), then put the page on the
            // normal list.  Very important to do this after (not before)
            // returning the MDL to the free list because MdpAllocate expects
            // MDL's to be available from pages on the page list.
            //
            if (PageIsOnUsedPageList &&
                Page->OnUsedPageList &&
                (0 != ExQueryDepthSList(&Page->FreeList)))
            {
                RemoveEntryList(&Page->PageLink);
                Page->OnUsedPageList = FALSE;
                InsertTailList(&CpuPool->PageList, &Page->PageLink);
                CpuPool->PagesOnUsedPageList--;

                PageIsOnUsedPageList = FALSE;

#if SHOW_DEBUG_OUTPUT
                DbgPrint(
                    "[%d] %c%c%c%c page moved off of used-page list\n",
                    CpuPool->OwnerCpu,
                    Pool->Tag, Pool->Tag >> 8, Pool->Tag >> 16, Pool->Tag >> 24);
#endif
            }

            // Perform the scavenge if we previously noted we needed to do so.
            //
            if (Scavenge)
            {
                MdppScavengePool(CpuPool);
            }
        }

        KeLowerIrql(OldIrql);
    }

    // If we missed being able to put this page back on the normal list.
    // note it.
    //
    if (PageIsOnUsedPageList)
    {
        CpuPool->MissedPageMove = TRUE;
    }
}