windows-server-2003/base/ntos/mm/dynmem.c

/*++

Copyright (c) 1999  Microsoft Corporation

Module Name:

    dynmem.c

Abstract:

    This module contains the routines which implement dynamically adding
    and removing physical memory from the system.

Author:

    Landy Wang (landyw) 05-Feb-1999

Revision History:

--*/

#include "mi.h"

KGUARDED_MUTEX MmDynamicMemoryMutex;

LOGICAL MiTrimRemovalPagesOnly = FALSE;

#if DBG
ULONG MiShowStuckPages;
ULONG MiDynmemData[9];
#endif

#if defined (_MI_COMPRESSION)
extern PMM_SET_COMPRESSION_THRESHOLD MiSetCompressionThreshold;
#endif

//
// Leave the low 3 bits clear as this will be inserted into the PFN PteAddress.
//

#define PFN_REMOVED     ((PMMPTE)(INT_PTR)(int)0x99887768)

PFN_COUNT
MiRemovePhysicalPages (
    IN PFN_NUMBER StartPage,
    IN PFN_NUMBER EndPage
    );

NTSTATUS
MiRemovePhysicalMemory (
    IN PPHYSICAL_ADDRESS StartAddress,
    IN OUT PLARGE_INTEGER NumberOfBytes,
    IN LOGICAL PermanentRemoval,
    IN ULONG Flags
    );

#ifdef ALLOC_PRAGMA
#pragma alloc_text(PAGE,MmRemovePhysicalMemory)
#pragma alloc_text(PAGE,MmMarkPhysicalMemoryAsBad)
#pragma alloc_text(PAGELK,MmAddPhysicalMemory)
#pragma alloc_text(PAGELK,MmAddPhysicalMemoryEx)
#pragma alloc_text(PAGELK,MiRemovePhysicalMemory)
#pragma alloc_text(PAGELK,MmMarkPhysicalMemoryAsGood)
#pragma alloc_text(PAGELK,MmGetPhysicalMemoryRanges)
#pragma alloc_text(PAGELK,MiRemovePhysicalPages)
#endif


NTSTATUS
MmAddPhysicalMemory (
    IN PPHYSICAL_ADDRESS StartAddress,
    IN OUT PLARGE_INTEGER NumberOfBytes
    )

/*++

Routine Description:

    A wrapper for MmAddPhysicalMemoryEx.

Arguments:

    StartAddress  - Supplies the starting physical address.

    NumberOfBytes  - Supplies a pointer to the number of bytes being added.
                     If any bytes were added (ie: STATUS_SUCCESS is being
                     returned), the actual amount is returned here.

Return Value:

    NTSTATUS.

Environment:

    Kernel mode.  PASSIVE level.  No locks held.

--*/

{
    return MmAddPhysicalMemoryEx (StartAddress, NumberOfBytes, 0);
}


NTSTATUS
MmAddPhysicalMemoryEx (
    IN PPHYSICAL_ADDRESS StartAddress,
    IN OUT PLARGE_INTEGER NumberOfBytes,
    IN ULONG Flags
    )

/*++

Routine Description:

    This routine adds the specified physical address range to the system.
    This includes initializing PFN database entries and adding it to the
    freelists.

Arguments:

    StartAddress  - Supplies the starting physical address.

    NumberOfBytes  - Supplies a pointer to the number of bytes being added.
                     If any bytes were added (ie: STATUS_SUCCESS is being
                     returned), the actual amount is returned here.

    Flags  - Supplies relevant flags describing the memory range.

Return Value:

    NTSTATUS.

Environment:

    Kernel mode.  PASSIVE level.  No locks held.

--*/

{
    ULONG i;
    PMMPFN Pfn1;
    KIRQL OldIrql;
    LOGICAL Inserted;
    LOGICAL Updated;
    MMPTE TempPte;
    PMMPTE PointerPte;
    PMMPTE LastPte;
    PFN_NUMBER PagesToReturn;
    PFN_NUMBER NumberOfPages;
    PFN_NUMBER start;
    PFN_NUMBER count;
    PFN_NUMBER StartPage;
    PFN_NUMBER EndPage;
    PFN_NUMBER PageFrameIndex;
    PFN_NUMBER Page;
    PFN_NUMBER LastPage;
    PFN_NUMBER TotalPagesAllowed;
    PFN_COUNT PagesNeeded;
    PPHYSICAL_MEMORY_DESCRIPTOR OldPhysicalMemoryBlock;
    PPHYSICAL_MEMORY_DESCRIPTOR NewPhysicalMemoryBlock;
    PPHYSICAL_MEMORY_RUN NewRun;
    LOGICAL PfnDatabaseIsPhysical;

    ASSERT (KeGetCurrentIrql() == PASSIVE_LEVEL);

    if (BYTE_OFFSET(StartAddress->LowPart) != 0) {
        return STATUS_INVALID_PARAMETER_1;
    }

    if (BYTE_OFFSET(NumberOfBytes->LowPart) != 0) {
        return STATUS_INVALID_PARAMETER_2;
    }

#if defined (_MI_COMPRESSION)
    if (Flags & ~MM_PHYSICAL_MEMORY_PRODUCED_VIA_COMPRESSION) {
        return STATUS_INVALID_PARAMETER_3;
    }
#else
    if (Flags != 0) {
        return STATUS_INVALID_PARAMETER_3;
    }
#endif

    //
    // The system must be configured for dynamic memory addition.  This is
    // critical as only then is the database guaranteed to be non-sparse.
    //
    
    if (MmDynamicPfn == 0) {
        return STATUS_NOT_SUPPORTED;
    }

    if (MI_IS_PHYSICAL_ADDRESS(MmPfnDatabase)) {
        PfnDatabaseIsPhysical = TRUE;
    }
    else {
        PfnDatabaseIsPhysical = FALSE;
    }

    StartPage = (PFN_NUMBER)(StartAddress->QuadPart >> PAGE_SHIFT);
    NumberOfPages = (PFN_NUMBER)(NumberOfBytes->QuadPart >> PAGE_SHIFT);

    EndPage = StartPage + NumberOfPages;

    if (StartPage >= EndPage) {
        return STATUS_INVALID_PARAMETER_1;
    }

    if (EndPage - 1 > MmHighestPossiblePhysicalPage) {

        //
        // Truncate the request into something that can be mapped by the PFN
        // database.
        //

        EndPage = MmHighestPossiblePhysicalPage + 1;
        NumberOfPages = EndPage - StartPage;
    }

    if (StartPage >= EndPage) {
        return STATUS_INVALID_PARAMETER_1;
    }

    //
    // Ensure that the memory being added does not exceed the license
    // restrictions.
    //

    if (ExVerifySuite(DataCenter) == TRUE) {
        TotalPagesAllowed = MI_DTC_MAX_PAGES;
    }
    else if ((MmProductType != 0x00690057) &&
             (ExVerifySuite(Enterprise) == TRUE)) {

        TotalPagesAllowed = MI_ADS_MAX_PAGES;
    }
    else {
        TotalPagesAllowed = MI_DEFAULT_MAX_PAGES;
    }

    if (MmNumberOfPhysicalPages + NumberOfPages > TotalPagesAllowed) {

        //
        // Truncate the request appropriately.
        //

        NumberOfPages = TotalPagesAllowed - MmNumberOfPhysicalPages;
        EndPage = StartPage + NumberOfPages;
    }

    //
    // The range cannot wrap.
    //

    if (StartPage >= EndPage) {
        return STATUS_INVALID_PARAMETER_1;
    }

    KeAcquireGuardedMutex (&MmDynamicMemoryMutex);

    OldPhysicalMemoryBlock = MmPhysicalMemoryBlock;

    i = (sizeof(PHYSICAL_MEMORY_DESCRIPTOR) +
         (sizeof(PHYSICAL_MEMORY_RUN) * (MmPhysicalMemoryBlock->NumberOfRuns + 1)));

    NewPhysicalMemoryBlock = ExAllocatePoolWithTag (NonPagedPool,
                                                    i,
                                                    '  mM');

    if (NewPhysicalMemoryBlock == NULL) {
        KeReleaseGuardedMutex (&MmDynamicMemoryMutex);
        return STATUS_INSUFFICIENT_RESOURCES;
    }

    //
    // The range cannot overlap any ranges that are already present.
    //

    start = 0;
    TempPte = ValidKernelPte;

    MmLockPagableSectionByHandle (ExPageLockHandle);

    LOCK_PFN (OldIrql);

#if defined (_MI_COMPRESSION)

    //
    // Adding compression-generated ranges can only be done if the hardware
    // has already successfully announced itself.
    //

    if (Flags & MM_PHYSICAL_MEMORY_PRODUCED_VIA_COMPRESSION) {
        if (MiSetCompressionThreshold == NULL) {
            UNLOCK_PFN (OldIrql);
            MmUnlockPagableImageSection(ExPageLockHandle);
            KeReleaseGuardedMutex (&MmDynamicMemoryMutex);
            ExFreePool (NewPhysicalMemoryBlock);
            return STATUS_NOT_SUPPORTED;
        }
    }
#endif

    do {

        count = MmPhysicalMemoryBlock->Run[start].PageCount;
        Page = MmPhysicalMemoryBlock->Run[start].BasePage;

        if (count != 0) {

            LastPage = Page + count;

            if ((StartPage < Page) && (EndPage > Page)) {
                UNLOCK_PFN (OldIrql);
                MmUnlockPagableImageSection(ExPageLockHandle);
                KeReleaseGuardedMutex (&MmDynamicMemoryMutex);
                ExFreePool (NewPhysicalMemoryBlock);
                return STATUS_CONFLICTING_ADDRESSES;
            }

            if ((StartPage >= Page) && (StartPage < LastPage)) {
                UNLOCK_PFN (OldIrql);
                MmUnlockPagableImageSection(ExPageLockHandle);
                KeReleaseGuardedMutex (&MmDynamicMemoryMutex);
                ExFreePool (NewPhysicalMemoryBlock);
                return STATUS_CONFLICTING_ADDRESSES;
            }
        }

        start += 1;

    } while (start != MmPhysicalMemoryBlock->NumberOfRuns);

    //
    // Fill any gaps in the (sparse) PFN database needed for these pages,
    // unless the PFN database was physically allocated and completely
    // committed up front.
    //

    PagesNeeded = 0;

    if (PfnDatabaseIsPhysical == FALSE) {
        PointerPte = MiGetPteAddress (MI_PFN_ELEMENT(StartPage));
        LastPte = MiGetPteAddress ((PCHAR)(MI_PFN_ELEMENT(EndPage)) - 1);
    
        while (PointerPte <= LastPte) {
            if (PointerPte->u.Hard.Valid == 0) {
                PagesNeeded += 1;
            }
            PointerPte += 1;
        }
    
        if (MmAvailablePages < PagesNeeded) {
            UNLOCK_PFN (OldIrql);
            MmUnlockPagableImageSection(ExPageLockHandle);
            KeReleaseGuardedMutex (&MmDynamicMemoryMutex);
            ExFreePool (NewPhysicalMemoryBlock);
            return STATUS_INSUFFICIENT_RESOURCES;
        }
    
        PointerPte = MiGetPteAddress (MI_PFN_ELEMENT(StartPage));
    
        while (PointerPte <= LastPte) {
            if (PointerPte->u.Hard.Valid == 0) {
    
                PageFrameIndex = MiRemoveZeroPage(MI_GET_PAGE_COLOR_FROM_PTE (PointerPte));
    
                MiInitializePfn (PageFrameIndex, PointerPte, 0);
    
                TempPte.u.Hard.PageFrameNumber = PageFrameIndex;

                MI_WRITE_VALID_PTE (PointerPte, TempPte);
            }
            PointerPte += 1;
        }
        MI_DECREMENT_RESIDENT_AVAILABLE (PagesNeeded, MM_RESAVAIL_ALLOCATE_HOTADD_PFNDB);
    }

    //
    // If the new range is adjacent to an existing range, just merge it into
    // the old block.  Otherwise use the new block as a new entry will have to
    // be used.
    //

    NewPhysicalMemoryBlock->NumberOfRuns = MmPhysicalMemoryBlock->NumberOfRuns + 1;
    NewPhysicalMemoryBlock->NumberOfPages = MmPhysicalMemoryBlock->NumberOfPages + NumberOfPages;

    NewRun = &NewPhysicalMemoryBlock->Run[0];
    start = 0;
    Inserted = FALSE;
    Updated = FALSE;

    do {

        Page = MmPhysicalMemoryBlock->Run[start].BasePage;
        count = MmPhysicalMemoryBlock->Run[start].PageCount;

        if (Inserted == FALSE) {

            //
            // Note overlaps into adjacent ranges were already checked above.
            //

            if (StartPage == Page + count) {
                MmPhysicalMemoryBlock->Run[start].PageCount += NumberOfPages;
                OldPhysicalMemoryBlock = NewPhysicalMemoryBlock;
                MmPhysicalMemoryBlock->NumberOfPages += NumberOfPages;

                //
                // Coalesce below and above to avoid leaving zero length gaps
                // as these gaps would prevent callers from removing ranges
                // the span them.
                //

                if (start + 1 < MmPhysicalMemoryBlock->NumberOfRuns) {

                    start += 1;
                    Page = MmPhysicalMemoryBlock->Run[start].BasePage;
                    count = MmPhysicalMemoryBlock->Run[start].PageCount;

                    if (StartPage + NumberOfPages == Page) {
                        MmPhysicalMemoryBlock->Run[start - 1].PageCount +=
                            count;
                        MmPhysicalMemoryBlock->NumberOfRuns -= 1;

                        //
                        // Copy any remaining entries.
                        //
    
                        if (start != MmPhysicalMemoryBlock->NumberOfRuns) {
                            RtlMoveMemory (&MmPhysicalMemoryBlock->Run[start],
                                           &MmPhysicalMemoryBlock->Run[start + 1],
                                           (MmPhysicalMemoryBlock->NumberOfRuns - start) * sizeof (PHYSICAL_MEMORY_RUN));
                        }
                    }
                }
                Updated = TRUE;
                break;
            }

            if (StartPage + NumberOfPages == Page) {
                MmPhysicalMemoryBlock->Run[start].BasePage = StartPage;
                MmPhysicalMemoryBlock->Run[start].PageCount += NumberOfPages;
                OldPhysicalMemoryBlock = NewPhysicalMemoryBlock;
                MmPhysicalMemoryBlock->NumberOfPages += NumberOfPages;
                Updated = TRUE;
                break;
            }

            if (StartPage + NumberOfPages <= Page) {

                if (start + 1 < MmPhysicalMemoryBlock->NumberOfRuns) {

                    if (StartPage + NumberOfPages <= MmPhysicalMemoryBlock->Run[start + 1].BasePage) {
                        //
                        // Don't insert here - the new entry really belongs
                        // (at least) one entry further down.
                        //

                        continue;
                    }
                }

                NewRun->BasePage = StartPage;
                NewRun->PageCount = NumberOfPages;
                NewRun += 1;
                Inserted = TRUE;
                Updated = TRUE;
            }
        }

        *NewRun = MmPhysicalMemoryBlock->Run[start];
        NewRun += 1;

        start += 1;

    } while (start != MmPhysicalMemoryBlock->NumberOfRuns);

    //
    // If the memory block has not been updated, then the new entry must
    // be added at the very end.
    //

    if (Updated == FALSE) {
        ASSERT (Inserted == FALSE);
        NewRun->BasePage = StartPage;
        NewRun->PageCount = NumberOfPages;
        Inserted = TRUE;
    }

    //
    // Repoint the MmPhysicalMemoryBlock at the new chunk, free the old after
    // releasing the PFN lock.
    //

    if (Inserted == TRUE) {
        OldPhysicalMemoryBlock = MmPhysicalMemoryBlock;
        MmPhysicalMemoryBlock = NewPhysicalMemoryBlock;
    }

    //
    // Note that the page directory (page parent entries on Win64) must be
    // filled in at system boot so that already-created processes do not fault
    // when referencing the new PFNs.
    //

    //
    // Walk through the memory descriptors and add pages to the
    // free list in the PFN database.
    //

    PageFrameIndex = StartPage;
    Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);

    if (EndPage - 1 > MmHighestPhysicalPage) {
        MmHighestPhysicalPage = EndPage - 1;
    }

    while (PageFrameIndex < EndPage) {

        ASSERT (Pfn1->u2.ShareCount == 0);
        ASSERT (Pfn1->u3.e2.ShortFlags == 0);
        ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
        ASSERT64 (Pfn1->UsedPageTableEntries == 0);
        ASSERT (Pfn1->OriginalPte.u.Long == ZeroKernelPte.u.Long);
        ASSERT (Pfn1->u4.PteFrame == 0);
        ASSERT ((Pfn1->PteAddress == PFN_REMOVED) ||
                (Pfn1->PteAddress == (PMMPTE)(UINT_PTR)0));

        //
        // Initialize the color for NUMA purposes.
        //

        MiDetermineNode (PageFrameIndex, Pfn1);

        //
        // Set the PTE address to the physical page for
        // virtual address alignment checking.
        //

        Pfn1->PteAddress = (PMMPTE)(PageFrameIndex << PTE_SHIFT);

        MiInsertPageInFreeList (PageFrameIndex);

        PageFrameIndex += 1;
        
        Pfn1 += 1;
    }

    MmNumberOfPhysicalPages += (PFN_COUNT)NumberOfPages;

    //
    // Only non-compression ranges get to contribute to ResidentAvailable as
    // adding compression ranges to this could crash the system.
    //
    // For the same reason, compression range additions also need to subtract
    // from AvailablePages the amount the above MiInsertPageInFreeList added.
    //

    PagesToReturn = NumberOfPages;

#if defined (_MI_COMPRESSION)
    if (Flags & MM_PHYSICAL_MEMORY_PRODUCED_VIA_COMPRESSION) {
        MmAvailablePages -= (PFN_COUNT) NumberOfPages;

        //
        // Signal applications if allocating these pages caused a threshold cross.
        //

        MiNotifyMemoryEvents ();

        MiNumberOfCompressionPages += NumberOfPages;
        PagesToReturn = 0;
    }
    else {

        //
        // Since real (noncompression-generated) physical memory was added,
        // rearm the interrupt to occur at a higher threshold.
        //

        MiArmCompressionInterrupt ();
    }
#endif

    RtlSetBits (&MiPfnBitMap, (ULONG) StartPage, (ULONG) (EndPage - StartPage));

    UNLOCK_PFN (OldIrql);

    MI_INCREMENT_RESIDENT_AVAILABLE (PagesToReturn,
                                     MM_RESAVAIL_FREE_HOTADD_MEMORY);

    InterlockedExchangeAdd ((PLONG)&SharedUserData->NumberOfPhysicalPages,
                            (LONG) NumberOfPages);

    //
    // Carefully increase all commit limits to reflect the additional memory -
    // notice the current usage must be done first so no one else cuts the
    // line.
    //

    InterlockedExchangeAddSizeT (&MmTotalCommittedPages, PagesNeeded);

    InterlockedExchangeAddSizeT (&MmTotalCommitLimitMaximum, NumberOfPages);

    InterlockedExchangeAddSizeT (&MmTotalCommitLimit, NumberOfPages);

    MmUnlockPagableImageSection(ExPageLockHandle);

    KeReleaseGuardedMutex (&MmDynamicMemoryMutex);

    ExFreePool (OldPhysicalMemoryBlock);

    //
    // Indicate number of bytes actually added to our caller.
    //

    NumberOfBytes->QuadPart = (ULONGLONG)NumberOfPages * PAGE_SIZE;

    return STATUS_SUCCESS;
}


NTSTATUS
MiRemovePhysicalMemory (
    IN PPHYSICAL_ADDRESS StartAddress,
    IN OUT PLARGE_INTEGER NumberOfBytes,
    IN LOGICAL PermanentRemoval,
    IN ULONG Flags
    )

/*++

Routine Description:

    This routine attempts to remove the specified physical address range
    from the system.

Arguments:

    StartAddress  - Supplies the starting physical address.

    NumberOfBytes  - Supplies a pointer to the number of bytes being removed.

    PermanentRemoval  - Supplies TRUE if the memory is being permanently
                        (ie: physically) removed.  FALSE if not (ie: just a
                        bad page detected via ECC which is being marked
                        "don't-use".

Return Value:

    NTSTATUS.

Environment:

    Kernel mode.  PASSIVE level.  No locks held.

--*/

{
    ULONG i;
    ULONG Additional;
    PFN_NUMBER Page;
    PFN_NUMBER LastPage;
    PFN_NUMBER OriginalLastPage;
    PFN_NUMBER start;
    PFN_NUMBER PagesReleased;
    PFN_NUMBER ResAvailPagesReleased;
    PMMPFN Pfn1;
    PMMPFN StartPfn;
    PMMPFN EndPfn;
    KIRQL OldIrql;
    PFN_NUMBER StartPage;
    PFN_NUMBER EndPage;
    PFN_COUNT NumberOfPages;
    PFN_COUNT ParityPages;
    SPFN_NUMBER MaxPages;
    PFN_NUMBER PageFrameIndex;
    PFN_NUMBER RemovedPages;
    PFN_NUMBER RemovedPagesThisPass;
    LOGICAL Inserted;
    NTSTATUS Status;
    PMMPTE PointerPte;
    PMMPTE EndPte;
    PVOID VirtualAddress;
    PPHYSICAL_MEMORY_DESCRIPTOR OldPhysicalMemoryBlock;
    PPHYSICAL_MEMORY_DESCRIPTOR NewPhysicalMemoryBlock;
    PPHYSICAL_MEMORY_RUN NewRun;
    LOGICAL PfnDatabaseIsPhysical;
    PFN_NUMBER HighestPossiblePhysicalPage;
    PFN_COUNT FluidPages;
    MMPTE_FLUSH_LIST PteFlushList;

    ASSERT (KeGetCurrentIrql() == PASSIVE_LEVEL);

    ASSERT (BYTE_OFFSET(NumberOfBytes->LowPart) == 0);
    ASSERT (BYTE_OFFSET(StartAddress->LowPart) == 0);

    if (MI_IS_PHYSICAL_ADDRESS(MmPfnDatabase)) {

        if (PermanentRemoval == TRUE) {

            //
            // The system must be configured for dynamic memory addition.  This
            // is not strictly required to remove the memory, but it's better
            // to check for it now under the assumption that the administrator
            // is probably going to want to add this range of memory back in -
            // better to give the error now and refuse the removal than to
            // refuse the addition later.
            //
        
            if (MmDynamicPfn == 0) {
                return STATUS_NOT_SUPPORTED;
            }
        }
    
        PfnDatabaseIsPhysical = TRUE;
    }
    else {
        PfnDatabaseIsPhysical = FALSE;
    }

    if (PermanentRemoval == TRUE) {
        HighestPossiblePhysicalPage = MmHighestPossiblePhysicalPage;
        FluidPages = 100;
    }
    else {
        HighestPossiblePhysicalPage = MmHighestPhysicalPage;
        FluidPages = 0;
    }

    StartPage = (PFN_NUMBER)(StartAddress->QuadPart >> PAGE_SHIFT);
    NumberOfPages = (PFN_COUNT)(NumberOfBytes->QuadPart >> PAGE_SHIFT);

    EndPage = StartPage + NumberOfPages;

    if (StartPage >= EndPage) {
        return STATUS_INVALID_PARAMETER_1;
    }

    if (EndPage - 1 > HighestPossiblePhysicalPage) {

        //
        // Truncate the request into something that can be mapped by the PFN
        // database.
        //

        EndPage = MmHighestPossiblePhysicalPage + 1;
        NumberOfPages = (PFN_COUNT)(EndPage - StartPage);
    }

    if (StartPage >= EndPage) {
        return STATUS_INVALID_PARAMETER_1;
    }

    //
    // The range cannot wrap.
    //

    if (StartPage >= EndPage) {
        return STATUS_INVALID_PARAMETER_1;
    }

#if !defined (_MI_COMPRESSION)
    if (Flags != 0) {
        return STATUS_INVALID_PARAMETER_4;
    }
#endif

    StartPfn = MI_PFN_ELEMENT (StartPage);
    EndPfn = MI_PFN_ELEMENT (EndPage);

    KeAcquireGuardedMutex (&MmDynamicMemoryMutex);

    //
    // Make sure the caller is freeing real memory (ie: PFN-backed).
    //

    if (RtlAreBitsSet (&MiPfnBitMap,
                       (PFN_COUNT) StartPage,
                       (PFN_COUNT) (EndPage - StartPage)) == FALSE) {

        KeReleaseGuardedMutex (&MmDynamicMemoryMutex);
        return STATUS_INVALID_PARAMETER_1;
    }

#if DBG
    MiDynmemData[0] += 1;
#endif

    //
    // Attempt to decrease all commit limits to reflect the removed memory.
    //

    if (MiChargeTemporaryCommitmentForReduction (NumberOfPages + FluidPages) == FALSE) {
#if DBG
        MiDynmemData[1] += 1;
#endif
        KeReleaseGuardedMutex (&MmDynamicMemoryMutex);
        return STATUS_INSUFFICIENT_RESOURCES;
    }

    //
    // Reduce the systemwide commit limit - note this is carefully done
    // *PRIOR* to returning this commitment so no one else (including a DPC
    // in this very thread) can consume past the limit.
    //

    InterlockedExchangeAddSizeT (&MmTotalCommitLimit, 0 - (PFN_NUMBER)NumberOfPages);

    InterlockedExchangeAddSizeT (&MmTotalCommitLimitMaximum, 0 - (PFN_NUMBER)NumberOfPages);

    //
    // Now that the systemwide commit limit has been lowered, the amount
    // we have removed can be safely returned.
    //

    MiReturnCommitment (NumberOfPages + FluidPages);

    MmLockPagableSectionByHandle (ExPageLockHandle);

    //
    // Check for outstanding promises that cannot be broken.
    //

    LOCK_PFN (OldIrql);

    if (PermanentRemoval == FALSE) {

        //
        // If it's just the removal of ECC-marked bad pages, then don't
        // allow the caller to remove any pages that have already been
        // ECC-removed.  This is to prevent recursive erroneous charges.
        //

        for (Pfn1 = StartPfn; Pfn1 < EndPfn; Pfn1 += 1) {
            if (Pfn1->u3.e1.ParityError == 1) {
                UNLOCK_PFN (OldIrql);
                Status = STATUS_INVALID_PARAMETER_2;
                goto giveup2;
            }
        }
    }

    MaxPages = MI_NONPAGABLE_MEMORY_AVAILABLE() - FluidPages;

    if ((SPFN_NUMBER)NumberOfPages > MaxPages) {
#if DBG
        MiDynmemData[2] += 1;
#endif
        UNLOCK_PFN (OldIrql);
        Status = STATUS_INSUFFICIENT_RESOURCES;
        goto giveup2;
    }

    //
    // The range must be contained in a single entry.  It is
    // permissible for it to be part of a single entry, but it
    // must not cross multiple entries.
    //

    Additional = (ULONG)-2;

    start = 0;
    do {

        Page = MmPhysicalMemoryBlock->Run[start].BasePage;
        LastPage = Page + MmPhysicalMemoryBlock->Run[start].PageCount;

        if ((StartPage >= Page) && (EndPage <= LastPage)) {
            if ((StartPage == Page) && (EndPage == LastPage)) {
                Additional = (ULONG)-1;
            }
            else if ((StartPage == Page) || (EndPage == LastPage)) {
                Additional = 0;
            }
            else {
                Additional = 1;
            }
            break;
        }

        start += 1;

    } while (start != MmPhysicalMemoryBlock->NumberOfRuns);

    if (Additional == (ULONG)-2) {
#if DBG
        MiDynmemData[3] += 1;
#endif
        UNLOCK_PFN (OldIrql);
        Status = STATUS_CONFLICTING_ADDRESSES;
        goto giveup2;
    }

    for (Pfn1 = StartPfn; Pfn1 < EndPfn; Pfn1 += 1) {
        Pfn1->u3.e1.RemovalRequested = 1;
    }

    if (PermanentRemoval == TRUE) {
        MmNumberOfPhysicalPages -= NumberOfPages;

        InterlockedExchangeAdd ((PLONG)&SharedUserData->NumberOfPhysicalPages,
                                0 - NumberOfPages);
    }

#if defined (_MI_COMPRESSION)

    //
    // Only removal of non-compression ranges decrement ResidentAvailable as
    // only those ranges actually incremented this when they were added.
    //

    if ((Flags & MM_PHYSICAL_MEMORY_PRODUCED_VIA_COMPRESSION) == 0) {
        MI_DECREMENT_RESIDENT_AVAILABLE (NumberOfPages, MM_RESAVAIL_ALLOCATE_HOTREMOVE_MEMORY);

        //
        // Since real (noncompression-generated) physical memory is being
        // removed, rearm the interrupt to occur at a lower threshold.
        //

        if (PermanentRemoval == TRUE) {
            MiArmCompressionInterrupt ();
        }
    }
#else
    MI_DECREMENT_RESIDENT_AVAILABLE (NumberOfPages, MM_RESAVAIL_ALLOCATE_HOTREMOVE_MEMORY);
#endif

    //
    // The free and zero lists must be pruned now before releasing the PFN
    // lock otherwise if another thread allocates the page from these lists,
    // the allocation will clear the RemovalRequested flag forever.
    //

    RemovedPages = MiRemovePhysicalPages (StartPage, EndPage);

#if defined (_MI_COMPRESSION)

    //
    // Compression range removals add back into AvailablePages the same
    // amount that MiUnlinkPageFromList removes (as the original addition
    // of these ranges never bumps this counter).
    //

    if (Flags & MM_PHYSICAL_MEMORY_PRODUCED_VIA_COMPRESSION) {
        MmAvailablePages += (PFN_COUNT) RemovedPages;

        //
        // Signal applications if allocating these pages caused a threshold cross.
        //

        MiNotifyMemoryEvents ();

        MiNumberOfCompressionPages -= RemovedPages;
    }
#endif

    if (RemovedPages != NumberOfPages) {

#if DBG
retry:
#endif
    
        Pfn1 = StartPfn;
    
        InterlockedIncrement (&MiDelayPageFaults);
    
        for (i = 0; i < 5; i += 1) {
    
            UNLOCK_PFN (OldIrql);
    
            //
            // Attempt to move pages to the standby list.  Note that only the
            // pages with RemovalRequested set are moved.
            //
    
            MiTrimRemovalPagesOnly = TRUE;
    
            MmEmptyAllWorkingSets ();
    
            MiTrimRemovalPagesOnly = FALSE;
    
            MiFlushAllPages ();
    
            KeDelayExecutionThread (KernelMode, FALSE, (PLARGE_INTEGER)&MmHalfSecond);
    
            if (i >= 2) {

                //
                // Purge the transition list as transition pages keep
                // page tables from being taken and we need to try harder.
                //

                MiPurgeTransitionList ();
            }

            LOCK_PFN (OldIrql);
    
            RemovedPagesThisPass = MiRemovePhysicalPages (StartPage, EndPage);

            RemovedPages += RemovedPagesThisPass;
    
#if defined (_MI_COMPRESSION)

            //
            // Compression range removals add back into AvailablePages the same
            // amount that MiUnlinkPageFromList removes (as the original
            // addition of these ranges never bumps this counter).
            //

            if (Flags & MM_PHYSICAL_MEMORY_PRODUCED_VIA_COMPRESSION) {
                MmAvailablePages += (PFN_COUNT) RemovedPagesThisPass;

                //
                // Signal applications if allocating these pages
                // caused a threshold cross.
                //

                MiNotifyMemoryEvents ();

                MiNumberOfCompressionPages -= RemovedPagesThisPass;
            }

#endif

            if (RemovedPages == NumberOfPages) {
                break;
            }
    
            //
            // RemovedPages doesn't include pages that were freed directly
            // to the bad page list via MiDecrementReferenceCount or by
            // ECC marking.  So use the above check purely as an optimization -
            // and walk here before ever giving up.
            //

            for ( ; Pfn1 < EndPfn; Pfn1 += 1) {
                if (Pfn1->u3.e1.PageLocation != BadPageList) {
                    break;
                }
            }

            if (Pfn1 == EndPfn) {
                RemovedPages = NumberOfPages;
                break;
            }
        }

        InterlockedDecrement (&MiDelayPageFaults);
    }

    if (RemovedPages != NumberOfPages) {
#if DBG
        MiDynmemData[4] += 1;
        if (MiShowStuckPages != 0) {

            RemovedPages = 0;
            for (Pfn1 = StartPfn; Pfn1 < EndPfn; Pfn1 += 1) {
                if (Pfn1->u3.e1.PageLocation != BadPageList) {
                    RemovedPages += 1;
                }
            }

            ASSERT (RemovedPages != 0);

            DbgPrint("MiRemovePhysicalMemory : could not get %d of %d pages\n",
                RemovedPages, NumberOfPages);

            if (MiShowStuckPages & 0x2) {

                ULONG PfnsPrinted;
                ULONG EnoughShown;
                PMMPFN FirstPfn;
                PFN_COUNT PfnCount;

                PfnCount = 0;
                PfnsPrinted = 0;
                EnoughShown = 100;
    
                //
                // Initializing FirstPfn is not needed for correctness
                // but without it the compiler cannot compile this code
                // W4 to check for use of uninitialized variables.
                //

                FirstPfn = NULL;

                if (MiShowStuckPages & 0x4) {
                    EnoughShown = (ULONG)-1;
                }
    
                DbgPrint("Stuck PFN list: ");
                for (Pfn1 = StartPfn; Pfn1 < EndPfn; Pfn1 += 1) {
                    if (Pfn1->u3.e1.PageLocation != BadPageList) {
                        if (PfnCount == 0) {
                            FirstPfn = Pfn1;
                        }
                        PfnCount += 1;
                    }
                    else {
                        if (PfnCount != 0) {
                            DbgPrint("%x -> %x ; ", MI_PFN_ELEMENT_TO_INDEX (FirstPfn),
                                                    MI_PFN_ELEMENT_TO_INDEX (FirstPfn + PfnCount - 1));
                            PfnsPrinted += 1;
                            if (PfnsPrinted == EnoughShown) {
                                break;
                            }
                            PfnCount = 0;
                        }
                    }
                }
                if (PfnCount != 0) {
                    DbgPrint("%x -> %x ; ", MI_PFN_ELEMENT_TO_INDEX (FirstPfn),
                                            MI_PFN_ELEMENT_TO_INDEX (FirstPfn + PfnCount - 1));
                }
                DbgPrint("\n");
            }
            if (MiShowStuckPages & 0x8) {
                DbgBreakPoint ();
            }
            if (MiShowStuckPages & 0x10) {
                goto retry;
            }
        }
#endif
        UNLOCK_PFN (OldIrql);
        Status = STATUS_NO_MEMORY;
        goto giveup;
    }

#if DBG
    for (Pfn1 = StartPfn; Pfn1 < EndPfn; Pfn1 += 1) {
        ASSERT (Pfn1->u3.e1.PageLocation == BadPageList);
    }
#endif

    //
    // All the pages in the range have been removed.
    //

    if (PermanentRemoval == FALSE) {

        //
        // If it's just the removal of ECC-marked bad pages, then no
        // adjustment to the physical memory block ranges or PFN database
        // trimming is needed.  Exit now.
        //

        for (Pfn1 = StartPfn; Pfn1 < EndPfn; Pfn1 += 1) {
            ASSERT (Pfn1->u3.e1.ParityError == 0);
            Pfn1->u3.e1.ParityError = 1;
        }

        UNLOCK_PFN (OldIrql);

        MmUnlockPagableImageSection(ExPageLockHandle);
    
        KeReleaseGuardedMutex (&MmDynamicMemoryMutex);
    
        NumberOfBytes->QuadPart = (ULONGLONG)NumberOfPages * PAGE_SIZE;
    
        return STATUS_SUCCESS;
    }

    //
    // Update the physical memory blocks and other associated housekeeping.
    //

    if (Additional == 0) {

        //
        // The range can be split off from an end of an existing chunk so no
        // pool growth or shrinkage is required.
        //

        NewPhysicalMemoryBlock = MmPhysicalMemoryBlock;
        OldPhysicalMemoryBlock = NULL;
    }
    else {

        //
        // The range cannot be split off from an end of an existing chunk so
        // pool growth or shrinkage is required.
        //

        UNLOCK_PFN (OldIrql);

        i = (sizeof(PHYSICAL_MEMORY_DESCRIPTOR) +
             (sizeof(PHYSICAL_MEMORY_RUN) * (MmPhysicalMemoryBlock->NumberOfRuns + Additional)));

        NewPhysicalMemoryBlock = ExAllocatePoolWithTag (NonPagedPool,
                                                        i,
                                                        '  mM');

        if (NewPhysicalMemoryBlock == NULL) {
            Status = STATUS_INSUFFICIENT_RESOURCES;
#if DBG
            MiDynmemData[5] += 1;
#endif
            goto giveup;
        }

        OldPhysicalMemoryBlock = MmPhysicalMemoryBlock;
        RtlZeroMemory (NewPhysicalMemoryBlock, i);

        LOCK_PFN (OldIrql);
    }

    //
    // Remove or split the requested range from the existing memory block.
    //

    NewPhysicalMemoryBlock->NumberOfRuns = MmPhysicalMemoryBlock->NumberOfRuns + Additional;
    NewPhysicalMemoryBlock->NumberOfPages = MmPhysicalMemoryBlock->NumberOfPages - NumberOfPages;

    NewRun = &NewPhysicalMemoryBlock->Run[0];
    start = 0;
    Inserted = FALSE;

    do {

        Page = MmPhysicalMemoryBlock->Run[start].BasePage;
        LastPage = Page + MmPhysicalMemoryBlock->Run[start].PageCount;

        if (Inserted == FALSE) {

            if ((StartPage >= Page) && (EndPage <= LastPage)) {

                if ((StartPage == Page) && (EndPage == LastPage)) {
                    ASSERT (Additional == -1);
                    start += 1;
                    continue;
                }
                else if ((StartPage == Page) || (EndPage == LastPage)) {
                    ASSERT (Additional == 0);
                    if (StartPage == Page) {
                        MmPhysicalMemoryBlock->Run[start].BasePage += NumberOfPages;
                    }
                    MmPhysicalMemoryBlock->Run[start].PageCount -= NumberOfPages;
                }
                else {
                    ASSERT (Additional == 1);

                    OriginalLastPage = LastPage;

                    MmPhysicalMemoryBlock->Run[start].PageCount =
                        StartPage - MmPhysicalMemoryBlock->Run[start].BasePage;

                    *NewRun = MmPhysicalMemoryBlock->Run[start];
                    NewRun += 1;

                    NewRun->BasePage = EndPage;
                    NewRun->PageCount = OriginalLastPage - EndPage;
                    NewRun += 1;

                    start += 1;
                    continue;
                }

                Inserted = TRUE;
            }
        }

        *NewRun = MmPhysicalMemoryBlock->Run[start];
        NewRun += 1;
        start += 1;

    } while (start != MmPhysicalMemoryBlock->NumberOfRuns);

    //
    // Repoint the MmPhysicalMemoryBlock at the new chunk.
    // Free the old block after releasing the PFN lock.
    //

    MmPhysicalMemoryBlock = NewPhysicalMemoryBlock;

    if (EndPage - 1 == MmHighestPhysicalPage) {
        MmHighestPhysicalPage = StartPage - 1;
    }

    //
    // Throw away all the removed pages that are currently enqueued.
    //

    ParityPages = 0;
    for (Pfn1 = StartPfn; Pfn1 < EndPfn; Pfn1 += 1) {

        ASSERT (Pfn1->u3.e1.PageLocation == BadPageList);
        ASSERT (Pfn1->u3.e1.RemovalRequested == 1);

        //
        // Some pages may have already been ECC-removed.  For these pages,
        // the commit limits and resident available pages have already been
        // adjusted - tally them here so we can undo the extraneous charge
        // just applied.
        //
    
        if (Pfn1->u3.e1.ParityError == 1) {
            ParityPages += 1;
        }

        MiUnlinkPageFromList (Pfn1);

        ASSERT (Pfn1->u1.Flink == 0);
        ASSERT (Pfn1->u2.Blink == 0);
        ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
        ASSERT64 (Pfn1->UsedPageTableEntries == 0);

        Pfn1->PteAddress = PFN_REMOVED;

        //
        // Note this clears ParityError among other flags...
        //

        Pfn1->u3.e2.ShortFlags = 0;
        Pfn1->OriginalPte.u.Long = ZeroKernelPte.u.Long;
        Pfn1->u4.PteFrame = 0;
    }

    //
    // Now that the removed pages have been discarded, eliminate the PFN
    // entries that mapped them.  Straddling entries left over from an
    // adjacent earlier removal are not collapsed at this point.
    //
    //

    PagesReleased = 0;
    PteFlushList.Count = 0;

    if (PfnDatabaseIsPhysical == FALSE) {

        VirtualAddress = (PVOID)ROUND_TO_PAGES(MI_PFN_ELEMENT(StartPage));
        PointerPte = MiGetPteAddress (VirtualAddress);
        EndPte = MiGetPteAddress (PAGE_ALIGN(MI_PFN_ELEMENT(EndPage)));

        while (PointerPte < EndPte) {
            PageFrameIndex = MI_GET_PAGE_FRAME_FROM_PTE (PointerPte);
            Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
            ASSERT (Pfn1->u2.ShareCount == 1);
            ASSERT (Pfn1->u3.e2.ReferenceCount == 1);
            Pfn1->u2.ShareCount = 0;
            MI_SET_PFN_DELETED (Pfn1);
#if DBG
            Pfn1->u3.e1.PageLocation = StandbyPageList;
#endif //DBG
            MiDecrementReferenceCount (Pfn1, PageFrameIndex);
    
            MI_WRITE_INVALID_PTE (PointerPte, ZeroKernelPte);

            if (PteFlushList.Count < MM_MAXIMUM_FLUSH_COUNT) {
                PteFlushList.FlushVa[PteFlushList.Count] = VirtualAddress;
                PteFlushList.Count += 1;
            }

            PagesReleased += 1;
            PointerPte += 1;
            VirtualAddress = (PVOID)((PCHAR)VirtualAddress + PAGE_SIZE);
        }
    }

    if (PteFlushList.Count != 0) {
        MiFlushPteList (&PteFlushList, TRUE);
    }

#if DBG
    MiDynmemData[6] += 1;
#endif

    RtlClearBits (&MiPfnBitMap,
                  (ULONG) MI_PFN_ELEMENT_TO_INDEX (StartPfn),
                  (ULONG) (EndPfn - StartPfn));

    UNLOCK_PFN (OldIrql);

    //
    // Give back anything that has been double-charged.
    //

    ResAvailPagesReleased = PagesReleased;

    if (ParityPages != 0) {
        ResAvailPagesReleased += ParityPages;
    }

    if (ResAvailPagesReleased != 0) {
        MI_INCREMENT_RESIDENT_AVAILABLE (ResAvailPagesReleased,
                                         MM_RESAVAIL_FREE_HOTREMOVE_MEMORY1);
    }

    //
    // Give back anything that has been double-charged.
    //

    if (ParityPages != 0) {
        InterlockedExchangeAddSizeT (&MmTotalCommitLimitMaximum, ParityPages);
        InterlockedExchangeAddSizeT (&MmTotalCommitLimit, ParityPages);
    }

    if (PagesReleased != 0) {
        MiReturnCommitment (PagesReleased);
    }

    MmUnlockPagableImageSection(ExPageLockHandle);

    KeReleaseGuardedMutex (&MmDynamicMemoryMutex);

    if (OldPhysicalMemoryBlock != NULL) {
        ExFreePool (OldPhysicalMemoryBlock);
    }

    NumberOfBytes->QuadPart = (ULONGLONG)NumberOfPages * PAGE_SIZE;

    return STATUS_SUCCESS;

giveup:

    //
    // All the pages in the range were not obtained.  Back everything out.
    //

    PageFrameIndex = StartPage;
    Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);

    LOCK_PFN (OldIrql);

    while (PageFrameIndex < EndPage) {

        ASSERT (Pfn1->u3.e1.RemovalRequested == 1);

        Pfn1->u3.e1.RemovalRequested = 0;

        if (Pfn1->u3.e1.PageLocation == BadPageList) {
            MiUnlinkPageFromList (Pfn1);
            MiInsertPageInFreeList (PageFrameIndex);
        }

        Pfn1 += 1;
        PageFrameIndex += 1;
    }

    ResAvailPagesReleased = NumberOfPages;

#if defined (_MI_COMPRESSION)

    //
    // Only removal of non-compression ranges decrement ResidentAvailable as
    // only those ranges actually incremented this when they were added.
    //

    if (Flags & MM_PHYSICAL_MEMORY_PRODUCED_VIA_COMPRESSION) {

        //
        // Compression range removals add back into AvailablePages the same
        // amount that MiUnlinkPageFromList removes (as the original
        // addition of these ranges never bumps this counter).
        //

        ResAvailPagesReleased = 0;
        MmAvailablePages -= (PFN_COUNT) RemovedPages;

        //
        // Signal applications if allocating these pages caused a threshold cross.
        //

        MiNotifyMemoryEvents ();

        MiNumberOfCompressionPages += RemovedPages;
    }
#endif

    if (PermanentRemoval == TRUE) {
        MmNumberOfPhysicalPages += NumberOfPages;

        InterlockedExchangeAdd ((PLONG)&SharedUserData->NumberOfPhysicalPages,
                                NumberOfPages);

#if defined (_MI_COMPRESSION)

        //
        // Rearm the interrupt to occur at the original threshold.
        //

        if ((Flags & MM_PHYSICAL_MEMORY_PRODUCED_VIA_COMPRESSION) == 0) {
            MiArmCompressionInterrupt ();
        }
#endif
    }

    UNLOCK_PFN (OldIrql);

    MI_INCREMENT_RESIDENT_AVAILABLE (ResAvailPagesReleased, MM_RESAVAIL_FREE_HOTREMOVE_FAILED);

giveup2:

    InterlockedExchangeAddSizeT (&MmTotalCommitLimitMaximum, NumberOfPages);
    InterlockedExchangeAddSizeT (&MmTotalCommitLimit, NumberOfPages);

    MmUnlockPagableImageSection(ExPageLockHandle);
    KeReleaseGuardedMutex (&MmDynamicMemoryMutex);

    return Status;
}


NTSTATUS
MmRemovePhysicalMemory (
    IN PPHYSICAL_ADDRESS StartAddress,
    IN OUT PLARGE_INTEGER NumberOfBytes
    )

/*++

Routine Description:

    A wrapper for MmRemovePhysicalMemoryEx.

Arguments:

    StartAddress  - Supplies the starting physical address.

    NumberOfBytes  - Supplies a pointer to the number of bytes being removed.

Return Value:

    NTSTATUS.

Environment:

    Kernel mode.  PASSIVE level.  No locks held.

--*/

{
    return MmRemovePhysicalMemoryEx (StartAddress, NumberOfBytes, 0);
}

NTSTATUS
MmRemovePhysicalMemoryEx (
    IN PPHYSICAL_ADDRESS StartAddress,
    IN OUT PLARGE_INTEGER NumberOfBytes,
    IN ULONG Flags
    )

/*++

Routine Description:

    This routine attempts to remove the specified physical address range
    from the system.

Arguments:

    StartAddress  - Supplies the starting physical address.

    NumberOfBytes  - Supplies a pointer to the number of bytes being removed.

    Flags  - Supplies relevant flags describing the memory range.

Return Value:

    NTSTATUS.

Environment:

    Kernel mode.  PASSIVE level.  No locks held.

--*/

{
    NTSTATUS Status;
#if defined (_X86_) || defined (_AMD64_)
    BOOLEAN CachesFlushed;
#endif
#if defined(_IA64_)
    PVOID VirtualAddress;
    PVOID SingleVirtualAddress;
    SIZE_T SizeInBytes;
    SIZE_T MapSizeInBytes;
    PFN_COUNT NumberOfPages;
    PFN_COUNT i;
    PFN_NUMBER StartPage;
#endif

    PAGED_CODE();

#if defined (_MI_COMPRESSION_SUPPORTED_)
    if (Flags & MM_PHYSICAL_MEMORY_PRODUCED_VIA_COMPRESSION) {
        return STATUS_NOT_SUPPORTED;
    }
#else
    if (Flags != 0) {
        return STATUS_INVALID_PARAMETER_3;
    }
#endif

#if defined (_X86_) || defined (_AMD64_)

    //
    // Issue a cache invalidation here just as a test to make sure the
    // machine can support it.  If not, then don't bother trying to remove
    // any memory.
    //

    CachesFlushed = KeInvalidateAllCaches ();
    if (CachesFlushed == FALSE) {
        return STATUS_NOT_SUPPORTED;
    }
#endif

#if defined(_IA64_)

    //
    // Pick up at least a single PTE mapping now as we do not want to fail this
    // call if no PTEs are available after a successful remove.  Resorting to
    // actually using this PTE should be a very rare case indeed.
    //

    SingleVirtualAddress = (PMMPTE)MiMapSinglePage (NULL,
                                                    0,
                                                    MmCached,
                                                    HighPagePriority);

    if (SingleVirtualAddress == NULL) {
        return STATUS_INSUFFICIENT_RESOURCES;
    }

#endif

    Status = MiRemovePhysicalMemory (StartAddress, NumberOfBytes, TRUE, Flags);

    if (NT_SUCCESS (Status)) {

#if defined (_X86_) || defined (_AMD64_)
        CachesFlushed = KeInvalidateAllCaches ();
        ASSERT (CachesFlushed == TRUE);
#endif

#if defined(_IA64_)
        SizeInBytes = (SIZE_T)NumberOfBytes->QuadPart;

        //
        // Flush the entire TB to remove any KSEG translations that may map the
        // pages being removed.  Otherwise hardware or software speculation
        // can reference the memory speculatively which would crash the machine.
        //

        KeFlushEntireTb (TRUE, TRUE);

        //
        // Establish an uncached mapping to the pages being removed.
        //

        MapSizeInBytes = SizeInBytes;

        //
        // Initializing VirtualAddress is not needed for correctness
        // but without it the compiler cannot compile this code
        // W4 to check for use of uninitialized variables.
        //

        VirtualAddress = NULL;

        while (MapSizeInBytes > PAGE_SIZE) {

            VirtualAddress = MmMapIoSpace (*StartAddress,
                                           MapSizeInBytes,
                                           MmNonCached);

            if (VirtualAddress != NULL) {
                break;
            }

            MapSizeInBytes = MapSizeInBytes >> 1;
        }

        if (MapSizeInBytes <= PAGE_SIZE) {

            StartPage = (PFN_NUMBER)(StartAddress->QuadPart >> PAGE_SHIFT);

            NumberOfPages = (PFN_COUNT)(NumberOfBytes->QuadPart >> PAGE_SHIFT);

            for (i = 0; i < NumberOfPages; i += 1) {

                SingleVirtualAddress = (PMMPTE)MiMapSinglePage (SingleVirtualAddress,
                                                                StartPage,
                                                                MmCached,
                                                                HighPagePriority);

                KeSweepCacheRangeWithDrain (TRUE,
                                            SingleVirtualAddress,
                                            PAGE_SIZE);

                StartPage += 1;
            }
        }
        else {

            //
            // Drain all pending transactions and prefetches and perform cache
            // evictions.  Only drain 4gb max at a time as this API takes a
            // ULONG.
            //

            while (SizeInBytes > _4gb) {
                KeSweepCacheRangeWithDrain (TRUE, VirtualAddress, _4gb - 1);
                SizeInBytes -= (_4gb - 1);
            }

            KeSweepCacheRangeWithDrain (TRUE,
                                        VirtualAddress,
                                        (ULONG)SizeInBytes);

            MmUnmapIoSpace (VirtualAddress, NumberOfBytes->QuadPart);
        }
#endif
    }

#if defined(_IA64_)
    MiUnmapSinglePage (SingleVirtualAddress);
#endif

    return Status;
}

NTSTATUS
MmMarkPhysicalMemoryAsBad (
    IN PPHYSICAL_ADDRESS StartAddress,
    IN OUT PLARGE_INTEGER NumberOfBytes
    )

/*++

Routine Description:

    This routine attempts to mark the specified physical address range
    as bad so the system will not use it.  This is generally done for pages
    which contain ECC errors.

    Note that this is different from removing pages permanently (ie: physically
    removing the memory board) which should be done via the
    MmRemovePhysicalMemory API.

    The caller is responsible for maintaining a global table so that subsequent
    boots can examine it and remove the ECC pages before loading the kernel.

Arguments:

    StartAddress  - Supplies the starting physical address.

    NumberOfBytes  - Supplies a pointer to the number of bytes being removed.

Return Value:

    NTSTATUS.

Environment:

    Kernel mode.  PASSIVE level.  No locks held.

--*/

{
    PAGED_CODE();

    return MiRemovePhysicalMemory (StartAddress, NumberOfBytes, FALSE, 0);
}

NTSTATUS
MmMarkPhysicalMemoryAsGood (
    IN PPHYSICAL_ADDRESS StartAddress,
    IN OUT PLARGE_INTEGER NumberOfBytes
    )

/*++

Routine Description:

    This routine attempts to mark the specified physical address range
    as good so the system will use it.  This is generally done for pages
    which used to (but presumably no longer do) contain ECC errors.

    Note that this is different from adding pages permanently (ie: physically
    inserting a new memory board) which should be done via the
    MmAddPhysicalMemory API.

    The caller is responsible for removing these entries from a global table
    so that subsequent boots will use the pages.

Arguments:

    StartAddress  - Supplies the starting physical address.

    NumberOfBytes  - Supplies a pointer to the number of bytes being removed.

Return Value:

    NTSTATUS.

Environment:

    Kernel mode.  PASSIVE level.  No locks held.

--*/

{
    PMMPFN Pfn1;
    KIRQL OldIrql;
    PFN_NUMBER NumberOfPages;
    PFN_NUMBER start;
    PFN_NUMBER count;
    PFN_NUMBER StartPage;
    PFN_NUMBER EndPage;
    PFN_NUMBER PageFrameIndex;
    PFN_NUMBER Page;
    PFN_NUMBER LastPage;

    ASSERT (KeGetCurrentIrql() == PASSIVE_LEVEL);

    ASSERT (BYTE_OFFSET(NumberOfBytes->LowPart) == 0);
    ASSERT (BYTE_OFFSET(StartAddress->LowPart) == 0);

    StartPage = (PFN_NUMBER)(StartAddress->QuadPart >> PAGE_SHIFT);
    NumberOfPages = (PFN_NUMBER)(NumberOfBytes->QuadPart >> PAGE_SHIFT);

    EndPage = StartPage + NumberOfPages;

    KeAcquireGuardedMutex (&MmDynamicMemoryMutex);

    if (EndPage - 1 > MmHighestPhysicalPage) {

        //
        // Truncate the request into something that can be mapped by the PFN
        // database.
        //

        EndPage = MmHighestPhysicalPage + 1;
        NumberOfPages = EndPage - StartPage;
    }

    //
    // The range cannot wrap.
    //

    if (StartPage >= EndPage) {
        KeReleaseGuardedMutex (&MmDynamicMemoryMutex);
        return STATUS_INVALID_PARAMETER_1;
    }

    //
    // The request must lie within an already present range.
    //

    start = 0;

    MmLockPagableSectionByHandle (ExPageLockHandle);

    LOCK_PFN (OldIrql);

    do {

        count = MmPhysicalMemoryBlock->Run[start].PageCount;
        Page = MmPhysicalMemoryBlock->Run[start].BasePage;

        if (count != 0) {

            LastPage = Page + count;

            if ((StartPage >= Page) && (EndPage <= LastPage)) {
                break;
            }
        }

        start += 1;

    } while (start != MmPhysicalMemoryBlock->NumberOfRuns);

    if (start == MmPhysicalMemoryBlock->NumberOfRuns) {
        UNLOCK_PFN (OldIrql);
        MmUnlockPagableImageSection(ExPageLockHandle);
        KeReleaseGuardedMutex (&MmDynamicMemoryMutex);
        return STATUS_CONFLICTING_ADDRESSES;
    }

    //
    // Walk through the range and add only pages previously removed to the
    // free list in the PFN database.
    //

    PageFrameIndex = StartPage;
    Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
    NumberOfPages = 0;

    while (PageFrameIndex < EndPage) {

        if ((Pfn1->u3.e1.ParityError == 1) &&
            (Pfn1->u3.e1.RemovalRequested == 1) &&
            (Pfn1->u3.e1.PageLocation == BadPageList)) {

            Pfn1->u3.e1.ParityError = 0;
            Pfn1->u3.e1.RemovalRequested = 0;
            MiUnlinkPageFromList (Pfn1);
            MiInsertPageInFreeList (PageFrameIndex);
            NumberOfPages += 1;
        }

        Pfn1 += 1;
        PageFrameIndex += 1;
    }

    UNLOCK_PFN (OldIrql);

    MI_INCREMENT_RESIDENT_AVAILABLE (NumberOfPages, MM_RESAVAIL_FREE_HOTADD_ECC);

    //
    // Increase all commit limits to reflect the additional memory.
    //

    InterlockedExchangeAddSizeT (&MmTotalCommitLimitMaximum, NumberOfPages);

    InterlockedExchangeAddSizeT (&MmTotalCommitLimit, NumberOfPages);

    MmUnlockPagableImageSection(ExPageLockHandle);

    KeReleaseGuardedMutex (&MmDynamicMemoryMutex);

    //
    // Indicate number of bytes actually added to our caller.
    //

    NumberOfBytes->QuadPart = (ULONGLONG)NumberOfPages * PAGE_SIZE;

    return STATUS_SUCCESS;
}

PPHYSICAL_MEMORY_RANGE
MmGetPhysicalMemoryRanges (
    VOID
    )

/*++

Routine Description:

    This routine returns the virtual address of a nonpaged pool block which
    contains the physical memory ranges in the system.

    The returned block contains physical address and page count pairs.
    The last entry contains zero for both.

    The caller must understand that this block can change at any point before
    or after this snapshot.

    It is the caller's responsibility to free this block.

Arguments:

    None.

Return Value:

    NULL on failure.

Environment:

    Kernel mode.  PASSIVE level.  No locks held.

--*/

{
    ULONG i;
    KIRQL OldIrql;
    PPHYSICAL_MEMORY_RANGE p;
    PPHYSICAL_MEMORY_RANGE PhysicalMemoryBlock;

    ASSERT (KeGetCurrentIrql() == PASSIVE_LEVEL);

    KeAcquireGuardedMutex (&MmDynamicMemoryMutex);

    i = sizeof(PHYSICAL_MEMORY_RANGE) * (MmPhysicalMemoryBlock->NumberOfRuns + 1);

    PhysicalMemoryBlock = ExAllocatePoolWithTag (NonPagedPool,
                                                 i,
                                                 'hPmM');

    if (PhysicalMemoryBlock == NULL) {
        KeReleaseGuardedMutex (&MmDynamicMemoryMutex);
        return NULL;
    }

    p = PhysicalMemoryBlock;

    MmLockPagableSectionByHandle (ExPageLockHandle);

    LOCK_PFN (OldIrql);

    ASSERT (i == (sizeof(PHYSICAL_MEMORY_RANGE) * (MmPhysicalMemoryBlock->NumberOfRuns + 1)));

    for (i = 0; i < MmPhysicalMemoryBlock->NumberOfRuns; i += 1) {
        p->BaseAddress.QuadPart = (LONGLONG)MmPhysicalMemoryBlock->Run[i].BasePage * PAGE_SIZE;
        p->NumberOfBytes.QuadPart = (LONGLONG)MmPhysicalMemoryBlock->Run[i].PageCount * PAGE_SIZE;
        p += 1;
    }

    p->BaseAddress.QuadPart = 0;
    p->NumberOfBytes.QuadPart = 0;

    UNLOCK_PFN (OldIrql);

    MmUnlockPagableImageSection(ExPageLockHandle);

    KeReleaseGuardedMutex (&MmDynamicMemoryMutex);

    return PhysicalMemoryBlock;
}

PFN_COUNT
MiRemovePhysicalPages (
    IN PFN_NUMBER StartPage,
    IN PFN_NUMBER EndPage
    )

/*++

Routine Description:

    This routine searches the PFN database for free, zeroed or standby pages
    that are marked for removal.

Arguments:

    StartPage - Supplies the low physical frame number to remove.

    EndPage - Supplies the last physical frame number to remove.

Return Value:

    Returns the number of pages removed from the free, zeroed and standby lists.

Environment:

    Kernel mode, PFN lock held.  Since this routine is PAGELK, the caller is
    responsible for locking it down and unlocking it on return.

--*/

{
    PMMPFN Pfn1;
    PMMPFN Pfn2;
    PMMPFN PfnNextColored;
    PMMPFN PfnNextFlink;
    PMMPFN PfnLastColored;
    PFN_NUMBER Page;
    LOGICAL RemovePage;
    ULONG Color;
    PMMCOLOR_TABLES ColorHead;
    PFN_NUMBER MovedPage;
    MMLISTS MemoryList;
    PFN_NUMBER PageNextColored;
    PFN_NUMBER PageNextFlink;
    PFN_NUMBER PageLastColored;
    PFN_COUNT NumberOfPages;
    PMMPFNLIST ListHead;
    LOGICAL RescanNeeded;

    MM_PFN_LOCK_ASSERT();

    NumberOfPages = 0;

rescan:

    //
    // Grab all zeroed (and then free) pages first directly from the
    // colored lists to avoid multiple walks down these singly linked lists.
    // Handle transition pages last.
    //

    for (MemoryList = ZeroedPageList; MemoryList <= FreePageList; MemoryList += 1) {

        ListHead = MmPageLocationList[MemoryList];

        for (Color = 0; Color < MmSecondaryColors; Color += 1) {
            ColorHead = &MmFreePagesByColor[MemoryList][Color];

            MovedPage = (PFN_NUMBER) MM_EMPTY_LIST;

            while (ColorHead->Flink != MM_EMPTY_LIST) {

                Page = ColorHead->Flink;
    
                Pfn1 = MI_PFN_ELEMENT(Page);

                ASSERT ((MMLISTS)Pfn1->u3.e1.PageLocation == MemoryList);

                // 
                // The Flink and Blink must be nonzero here for the page
                // to be on the listhead.  Only code that scans the
                // MmPhysicalMemoryBlock has to check for the zero case.
                //

                ASSERT (Pfn1->u1.Flink != 0);
                ASSERT (Pfn1->u2.Blink != 0);

                //
                // See if the page is desired by the caller.
                //
                // Systems utilizing memory compression may have more
                // pages on the zero, free and standby lists than we
                // want to give out.  Explicitly check MmAvailablePages
                // instead (and recheck whenever the PFN lock is
                // released and reacquired).
                //

                if ((Pfn1->u3.e1.RemovalRequested == 1) &&
                    (MmAvailablePages != 0)) {

                    ASSERT (Pfn1->u3.e1.ReadInProgress == 0);
    
                    MiUnlinkFreeOrZeroedPage (Pfn1);
    
                    MiInsertPageInList (&MmBadPageListHead, Page);

                    NumberOfPages += 1;
                }
                else {

                    //
                    // Unwanted so put the page on the end of list.
                    // If first time, save pfn.
                    //

                    if (MovedPage == MM_EMPTY_LIST) {
                        MovedPage = Page;
                    }
                    else if (Page == MovedPage) {

                        //
                        // No more pages available in this colored chain.
                        //

                        break;
                    }

                    //
                    // If the colored chain has more than one entry then
                    // put this page on the end.
                    //

                    PageNextColored = (PFN_NUMBER)Pfn1->OriginalPte.u.Long;

                    if (PageNextColored == MM_EMPTY_LIST) {

                        //
                        // No more pages available in this colored chain.
                        //

                        break;
                    }

                    ASSERT (Pfn1->u1.Flink != 0);
                    ASSERT (Pfn1->u1.Flink != MM_EMPTY_LIST);
                    ASSERT (Pfn1->u4.PteFrame != MI_MAGIC_AWE_PTEFRAME);

                    PfnNextColored = MI_PFN_ELEMENT(PageNextColored);
                    ASSERT ((MMLISTS)PfnNextColored->u3.e1.PageLocation == MemoryList);
                    ASSERT (PfnNextColored->u4.PteFrame != MI_MAGIC_AWE_PTEFRAME);

                    //
                    // Adjust the free page list so Page
                    // follows PageNextFlink.
                    //

                    PageNextFlink = Pfn1->u1.Flink;
                    PfnNextFlink = MI_PFN_ELEMENT(PageNextFlink);

                    ASSERT ((MMLISTS)PfnNextFlink->u3.e1.PageLocation == MemoryList);
                    ASSERT (PfnNextFlink->u4.PteFrame != MI_MAGIC_AWE_PTEFRAME);

                    PfnLastColored = ColorHead->Blink;
                    ASSERT (PfnLastColored != (PMMPFN)MM_EMPTY_LIST);
                    ASSERT (PfnLastColored->OriginalPte.u.Long == MM_EMPTY_LIST);
                    ASSERT (PfnLastColored->u4.PteFrame != MI_MAGIC_AWE_PTEFRAME);
                    ASSERT (PfnLastColored->u2.Blink != MM_EMPTY_LIST);

                    ASSERT ((MMLISTS)PfnLastColored->u3.e1.PageLocation == MemoryList);
                    PageLastColored = MI_PFN_ELEMENT_TO_INDEX (PfnLastColored);

                    if (ListHead->Flink == Page) {

                        ASSERT (Pfn1->u2.Blink == MM_EMPTY_LIST);
                        ASSERT (ListHead->Blink != Page);

                        ListHead->Flink = PageNextFlink;

                        PfnNextFlink->u2.Blink = MM_EMPTY_LIST;
                    }
                    else {

                        ASSERT (Pfn1->u2.Blink != MM_EMPTY_LIST);
                        ASSERT ((MMLISTS)(MI_PFN_ELEMENT((MI_PFN_ELEMENT(Pfn1->u2.Blink)->u1.Flink)))->u4.PteFrame != MI_MAGIC_AWE_PTEFRAME);
                        ASSERT ((MMLISTS)(MI_PFN_ELEMENT((MI_PFN_ELEMENT(Pfn1->u2.Blink)->u1.Flink)))->u3.e1.PageLocation == MemoryList);

                        MI_PFN_ELEMENT(Pfn1->u2.Blink)->u1.Flink = PageNextFlink;
                        PfnNextFlink->u2.Blink = Pfn1->u2.Blink;
                    }

#if DBG
                    if (PfnLastColored->u1.Flink == MM_EMPTY_LIST) {
                        ASSERT (ListHead->Blink == PageLastColored);
                    }
#endif

                    Pfn1->u1.Flink = PfnLastColored->u1.Flink;
                    Pfn1->u2.Blink = PageLastColored;

                    if (ListHead->Blink == PageLastColored) {
                        ListHead->Blink = Page;
                    }

                    //
                    // Adjust the colored chains.
                    //

                    if (PfnLastColored->u1.Flink != MM_EMPTY_LIST) {
                        ASSERT (MI_PFN_ELEMENT(PfnLastColored->u1.Flink)->u4.PteFrame != MI_MAGIC_AWE_PTEFRAME);
                        ASSERT ((MMLISTS)(MI_PFN_ELEMENT(PfnLastColored->u1.Flink)->u3.e1.PageLocation) == MemoryList);
                        MI_PFN_ELEMENT(PfnLastColored->u1.Flink)->u2.Blink = Page;
                    }

                    ASSERT (PfnLastColored->OriginalPte.u.Long == MM_EMPTY_LIST);
                    PfnLastColored->u1.Flink = Page;

                    ColorHead->Flink = PageNextColored;
                    PfnNextColored->u4.PteFrame = MM_EMPTY_LIST;

                    Pfn1->OriginalPte.u.Long = MM_EMPTY_LIST;
                    Pfn1->u4.PteFrame = PageLastColored;

                    PfnLastColored->OriginalPte.u.Long = Page;
                    ColorHead->Blink = Pfn1;
                }
            }
        }
    }

    RescanNeeded = FALSE;
    Pfn1 = MI_PFN_ELEMENT (StartPage);

    do {

        if ((Pfn1->u3.e1.PageLocation == StandbyPageList) &&
            (Pfn1->u1.Flink != 0) &&
            (Pfn1->u2.Blink != 0) &&
            (Pfn1->u3.e2.ReferenceCount == 0) &&
            (MmAvailablePages != 0)) {

            //
            // Systems utilizing memory compression may have more
            // pages on the zero, free and standby lists than we
            // want to give out.  Explicitly check MmAvailablePages
            // above instead (and recheck whenever the PFN lock is
            // released and reacquired).
            //

            ASSERT (Pfn1->u3.e1.ReadInProgress == 0);

            RemovePage = TRUE;

            if (Pfn1->u3.e1.RemovalRequested == 0) {

                //
                // This page is not directly needed for a hot remove - but if
                // it contains a chunk of prototype PTEs (and this chunk is
                // in a page that needs to be removed), then any pages
                // referenced by transition prototype PTEs must also be removed
                // before the desired page can be removed.
                //
                // The same analogy holds for page table, directory, parent
                // and extended parent pages.
                //

                Pfn2 = MI_PFN_ELEMENT (Pfn1->u4.PteFrame);
                if (Pfn2->u3.e1.RemovalRequested == 0) {
#if (_MI_PAGING_LEVELS >= 3)
                    Pfn2 = MI_PFN_ELEMENT (Pfn2->u4.PteFrame);
                    if (Pfn2->u3.e1.RemovalRequested == 0) {
                        RemovePage = FALSE;
                    }
                    else if (Pfn2->u2.ShareCount == 1) {
                        RescanNeeded = TRUE;
                    }
#if (_MI_PAGING_LEVELS >= 4)
                    Pfn2 = MI_PFN_ELEMENT (Pfn2->u4.PteFrame);
                    if (Pfn2->u3.e1.RemovalRequested == 0) {
                        RemovePage = FALSE;
                    }
                    else if (Pfn2->u2.ShareCount == 1) {
                        RescanNeeded = TRUE;
                    }
#endif
#else
                    RemovePage = FALSE;
#endif
                }
                else if (Pfn2->u2.ShareCount == 1) {
                    RescanNeeded = TRUE;
                }
            }
    
            if (RemovePage == TRUE) {

                //
                // This page is in the desired range - grab it.
                //
    
                MiUnlinkPageFromList (Pfn1);
                MiRestoreTransitionPte (Pfn1);
                MiInsertPageInList (&MmBadPageListHead, StartPage);
                NumberOfPages += 1;
            }
        }

        StartPage += 1;
        Pfn1 += 1;

    } while (StartPage < EndPage);

    if (RescanNeeded == TRUE) {

        //
        // A page table, directory or parent was freed by removing a transition
        // page from the cache.  Rescan from the top to pick it up.
        //

#if DBG
        MiDynmemData[7] += 1;
#endif

        goto rescan;
    }
#if DBG
    else {
        MiDynmemData[8] += 1;
    }
#endif

    return NumberOfPages;
}