mirror of https://github.com/tongzx/nt5src
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
2278 lines
62 KiB
2278 lines
62 KiB
/*++
|
|
|
|
Copyright (c) 1999 Microsoft Corporation
|
|
|
|
Module Name:
|
|
|
|
dynmem.c
|
|
|
|
Abstract:
|
|
|
|
This module contains the routines which implement dynamically adding
|
|
and removing physical memory from the system.
|
|
|
|
Author:
|
|
|
|
Landy Wang (landyw) 05-Feb-1999
|
|
|
|
Revision History:
|
|
|
|
--*/
|
|
|
|
#include "mi.h"
|
|
|
|
FAST_MUTEX MmDynamicMemoryMutex;
|
|
|
|
LOGICAL MiTrimRemovalPagesOnly = FALSE;
|
|
|
|
#if DBG
|
|
ULONG MiShowStuckPages;
|
|
ULONG MiDynmemData[9];
|
|
#endif
|
|
|
|
#if defined (_MI_COMPRESSION)
|
|
extern PMM_SET_COMPRESSION_THRESHOLD MiSetCompressionThreshold;
|
|
#endif
|
|
|
|
//
|
|
// Leave the low 3 bits clear as this will be inserted into the PFN PteAddress.
|
|
//
|
|
|
|
#define PFN_REMOVED ((PMMPTE)(INT_PTR)(int)0x99887768)
|
|
|
|
PFN_COUNT
|
|
MiRemovePhysicalPages (
|
|
IN PFN_NUMBER StartPage,
|
|
IN PFN_NUMBER EndPage
|
|
);
|
|
|
|
NTSTATUS
|
|
MiRemovePhysicalMemory (
|
|
IN PPHYSICAL_ADDRESS StartAddress,
|
|
IN OUT PLARGE_INTEGER NumberOfBytes,
|
|
IN LOGICAL PermanentRemoval,
|
|
IN ULONG Flags
|
|
);
|
|
|
|
#ifdef ALLOC_PRAGMA
|
|
#pragma alloc_text(PAGE,MmRemovePhysicalMemory)
|
|
#pragma alloc_text(PAGE,MmMarkPhysicalMemoryAsBad)
|
|
#pragma alloc_text(PAGELK,MmAddPhysicalMemory)
|
|
#pragma alloc_text(PAGELK,MmAddPhysicalMemoryEx)
|
|
#pragma alloc_text(PAGELK,MiRemovePhysicalMemory)
|
|
#pragma alloc_text(PAGELK,MmMarkPhysicalMemoryAsGood)
|
|
#pragma alloc_text(PAGELK,MmGetPhysicalMemoryRanges)
|
|
#pragma alloc_text(PAGELK,MiRemovePhysicalPages)
|
|
#endif
|
|
|
|
|
|
NTSTATUS
|
|
MmAddPhysicalMemory (
|
|
IN PPHYSICAL_ADDRESS StartAddress,
|
|
IN OUT PLARGE_INTEGER NumberOfBytes
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
A wrapper for MmAddPhysicalMemoryEx.
|
|
|
|
Arguments:
|
|
|
|
StartAddress - Supplies the starting physical address.
|
|
|
|
NumberOfBytes - Supplies a pointer to the number of bytes being added.
|
|
If any bytes were added (ie: STATUS_SUCCESS is being
|
|
returned), the actual amount is returned here.
|
|
|
|
Return Value:
|
|
|
|
NTSTATUS.
|
|
|
|
Environment:
|
|
|
|
Kernel mode. PASSIVE level. No locks held.
|
|
|
|
--*/
|
|
|
|
{
|
|
return MmAddPhysicalMemoryEx (StartAddress, NumberOfBytes, 0);
|
|
}
|
|
|
|
|
|
NTSTATUS
|
|
MmAddPhysicalMemoryEx (
|
|
IN PPHYSICAL_ADDRESS StartAddress,
|
|
IN OUT PLARGE_INTEGER NumberOfBytes,
|
|
IN ULONG Flags
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine adds the specified physical address range to the system.
|
|
This includes initializing PFN database entries and adding it to the
|
|
freelists.
|
|
|
|
Arguments:
|
|
|
|
StartAddress - Supplies the starting physical address.
|
|
|
|
NumberOfBytes - Supplies a pointer to the number of bytes being added.
|
|
If any bytes were added (ie: STATUS_SUCCESS is being
|
|
returned), the actual amount is returned here.
|
|
|
|
Flags - Supplies relevant flags describing the memory range.
|
|
|
|
Return Value:
|
|
|
|
NTSTATUS.
|
|
|
|
Environment:
|
|
|
|
Kernel mode. PASSIVE level. No locks held.
|
|
|
|
--*/
|
|
|
|
{
|
|
ULONG i;
|
|
PMMPFN Pfn1;
|
|
KIRQL OldIrql;
|
|
LOGICAL Inserted;
|
|
LOGICAL Updated;
|
|
MMPTE TempPte;
|
|
PMMPTE PointerPte;
|
|
PMMPTE LastPte;
|
|
PFN_NUMBER NumberOfPages;
|
|
PFN_NUMBER start;
|
|
PFN_NUMBER count;
|
|
PFN_NUMBER StartPage;
|
|
PFN_NUMBER EndPage;
|
|
PFN_NUMBER PageFrameIndex;
|
|
PFN_NUMBER Page;
|
|
PFN_NUMBER LastPage;
|
|
PFN_NUMBER TotalPagesAllowed;
|
|
PFN_COUNT PagesNeeded;
|
|
PPHYSICAL_MEMORY_DESCRIPTOR OldPhysicalMemoryBlock;
|
|
PPHYSICAL_MEMORY_DESCRIPTOR NewPhysicalMemoryBlock;
|
|
PPHYSICAL_MEMORY_RUN NewRun;
|
|
LOGICAL PfnDatabaseIsPhysical;
|
|
|
|
ASSERT (KeGetCurrentIrql() == PASSIVE_LEVEL);
|
|
|
|
#ifdef _MI_MESSAGE_SERVER
|
|
if (NumberOfBytes->LowPart & 0x1) {
|
|
MI_INSTRUMENT_QUEUE((PVOID)StartAddress);
|
|
return STATUS_NOT_SUPPORTED;
|
|
}
|
|
else if (NumberOfBytes->LowPart & 0x2) {
|
|
#if defined(_WIN64)
|
|
StartAddress->QuadPart = (ULONG_PTR)(MI_INSTRUMENTR_QUEUE());
|
|
#else
|
|
StartAddress->LowPart = PtrToUlong(MI_INSTRUMENTR_QUEUE());
|
|
#endif
|
|
return STATUS_NOT_SUPPORTED;
|
|
}
|
|
#endif
|
|
|
|
if (BYTE_OFFSET(StartAddress->LowPart) != 0) {
|
|
return STATUS_INVALID_PARAMETER_1;
|
|
}
|
|
|
|
if (BYTE_OFFSET(NumberOfBytes->LowPart) != 0) {
|
|
return STATUS_INVALID_PARAMETER_2;
|
|
}
|
|
|
|
#if defined (_MI_COMPRESSION)
|
|
if (Flags & ~MM_PHYSICAL_MEMORY_PRODUCED_VIA_COMPRESSION) {
|
|
return STATUS_INVALID_PARAMETER_3;
|
|
}
|
|
#else
|
|
if (Flags != 0) {
|
|
return STATUS_INVALID_PARAMETER_3;
|
|
}
|
|
#endif
|
|
|
|
//
|
|
// The system must be configured for dynamic memory addition. This is
|
|
// critical as only then is the database guaranteed to be non-sparse.
|
|
//
|
|
|
|
if (MmDynamicPfn == 0) {
|
|
return STATUS_NOT_SUPPORTED;
|
|
}
|
|
|
|
if (MI_IS_PHYSICAL_ADDRESS(MmPfnDatabase)) {
|
|
PfnDatabaseIsPhysical = TRUE;
|
|
}
|
|
else {
|
|
PfnDatabaseIsPhysical = FALSE;
|
|
}
|
|
|
|
StartPage = (PFN_NUMBER)(StartAddress->QuadPart >> PAGE_SHIFT);
|
|
NumberOfPages = (PFN_NUMBER)(NumberOfBytes->QuadPart >> PAGE_SHIFT);
|
|
|
|
EndPage = StartPage + NumberOfPages;
|
|
|
|
if (EndPage - 1 > MmHighestPossiblePhysicalPage) {
|
|
|
|
//
|
|
// Truncate the request into something that can be mapped by the PFN
|
|
// database.
|
|
//
|
|
|
|
EndPage = MmHighestPossiblePhysicalPage + 1;
|
|
NumberOfPages = EndPage - StartPage;
|
|
}
|
|
|
|
//
|
|
// Ensure that the memory being added does not exceed the license
|
|
// restrictions.
|
|
//
|
|
|
|
if (ExVerifySuite(DataCenter) == TRUE) {
|
|
TotalPagesAllowed = MI_DTC_MAX_PAGES;
|
|
}
|
|
else if ((MmProductType != 0x00690057) &&
|
|
(ExVerifySuite(Enterprise) == TRUE)) {
|
|
|
|
TotalPagesAllowed = MI_ADS_MAX_PAGES;
|
|
}
|
|
else {
|
|
TotalPagesAllowed = MI_DEFAULT_MAX_PAGES;
|
|
}
|
|
|
|
if (MmNumberOfPhysicalPages + NumberOfPages > TotalPagesAllowed) {
|
|
|
|
//
|
|
// Truncate the request appropriately.
|
|
//
|
|
|
|
NumberOfPages = TotalPagesAllowed - MmNumberOfPhysicalPages;
|
|
EndPage = StartPage + NumberOfPages;
|
|
}
|
|
|
|
//
|
|
// The range cannot wrap.
|
|
//
|
|
|
|
if (StartPage >= EndPage) {
|
|
return STATUS_INVALID_PARAMETER_1;
|
|
}
|
|
|
|
ExAcquireFastMutex (&MmDynamicMemoryMutex);
|
|
|
|
OldPhysicalMemoryBlock = MmPhysicalMemoryBlock;
|
|
|
|
i = (sizeof(PHYSICAL_MEMORY_DESCRIPTOR) +
|
|
(sizeof(PHYSICAL_MEMORY_RUN) * (MmPhysicalMemoryBlock->NumberOfRuns + 1)));
|
|
|
|
NewPhysicalMemoryBlock = ExAllocatePoolWithTag (NonPagedPool,
|
|
i,
|
|
' mM');
|
|
|
|
if (NewPhysicalMemoryBlock == NULL) {
|
|
ExReleaseFastMutex (&MmDynamicMemoryMutex);
|
|
return STATUS_INSUFFICIENT_RESOURCES;
|
|
}
|
|
|
|
//
|
|
// The range cannot overlap any ranges that are already present.
|
|
//
|
|
|
|
start = 0;
|
|
|
|
MmLockPagableSectionByHandle (ExPageLockHandle);
|
|
|
|
LOCK_PFN (OldIrql);
|
|
|
|
#if defined (_MI_COMPRESSION)
|
|
|
|
//
|
|
// Adding compression-generated ranges can only be done if the hardware
|
|
// has already successfully announced itself.
|
|
//
|
|
|
|
if (Flags & MM_PHYSICAL_MEMORY_PRODUCED_VIA_COMPRESSION) {
|
|
if (MiSetCompressionThreshold == NULL) {
|
|
UNLOCK_PFN (OldIrql);
|
|
MmUnlockPagableImageSection(ExPageLockHandle);
|
|
ExReleaseFastMutex (&MmDynamicMemoryMutex);
|
|
ExFreePool (NewPhysicalMemoryBlock);
|
|
return STATUS_NOT_SUPPORTED;
|
|
}
|
|
}
|
|
#endif
|
|
|
|
do {
|
|
|
|
count = MmPhysicalMemoryBlock->Run[start].PageCount;
|
|
Page = MmPhysicalMemoryBlock->Run[start].BasePage;
|
|
|
|
if (count != 0) {
|
|
|
|
LastPage = Page + count;
|
|
|
|
if ((StartPage < Page) && (EndPage > Page)) {
|
|
UNLOCK_PFN (OldIrql);
|
|
MmUnlockPagableImageSection(ExPageLockHandle);
|
|
ExReleaseFastMutex (&MmDynamicMemoryMutex);
|
|
ExFreePool (NewPhysicalMemoryBlock);
|
|
return STATUS_CONFLICTING_ADDRESSES;
|
|
}
|
|
|
|
if ((StartPage >= Page) && (StartPage < LastPage)) {
|
|
UNLOCK_PFN (OldIrql);
|
|
MmUnlockPagableImageSection(ExPageLockHandle);
|
|
ExReleaseFastMutex (&MmDynamicMemoryMutex);
|
|
ExFreePool (NewPhysicalMemoryBlock);
|
|
return STATUS_CONFLICTING_ADDRESSES;
|
|
}
|
|
}
|
|
|
|
start += 1;
|
|
|
|
} while (start != MmPhysicalMemoryBlock->NumberOfRuns);
|
|
|
|
//
|
|
// Fill any gaps in the (sparse) PFN database needed for these pages,
|
|
// unless the PFN database was physically allocated and completely
|
|
// committed up front.
|
|
//
|
|
|
|
PagesNeeded = 0;
|
|
|
|
if (PfnDatabaseIsPhysical == FALSE) {
|
|
PointerPte = MiGetPteAddress (MI_PFN_ELEMENT(StartPage));
|
|
LastPte = MiGetPteAddress ((PCHAR)(MI_PFN_ELEMENT(EndPage)) - 1);
|
|
|
|
while (PointerPte <= LastPte) {
|
|
if (PointerPte->u.Hard.Valid == 0) {
|
|
PagesNeeded += 1;
|
|
}
|
|
PointerPte += 1;
|
|
}
|
|
|
|
if (MmAvailablePages < PagesNeeded) {
|
|
UNLOCK_PFN (OldIrql);
|
|
MmUnlockPagableImageSection(ExPageLockHandle);
|
|
ExReleaseFastMutex (&MmDynamicMemoryMutex);
|
|
ExFreePool (NewPhysicalMemoryBlock);
|
|
return STATUS_INSUFFICIENT_RESOURCES;
|
|
}
|
|
|
|
TempPte = ValidKernelPte;
|
|
|
|
PointerPte = MiGetPteAddress (MI_PFN_ELEMENT(StartPage));
|
|
|
|
while (PointerPte <= LastPte) {
|
|
if (PointerPte->u.Hard.Valid == 0) {
|
|
|
|
PageFrameIndex = MiRemoveZeroPage(MI_GET_PAGE_COLOR_FROM_PTE (PointerPte));
|
|
|
|
MiInitializePfn (PageFrameIndex, PointerPte, 0);
|
|
|
|
TempPte.u.Hard.PageFrameNumber = PageFrameIndex;
|
|
*PointerPte = TempPte;
|
|
}
|
|
PointerPte += 1;
|
|
}
|
|
MmResidentAvailablePages -= PagesNeeded;
|
|
}
|
|
|
|
//
|
|
// If the new range is adjacent to an existing range, just merge it into
|
|
// the old block. Otherwise use the new block as a new entry will have to
|
|
// be used.
|
|
//
|
|
|
|
NewPhysicalMemoryBlock->NumberOfRuns = MmPhysicalMemoryBlock->NumberOfRuns + 1;
|
|
NewPhysicalMemoryBlock->NumberOfPages = MmPhysicalMemoryBlock->NumberOfPages + NumberOfPages;
|
|
|
|
NewRun = &NewPhysicalMemoryBlock->Run[0];
|
|
start = 0;
|
|
Inserted = FALSE;
|
|
Updated = FALSE;
|
|
|
|
do {
|
|
|
|
Page = MmPhysicalMemoryBlock->Run[start].BasePage;
|
|
count = MmPhysicalMemoryBlock->Run[start].PageCount;
|
|
|
|
if (Inserted == FALSE) {
|
|
|
|
//
|
|
// Note overlaps into adjacent ranges were already checked above.
|
|
//
|
|
|
|
if (StartPage == Page + count) {
|
|
MmPhysicalMemoryBlock->Run[start].PageCount += NumberOfPages;
|
|
OldPhysicalMemoryBlock = NewPhysicalMemoryBlock;
|
|
MmPhysicalMemoryBlock->NumberOfPages += NumberOfPages;
|
|
|
|
//
|
|
// Coalesce below and above to avoid leaving zero length gaps
|
|
// as these gaps would prevent callers from removing ranges
|
|
// the span them.
|
|
//
|
|
|
|
if (start + 1 < MmPhysicalMemoryBlock->NumberOfRuns) {
|
|
|
|
start += 1;
|
|
Page = MmPhysicalMemoryBlock->Run[start].BasePage;
|
|
count = MmPhysicalMemoryBlock->Run[start].PageCount;
|
|
|
|
if (StartPage + NumberOfPages == Page) {
|
|
MmPhysicalMemoryBlock->Run[start - 1].PageCount +=
|
|
count;
|
|
MmPhysicalMemoryBlock->NumberOfRuns -= 1;
|
|
|
|
//
|
|
// Copy any remaining entries.
|
|
//
|
|
|
|
if (start != MmPhysicalMemoryBlock->NumberOfRuns) {
|
|
RtlMoveMemory (&MmPhysicalMemoryBlock->Run[start],
|
|
&MmPhysicalMemoryBlock->Run[start + 1],
|
|
(MmPhysicalMemoryBlock->NumberOfRuns - start) * sizeof (PHYSICAL_MEMORY_RUN));
|
|
}
|
|
}
|
|
}
|
|
Updated = TRUE;
|
|
break;
|
|
}
|
|
|
|
if (StartPage + NumberOfPages == Page) {
|
|
MmPhysicalMemoryBlock->Run[start].BasePage = StartPage;
|
|
MmPhysicalMemoryBlock->Run[start].PageCount += NumberOfPages;
|
|
OldPhysicalMemoryBlock = NewPhysicalMemoryBlock;
|
|
MmPhysicalMemoryBlock->NumberOfPages += NumberOfPages;
|
|
Updated = TRUE;
|
|
break;
|
|
}
|
|
|
|
if (StartPage + NumberOfPages <= Page) {
|
|
|
|
if (start + 1 < MmPhysicalMemoryBlock->NumberOfRuns) {
|
|
|
|
if (StartPage + NumberOfPages <= MmPhysicalMemoryBlock->Run[start + 1].BasePage) {
|
|
//
|
|
// Don't insert here - the new entry really belongs
|
|
// (at least) one entry further down.
|
|
//
|
|
|
|
continue;
|
|
}
|
|
}
|
|
|
|
NewRun->BasePage = StartPage;
|
|
NewRun->PageCount = NumberOfPages;
|
|
NewRun += 1;
|
|
Inserted = TRUE;
|
|
Updated = TRUE;
|
|
}
|
|
}
|
|
|
|
*NewRun = MmPhysicalMemoryBlock->Run[start];
|
|
NewRun += 1;
|
|
|
|
start += 1;
|
|
|
|
} while (start != MmPhysicalMemoryBlock->NumberOfRuns);
|
|
|
|
//
|
|
// If the memory block has not been updated, then the new entry must
|
|
// be added at the very end.
|
|
//
|
|
|
|
if (Updated == FALSE) {
|
|
ASSERT (Inserted == FALSE);
|
|
NewRun->BasePage = StartPage;
|
|
NewRun->PageCount = NumberOfPages;
|
|
Inserted = TRUE;
|
|
}
|
|
|
|
//
|
|
// Repoint the MmPhysicalMemoryBlock at the new chunk, free the old after
|
|
// releasing the PFN lock.
|
|
//
|
|
|
|
if (Inserted == TRUE) {
|
|
OldPhysicalMemoryBlock = MmPhysicalMemoryBlock;
|
|
MmPhysicalMemoryBlock = NewPhysicalMemoryBlock;
|
|
}
|
|
|
|
//
|
|
// Note that the page directory (page parent entries on Win64) must be
|
|
// filled in at system boot so that already-created processes do not fault
|
|
// when referencing the new PFNs.
|
|
//
|
|
|
|
//
|
|
// Walk through the memory descriptors and add pages to the
|
|
// free list in the PFN database.
|
|
//
|
|
|
|
PageFrameIndex = StartPage;
|
|
Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
|
|
|
|
if (EndPage - 1 > MmHighestPhysicalPage) {
|
|
MmHighestPhysicalPage = EndPage - 1;
|
|
}
|
|
|
|
while (PageFrameIndex < EndPage) {
|
|
|
|
ASSERT (Pfn1->u2.ShareCount == 0);
|
|
ASSERT (Pfn1->u3.e2.ShortFlags == 0);
|
|
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
|
|
ASSERT64 (Pfn1->UsedPageTableEntries == 0);
|
|
ASSERT (Pfn1->OriginalPte.u.Long == ZeroKernelPte.u.Long);
|
|
ASSERT (Pfn1->u4.PteFrame == 0);
|
|
ASSERT ((Pfn1->PteAddress == PFN_REMOVED) ||
|
|
(Pfn1->PteAddress == (PMMPTE)(UINT_PTR)0));
|
|
|
|
//
|
|
// Initialize the color for NUMA purposes.
|
|
//
|
|
|
|
MiDetermineNode (PageFrameIndex, Pfn1);
|
|
|
|
//
|
|
// Set the PTE address to the physical page for
|
|
// virtual address alignment checking.
|
|
//
|
|
|
|
Pfn1->PteAddress = (PMMPTE)(PageFrameIndex << PTE_SHIFT);
|
|
|
|
MiInsertPageInFreeList (PageFrameIndex);
|
|
|
|
PageFrameIndex += 1;
|
|
|
|
Pfn1 += 1;
|
|
}
|
|
|
|
MmNumberOfPhysicalPages += (PFN_COUNT)NumberOfPages;
|
|
|
|
//
|
|
// Only non-compression ranges get to contribute to ResidentAvailable as
|
|
// adding compression ranges to this could crash the system.
|
|
//
|
|
// For the same reason, compression range additions also need to subtract
|
|
// from AvailablePages the amount the above MiInsertPageInFreeList added.
|
|
//
|
|
|
|
#if defined (_MI_COMPRESSION)
|
|
if (Flags & MM_PHYSICAL_MEMORY_PRODUCED_VIA_COMPRESSION) {
|
|
MmAvailablePages -= (PFN_COUNT) NumberOfPages;
|
|
MiNumberOfCompressionPages += NumberOfPages;
|
|
}
|
|
else {
|
|
MmResidentAvailablePages += NumberOfPages;
|
|
|
|
//
|
|
// Since real (noncompression-generated) physical memory was added,
|
|
// rearm the interrupt to occur at a higher threshold.
|
|
//
|
|
|
|
MiArmCompressionInterrupt ();
|
|
}
|
|
#else
|
|
MmResidentAvailablePages += NumberOfPages;
|
|
#endif
|
|
|
|
UNLOCK_PFN (OldIrql);
|
|
|
|
InterlockedExchangeAdd ((PLONG)&SharedUserData->NumberOfPhysicalPages,
|
|
(LONG) NumberOfPages);
|
|
|
|
//
|
|
// Carefully increase all commit limits to reflect the additional memory -
|
|
// notice the current usage must be done first so no one else cuts the
|
|
// line.
|
|
//
|
|
|
|
InterlockedExchangeAddSizeT (&MmTotalCommittedPages, PagesNeeded);
|
|
|
|
InterlockedExchangeAddSizeT (&MmTotalCommitLimitMaximum, NumberOfPages);
|
|
|
|
InterlockedExchangeAddSizeT (&MmTotalCommitLimit, NumberOfPages);
|
|
|
|
MmUnlockPagableImageSection(ExPageLockHandle);
|
|
|
|
ExReleaseFastMutex (&MmDynamicMemoryMutex);
|
|
|
|
ExFreePool (OldPhysicalMemoryBlock);
|
|
|
|
//
|
|
// Indicate number of bytes actually added to our caller.
|
|
//
|
|
|
|
NumberOfBytes->QuadPart = (ULONGLONG)NumberOfPages * PAGE_SIZE;
|
|
|
|
return STATUS_SUCCESS;
|
|
}
|
|
|
|
|
|
NTSTATUS
|
|
MiRemovePhysicalMemory (
|
|
IN PPHYSICAL_ADDRESS StartAddress,
|
|
IN OUT PLARGE_INTEGER NumberOfBytes,
|
|
IN LOGICAL PermanentRemoval,
|
|
IN ULONG Flags
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine attempts to remove the specified physical address range
|
|
from the system.
|
|
|
|
Arguments:
|
|
|
|
StartAddress - Supplies the starting physical address.
|
|
|
|
NumberOfBytes - Supplies a pointer to the number of bytes being removed.
|
|
|
|
PermanentRemoval - Supplies TRUE if the memory is being permanently
|
|
(ie: physically) removed. FALSE if not (ie: just a
|
|
bad page detected via ECC which is being marked
|
|
"don't-use".
|
|
|
|
Return Value:
|
|
|
|
NTSTATUS.
|
|
|
|
Environment:
|
|
|
|
Kernel mode. PASSIVE level. No locks held.
|
|
|
|
--*/
|
|
|
|
{
|
|
ULONG i;
|
|
ULONG Additional;
|
|
PFN_NUMBER Page;
|
|
PFN_NUMBER LastPage;
|
|
PFN_NUMBER OriginalLastPage;
|
|
PFN_NUMBER start;
|
|
PFN_NUMBER PagesReleased;
|
|
PMMPFN Pfn1;
|
|
PMMPFN StartPfn;
|
|
PMMPFN EndPfn;
|
|
KIRQL OldIrql;
|
|
PFN_NUMBER StartPage;
|
|
PFN_NUMBER EndPage;
|
|
PFN_COUNT NumberOfPages;
|
|
PFN_COUNT ParityPages;
|
|
SPFN_NUMBER MaxPages;
|
|
PFN_NUMBER PageFrameIndex;
|
|
PFN_NUMBER RemovedPages;
|
|
PFN_NUMBER RemovedPagesThisPass;
|
|
LOGICAL Inserted;
|
|
NTSTATUS Status;
|
|
PMMPTE PointerPte;
|
|
PMMPTE EndPte;
|
|
PVOID VirtualAddress;
|
|
PPHYSICAL_MEMORY_DESCRIPTOR OldPhysicalMemoryBlock;
|
|
PPHYSICAL_MEMORY_DESCRIPTOR NewPhysicalMemoryBlock;
|
|
PPHYSICAL_MEMORY_RUN NewRun;
|
|
LOGICAL PfnDatabaseIsPhysical;
|
|
PFN_NUMBER HighestPossiblePhysicalPage;
|
|
PFN_COUNT FluidPages;
|
|
|
|
ASSERT (KeGetCurrentIrql() == PASSIVE_LEVEL);
|
|
|
|
ASSERT (BYTE_OFFSET(NumberOfBytes->LowPart) == 0);
|
|
ASSERT (BYTE_OFFSET(StartAddress->LowPart) == 0);
|
|
|
|
if (MI_IS_PHYSICAL_ADDRESS(MmPfnDatabase)) {
|
|
|
|
if (PermanentRemoval == TRUE) {
|
|
|
|
//
|
|
// The system must be configured for dynamic memory addition. This
|
|
// is not strictly required to remove the memory, but it's better
|
|
// to check for it now under the assumption that the administrator
|
|
// is probably going to want to add this range of memory back in -
|
|
// better to give the error now and refuse the removal than to
|
|
// refuse the addition later.
|
|
//
|
|
|
|
if (MmDynamicPfn == 0) {
|
|
return STATUS_NOT_SUPPORTED;
|
|
}
|
|
}
|
|
|
|
PfnDatabaseIsPhysical = TRUE;
|
|
}
|
|
else {
|
|
PfnDatabaseIsPhysical = FALSE;
|
|
}
|
|
|
|
if (PermanentRemoval == TRUE) {
|
|
HighestPossiblePhysicalPage = MmHighestPossiblePhysicalPage;
|
|
FluidPages = 100;
|
|
}
|
|
else {
|
|
HighestPossiblePhysicalPage = MmHighestPhysicalPage;
|
|
FluidPages = 0;
|
|
}
|
|
|
|
StartPage = (PFN_NUMBER)(StartAddress->QuadPart >> PAGE_SHIFT);
|
|
NumberOfPages = (PFN_COUNT)(NumberOfBytes->QuadPart >> PAGE_SHIFT);
|
|
|
|
EndPage = StartPage + NumberOfPages;
|
|
|
|
if (EndPage - 1 > HighestPossiblePhysicalPage) {
|
|
|
|
//
|
|
// Truncate the request into something that can be mapped by the PFN
|
|
// database.
|
|
//
|
|
|
|
EndPage = MmHighestPossiblePhysicalPage + 1;
|
|
NumberOfPages = (PFN_COUNT)(EndPage - StartPage);
|
|
}
|
|
|
|
//
|
|
// The range cannot wrap.
|
|
//
|
|
|
|
if (StartPage >= EndPage) {
|
|
return STATUS_INVALID_PARAMETER_1;
|
|
}
|
|
|
|
#if !defined (_MI_COMPRESSION)
|
|
if (Flags != 0) {
|
|
return STATUS_INVALID_PARAMETER_4;
|
|
}
|
|
#endif
|
|
|
|
StartPfn = MI_PFN_ELEMENT (StartPage);
|
|
EndPfn = MI_PFN_ELEMENT (EndPage);
|
|
|
|
ExAcquireFastMutex (&MmDynamicMemoryMutex);
|
|
|
|
#if DBG
|
|
MiDynmemData[0] += 1;
|
|
#endif
|
|
|
|
//
|
|
// Attempt to decrease all commit limits to reflect the removed memory.
|
|
//
|
|
|
|
if (MiChargeTemporaryCommitmentForReduction (NumberOfPages + FluidPages) == FALSE) {
|
|
#if DBG
|
|
MiDynmemData[1] += 1;
|
|
#endif
|
|
ExReleaseFastMutex (&MmDynamicMemoryMutex);
|
|
return STATUS_INSUFFICIENT_RESOURCES;
|
|
}
|
|
|
|
//
|
|
// Reduce the systemwide commit limit - note this is carefully done
|
|
// *PRIOR* to returning this commitment so no one else (including a DPC
|
|
// in this very thread) can consume past the limit.
|
|
//
|
|
|
|
InterlockedExchangeAddSizeT (&MmTotalCommitLimit, 0 - NumberOfPages);
|
|
|
|
InterlockedExchangeAddSizeT (&MmTotalCommitLimitMaximum, 0 - NumberOfPages);
|
|
|
|
//
|
|
// Now that the systemwide commit limit has been lowered, the amount
|
|
// we have removed can be safely returned.
|
|
//
|
|
|
|
MiReturnCommitment (NumberOfPages + FluidPages);
|
|
|
|
MmLockPagableSectionByHandle (ExPageLockHandle);
|
|
|
|
//
|
|
// Check for outstanding promises that cannot be broken.
|
|
//
|
|
|
|
LOCK_PFN (OldIrql);
|
|
|
|
if (PermanentRemoval == FALSE) {
|
|
|
|
//
|
|
// If it's just the removal of ECC-marked bad pages, then don't
|
|
// allow the caller to remove any pages that have already been
|
|
// ECC-removed. This is to prevent recursive erroneous charges.
|
|
//
|
|
|
|
for (Pfn1 = StartPfn; Pfn1 < EndPfn; Pfn1 += 1) {
|
|
if (Pfn1->u3.e1.ParityError == 1) {
|
|
UNLOCK_PFN (OldIrql);
|
|
Status = STATUS_INVALID_PARAMETER_2;
|
|
goto giveup2;
|
|
}
|
|
}
|
|
}
|
|
|
|
MaxPages = MI_NONPAGABLE_MEMORY_AVAILABLE() - FluidPages;
|
|
|
|
if ((SPFN_NUMBER)NumberOfPages > MaxPages) {
|
|
#if DBG
|
|
MiDynmemData[2] += 1;
|
|
#endif
|
|
UNLOCK_PFN (OldIrql);
|
|
Status = STATUS_INSUFFICIENT_RESOURCES;
|
|
goto giveup2;
|
|
}
|
|
|
|
//
|
|
// The range must be contained in a single entry. It is
|
|
// permissible for it to be part of a single entry, but it
|
|
// must not cross multiple entries.
|
|
//
|
|
|
|
Additional = (ULONG)-2;
|
|
|
|
start = 0;
|
|
do {
|
|
|
|
Page = MmPhysicalMemoryBlock->Run[start].BasePage;
|
|
LastPage = Page + MmPhysicalMemoryBlock->Run[start].PageCount;
|
|
|
|
if ((StartPage >= Page) && (EndPage <= LastPage)) {
|
|
if ((StartPage == Page) && (EndPage == LastPage)) {
|
|
Additional = (ULONG)-1;
|
|
}
|
|
else if ((StartPage == Page) || (EndPage == LastPage)) {
|
|
Additional = 0;
|
|
}
|
|
else {
|
|
Additional = 1;
|
|
}
|
|
break;
|
|
}
|
|
|
|
start += 1;
|
|
|
|
} while (start != MmPhysicalMemoryBlock->NumberOfRuns);
|
|
|
|
if (Additional == (ULONG)-2) {
|
|
#if DBG
|
|
MiDynmemData[3] += 1;
|
|
#endif
|
|
UNLOCK_PFN (OldIrql);
|
|
Status = STATUS_CONFLICTING_ADDRESSES;
|
|
goto giveup2;
|
|
}
|
|
|
|
for (Pfn1 = StartPfn; Pfn1 < EndPfn; Pfn1 += 1) {
|
|
Pfn1->u3.e1.RemovalRequested = 1;
|
|
}
|
|
|
|
if (PermanentRemoval == TRUE) {
|
|
MmNumberOfPhysicalPages -= NumberOfPages;
|
|
|
|
InterlockedExchangeAdd ((PLONG)&SharedUserData->NumberOfPhysicalPages,
|
|
0 - NumberOfPages);
|
|
}
|
|
|
|
#if defined (_MI_COMPRESSION)
|
|
|
|
//
|
|
// Only removal of non-compression ranges decrement ResidentAvailable as
|
|
// only those ranges actually incremented this when they were added.
|
|
//
|
|
|
|
if ((Flags & MM_PHYSICAL_MEMORY_PRODUCED_VIA_COMPRESSION) == 0) {
|
|
MmResidentAvailablePages -= NumberOfPages;
|
|
|
|
//
|
|
// Since real (noncompression-generated) physical memory is being
|
|
// removed, rearm the interrupt to occur at a lower threshold.
|
|
//
|
|
|
|
if (PermanentRemoval == TRUE) {
|
|
MiArmCompressionInterrupt ();
|
|
}
|
|
}
|
|
#else
|
|
MmResidentAvailablePages -= NumberOfPages;
|
|
#endif
|
|
|
|
//
|
|
// The free and zero lists must be pruned now before releasing the PFN
|
|
// lock otherwise if another thread allocates the page from these lists,
|
|
// the allocation will clear the RemovalRequested flag forever.
|
|
//
|
|
|
|
RemovedPages = MiRemovePhysicalPages (StartPage, EndPage);
|
|
|
|
#if defined (_MI_COMPRESSION)
|
|
|
|
//
|
|
// Compression range removals add back into AvailablePages the same
|
|
// amount that MiUnlinkPageFromList removes (as the original addition
|
|
// of these ranges never bumps this counter).
|
|
//
|
|
|
|
if (Flags & MM_PHYSICAL_MEMORY_PRODUCED_VIA_COMPRESSION) {
|
|
MmAvailablePages += (PFN_COUNT) RemovedPages;
|
|
MiNumberOfCompressionPages -= RemovedPages;
|
|
}
|
|
#endif
|
|
|
|
if (RemovedPages != NumberOfPages) {
|
|
|
|
#if DBG
|
|
retry:
|
|
#endif
|
|
|
|
Pfn1 = StartPfn;
|
|
|
|
InterlockedIncrement (&MiDelayPageFaults);
|
|
|
|
for (i = 0; i < 5; i += 1) {
|
|
|
|
UNLOCK_PFN (OldIrql);
|
|
|
|
//
|
|
// Attempt to move pages to the standby list. Note that only the
|
|
// pages with RemovalRequested set are moved.
|
|
//
|
|
|
|
MiTrimRemovalPagesOnly = TRUE;
|
|
|
|
MmEmptyAllWorkingSets ();
|
|
|
|
MiTrimRemovalPagesOnly = FALSE;
|
|
|
|
MiFlushAllPages ();
|
|
|
|
KeDelayExecutionThread (KernelMode, FALSE, (PLARGE_INTEGER)&MmHalfSecond);
|
|
|
|
LOCK_PFN (OldIrql);
|
|
|
|
RemovedPagesThisPass = MiRemovePhysicalPages (StartPage, EndPage);
|
|
|
|
RemovedPages += RemovedPagesThisPass;
|
|
|
|
#if defined (_MI_COMPRESSION)
|
|
|
|
//
|
|
// Compression range removals add back into AvailablePages the same
|
|
// amount that MiUnlinkPageFromList removes (as the original
|
|
// addition of these ranges never bumps this counter).
|
|
//
|
|
|
|
if (Flags & MM_PHYSICAL_MEMORY_PRODUCED_VIA_COMPRESSION) {
|
|
MmAvailablePages += (PFN_COUNT) RemovedPagesThisPass;
|
|
MiNumberOfCompressionPages -= RemovedPagesThisPass;
|
|
}
|
|
|
|
#endif
|
|
|
|
if (RemovedPages == NumberOfPages) {
|
|
break;
|
|
}
|
|
|
|
//
|
|
// RemovedPages doesn't include pages that were freed directly
|
|
// to the bad page list via MiDecrementReferenceCount or by
|
|
// ECC marking. So use the above check purely as an optimization -
|
|
// and walk here before ever giving up.
|
|
//
|
|
|
|
for ( ; Pfn1 < EndPfn; Pfn1 += 1) {
|
|
if (Pfn1->u3.e1.PageLocation != BadPageList) {
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (Pfn1 == EndPfn) {
|
|
RemovedPages = NumberOfPages;
|
|
break;
|
|
}
|
|
}
|
|
|
|
InterlockedDecrement (&MiDelayPageFaults);
|
|
}
|
|
|
|
if (RemovedPages != NumberOfPages) {
|
|
#if DBG
|
|
MiDynmemData[4] += 1;
|
|
if (MiShowStuckPages != 0) {
|
|
|
|
RemovedPages = 0;
|
|
for (Pfn1 = StartPfn; Pfn1 < EndPfn; Pfn1 += 1) {
|
|
if (Pfn1->u3.e1.PageLocation != BadPageList) {
|
|
RemovedPages += 1;
|
|
}
|
|
}
|
|
|
|
ASSERT (RemovedPages != 0);
|
|
|
|
DbgPrint("MiRemovePhysicalMemory : could not get %d of %d pages\n",
|
|
RemovedPages, NumberOfPages);
|
|
|
|
if (MiShowStuckPages & 0x2) {
|
|
|
|
ULONG PfnsPrinted;
|
|
ULONG EnoughShown;
|
|
PMMPFN FirstPfn;
|
|
PFN_COUNT PfnCount;
|
|
|
|
PfnCount = 0;
|
|
PfnsPrinted = 0;
|
|
EnoughShown = 100;
|
|
|
|
//
|
|
// Initializing FirstPfn is not needed for correctness
|
|
// but without it the compiler cannot compile this code
|
|
// W4 to check for use of uninitialized variables.
|
|
//
|
|
|
|
FirstPfn = NULL;
|
|
|
|
if (MiShowStuckPages & 0x4) {
|
|
EnoughShown = (ULONG)-1;
|
|
}
|
|
|
|
DbgPrint("Stuck PFN list: ");
|
|
for (Pfn1 = StartPfn; Pfn1 < EndPfn; Pfn1 += 1) {
|
|
if (Pfn1->u3.e1.PageLocation != BadPageList) {
|
|
if (PfnCount == 0) {
|
|
FirstPfn = Pfn1;
|
|
}
|
|
PfnCount += 1;
|
|
}
|
|
else {
|
|
if (PfnCount != 0) {
|
|
DbgPrint("%x -> %x ; ", FirstPfn - MmPfnDatabase,
|
|
(FirstPfn - MmPfnDatabase) + PfnCount - 1);
|
|
PfnsPrinted += 1;
|
|
if (PfnsPrinted == EnoughShown) {
|
|
break;
|
|
}
|
|
PfnCount = 0;
|
|
}
|
|
}
|
|
}
|
|
if (PfnCount != 0) {
|
|
DbgPrint("%x -> %x ; ", FirstPfn - MmPfnDatabase,
|
|
(FirstPfn - MmPfnDatabase) + PfnCount - 1);
|
|
}
|
|
DbgPrint("\n");
|
|
}
|
|
if (MiShowStuckPages & 0x8) {
|
|
DbgBreakPoint ();
|
|
}
|
|
if (MiShowStuckPages & 0x10) {
|
|
goto retry;
|
|
}
|
|
}
|
|
#endif
|
|
UNLOCK_PFN (OldIrql);
|
|
Status = STATUS_NO_MEMORY;
|
|
goto giveup;
|
|
}
|
|
|
|
#if DBG
|
|
for (Pfn1 = StartPfn; Pfn1 < EndPfn; Pfn1 += 1) {
|
|
ASSERT (Pfn1->u3.e1.PageLocation == BadPageList);
|
|
}
|
|
#endif
|
|
|
|
//
|
|
// All the pages in the range have been removed.
|
|
//
|
|
|
|
if (PermanentRemoval == FALSE) {
|
|
|
|
//
|
|
// If it's just the removal of ECC-marked bad pages, then no
|
|
// adjustment to the physical memory block ranges or PFN database
|
|
// trimming is needed. Exit now.
|
|
//
|
|
|
|
for (Pfn1 = StartPfn; Pfn1 < EndPfn; Pfn1 += 1) {
|
|
ASSERT (Pfn1->u3.e1.ParityError == 0);
|
|
Pfn1->u3.e1.ParityError = 1;
|
|
}
|
|
|
|
UNLOCK_PFN (OldIrql);
|
|
|
|
MmUnlockPagableImageSection(ExPageLockHandle);
|
|
|
|
ExReleaseFastMutex (&MmDynamicMemoryMutex);
|
|
|
|
NumberOfBytes->QuadPart = (ULONGLONG)NumberOfPages * PAGE_SIZE;
|
|
|
|
return STATUS_SUCCESS;
|
|
}
|
|
|
|
//
|
|
// Update the physical memory blocks and other associated housekeeping.
|
|
//
|
|
|
|
if (Additional == 0) {
|
|
|
|
//
|
|
// The range can be split off from an end of an existing chunk so no
|
|
// pool growth or shrinkage is required.
|
|
//
|
|
|
|
NewPhysicalMemoryBlock = MmPhysicalMemoryBlock;
|
|
OldPhysicalMemoryBlock = NULL;
|
|
}
|
|
else {
|
|
|
|
//
|
|
// The range cannot be split off from an end of an existing chunk so
|
|
// pool growth or shrinkage is required.
|
|
//
|
|
|
|
UNLOCK_PFN (OldIrql);
|
|
|
|
i = (sizeof(PHYSICAL_MEMORY_DESCRIPTOR) +
|
|
(sizeof(PHYSICAL_MEMORY_RUN) * (MmPhysicalMemoryBlock->NumberOfRuns + Additional)));
|
|
|
|
NewPhysicalMemoryBlock = ExAllocatePoolWithTag (NonPagedPool,
|
|
i,
|
|
' mM');
|
|
|
|
if (NewPhysicalMemoryBlock == NULL) {
|
|
Status = STATUS_INSUFFICIENT_RESOURCES;
|
|
#if DBG
|
|
MiDynmemData[5] += 1;
|
|
#endif
|
|
goto giveup;
|
|
}
|
|
|
|
OldPhysicalMemoryBlock = MmPhysicalMemoryBlock;
|
|
RtlZeroMemory (NewPhysicalMemoryBlock, i);
|
|
|
|
LOCK_PFN (OldIrql);
|
|
}
|
|
|
|
//
|
|
// Remove or split the requested range from the existing memory block.
|
|
//
|
|
|
|
NewPhysicalMemoryBlock->NumberOfRuns = MmPhysicalMemoryBlock->NumberOfRuns + Additional;
|
|
NewPhysicalMemoryBlock->NumberOfPages = MmPhysicalMemoryBlock->NumberOfPages - NumberOfPages;
|
|
|
|
NewRun = &NewPhysicalMemoryBlock->Run[0];
|
|
start = 0;
|
|
Inserted = FALSE;
|
|
|
|
do {
|
|
|
|
Page = MmPhysicalMemoryBlock->Run[start].BasePage;
|
|
LastPage = Page + MmPhysicalMemoryBlock->Run[start].PageCount;
|
|
|
|
if (Inserted == FALSE) {
|
|
|
|
if ((StartPage >= Page) && (EndPage <= LastPage)) {
|
|
|
|
if ((StartPage == Page) && (EndPage == LastPage)) {
|
|
ASSERT (Additional == -1);
|
|
start += 1;
|
|
continue;
|
|
}
|
|
else if ((StartPage == Page) || (EndPage == LastPage)) {
|
|
ASSERT (Additional == 0);
|
|
if (StartPage == Page) {
|
|
MmPhysicalMemoryBlock->Run[start].BasePage += NumberOfPages;
|
|
}
|
|
MmPhysicalMemoryBlock->Run[start].PageCount -= NumberOfPages;
|
|
}
|
|
else {
|
|
ASSERT (Additional == 1);
|
|
|
|
OriginalLastPage = LastPage;
|
|
|
|
MmPhysicalMemoryBlock->Run[start].PageCount =
|
|
StartPage - MmPhysicalMemoryBlock->Run[start].BasePage;
|
|
|
|
*NewRun = MmPhysicalMemoryBlock->Run[start];
|
|
NewRun += 1;
|
|
|
|
NewRun->BasePage = EndPage;
|
|
NewRun->PageCount = OriginalLastPage - EndPage;
|
|
NewRun += 1;
|
|
|
|
start += 1;
|
|
continue;
|
|
}
|
|
|
|
Inserted = TRUE;
|
|
}
|
|
}
|
|
|
|
*NewRun = MmPhysicalMemoryBlock->Run[start];
|
|
NewRun += 1;
|
|
start += 1;
|
|
|
|
} while (start != MmPhysicalMemoryBlock->NumberOfRuns);
|
|
|
|
//
|
|
// Repoint the MmPhysicalMemoryBlock at the new chunk.
|
|
// Free the old block after releasing the PFN lock.
|
|
//
|
|
|
|
MmPhysicalMemoryBlock = NewPhysicalMemoryBlock;
|
|
|
|
if (EndPage - 1 == MmHighestPhysicalPage) {
|
|
MmHighestPhysicalPage = StartPage - 1;
|
|
}
|
|
|
|
//
|
|
// Throw away all the removed pages that are currently enqueued.
|
|
//
|
|
|
|
ParityPages = 0;
|
|
for (Pfn1 = StartPfn; Pfn1 < EndPfn; Pfn1 += 1) {
|
|
|
|
ASSERT (Pfn1->u3.e1.PageLocation == BadPageList);
|
|
ASSERT (Pfn1->u3.e1.RemovalRequested == 1);
|
|
|
|
//
|
|
// Some pages may have already been ECC-removed. For these pages,
|
|
// the commit limits and resident available pages have already been
|
|
// adjusted - tally them here so we can undo the extraneous charge
|
|
// just applied.
|
|
//
|
|
|
|
if (Pfn1->u3.e1.ParityError == 1) {
|
|
ParityPages += 1;
|
|
}
|
|
|
|
MiUnlinkPageFromList (Pfn1);
|
|
|
|
ASSERT (Pfn1->u1.Flink == 0);
|
|
ASSERT (Pfn1->u2.Blink == 0);
|
|
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
|
|
ASSERT64 (Pfn1->UsedPageTableEntries == 0);
|
|
|
|
Pfn1->PteAddress = PFN_REMOVED;
|
|
|
|
//
|
|
// Note this clears ParityError among other flags...
|
|
//
|
|
|
|
Pfn1->u3.e2.ShortFlags = 0;
|
|
Pfn1->OriginalPte.u.Long = ZeroKernelPte.u.Long;
|
|
Pfn1->u4.PteFrame = 0;
|
|
}
|
|
|
|
//
|
|
// Now that the removed pages have been discarded, eliminate the PFN
|
|
// entries that mapped them. Straddling entries left over from an
|
|
// adjacent earlier removal are not collapsed at this point.
|
|
//
|
|
//
|
|
|
|
PagesReleased = 0;
|
|
|
|
if (PfnDatabaseIsPhysical == FALSE) {
|
|
|
|
VirtualAddress = (PVOID)ROUND_TO_PAGES(MI_PFN_ELEMENT(StartPage));
|
|
PointerPte = MiGetPteAddress (VirtualAddress);
|
|
EndPte = MiGetPteAddress (PAGE_ALIGN(MI_PFN_ELEMENT(EndPage)));
|
|
|
|
while (PointerPte < EndPte) {
|
|
PageFrameIndex = MI_GET_PAGE_FRAME_FROM_PTE (PointerPte);
|
|
Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
|
|
ASSERT (Pfn1->u2.ShareCount == 1);
|
|
ASSERT (Pfn1->u3.e2.ReferenceCount == 1);
|
|
Pfn1->u2.ShareCount = 0;
|
|
MI_SET_PFN_DELETED (Pfn1);
|
|
#if DBG
|
|
Pfn1->u3.e1.PageLocation = StandbyPageList;
|
|
#endif //DBG
|
|
MiDecrementReferenceCount (PageFrameIndex);
|
|
|
|
KeFlushSingleTb (VirtualAddress,
|
|
TRUE,
|
|
TRUE,
|
|
(PHARDWARE_PTE)PointerPte,
|
|
ZeroKernelPte.u.Flush);
|
|
|
|
PagesReleased += 1;
|
|
PointerPte += 1;
|
|
VirtualAddress = (PVOID)((PCHAR)VirtualAddress + PAGE_SIZE);
|
|
}
|
|
|
|
MmResidentAvailablePages += PagesReleased;
|
|
}
|
|
|
|
#if DBG
|
|
MiDynmemData[6] += 1;
|
|
#endif
|
|
|
|
//
|
|
// Give back anything that has been double-charged.
|
|
//
|
|
|
|
if (ParityPages != 0) {
|
|
MmResidentAvailablePages += ParityPages;
|
|
}
|
|
|
|
UNLOCK_PFN (OldIrql);
|
|
|
|
//
|
|
// Give back anything that has been double-charged.
|
|
//
|
|
|
|
if (ParityPages != 0) {
|
|
InterlockedExchangeAddSizeT (&MmTotalCommitLimitMaximum, ParityPages);
|
|
InterlockedExchangeAddSizeT (&MmTotalCommitLimit, ParityPages);
|
|
}
|
|
|
|
if (PagesReleased != 0) {
|
|
MiReturnCommitment (PagesReleased);
|
|
}
|
|
|
|
MmUnlockPagableImageSection(ExPageLockHandle);
|
|
|
|
ExReleaseFastMutex (&MmDynamicMemoryMutex);
|
|
|
|
if (OldPhysicalMemoryBlock != NULL) {
|
|
ExFreePool (OldPhysicalMemoryBlock);
|
|
}
|
|
|
|
NumberOfBytes->QuadPart = (ULONGLONG)NumberOfPages * PAGE_SIZE;
|
|
|
|
return STATUS_SUCCESS;
|
|
|
|
giveup:
|
|
|
|
//
|
|
// All the pages in the range were not obtained. Back everything out.
|
|
//
|
|
|
|
PageFrameIndex = StartPage;
|
|
Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
|
|
|
|
LOCK_PFN (OldIrql);
|
|
|
|
while (PageFrameIndex < EndPage) {
|
|
|
|
ASSERT (Pfn1->u3.e1.RemovalRequested == 1);
|
|
|
|
Pfn1->u3.e1.RemovalRequested = 0;
|
|
|
|
if (Pfn1->u3.e1.PageLocation == BadPageList) {
|
|
MiUnlinkPageFromList (Pfn1);
|
|
MiInsertPageInFreeList (PageFrameIndex);
|
|
}
|
|
|
|
Pfn1 += 1;
|
|
PageFrameIndex += 1;
|
|
}
|
|
|
|
#if defined (_MI_COMPRESSION)
|
|
|
|
//
|
|
// Only removal of non-compression ranges decrement ResidentAvailable as
|
|
// only those ranges actually incremented this when they were added.
|
|
//
|
|
|
|
if ((Flags & MM_PHYSICAL_MEMORY_PRODUCED_VIA_COMPRESSION) == 0) {
|
|
MmResidentAvailablePages += NumberOfPages;
|
|
}
|
|
else {
|
|
|
|
//
|
|
// Compression range removals add back into AvailablePages the same
|
|
// amount that MiUnlinkPageFromList removes (as the original
|
|
// addition of these ranges never bumps this counter).
|
|
//
|
|
|
|
MmAvailablePages -= (PFN_COUNT) RemovedPages;
|
|
MiNumberOfCompressionPages += RemovedPages;
|
|
}
|
|
#else
|
|
MmResidentAvailablePages += NumberOfPages;
|
|
#endif
|
|
|
|
if (PermanentRemoval == TRUE) {
|
|
MmNumberOfPhysicalPages += NumberOfPages;
|
|
|
|
InterlockedExchangeAdd ((PLONG)&SharedUserData->NumberOfPhysicalPages,
|
|
NumberOfPages);
|
|
|
|
#if defined (_MI_COMPRESSION)
|
|
|
|
//
|
|
// Rearm the interrupt to occur at the original threshold.
|
|
//
|
|
|
|
if ((Flags & MM_PHYSICAL_MEMORY_PRODUCED_VIA_COMPRESSION) == 0) {
|
|
MiArmCompressionInterrupt ();
|
|
}
|
|
#endif
|
|
}
|
|
|
|
UNLOCK_PFN (OldIrql);
|
|
|
|
giveup2:
|
|
|
|
InterlockedExchangeAddSizeT (&MmTotalCommitLimitMaximum, NumberOfPages);
|
|
InterlockedExchangeAddSizeT (&MmTotalCommitLimit, NumberOfPages);
|
|
|
|
MmUnlockPagableImageSection(ExPageLockHandle);
|
|
ExReleaseFastMutex (&MmDynamicMemoryMutex);
|
|
|
|
return Status;
|
|
}
|
|
|
|
|
|
NTSTATUS
|
|
MmRemovePhysicalMemory (
|
|
IN PPHYSICAL_ADDRESS StartAddress,
|
|
IN OUT PLARGE_INTEGER NumberOfBytes
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
A wrapper for MmRemovePhysicalMemoryEx.
|
|
|
|
Arguments:
|
|
|
|
StartAddress - Supplies the starting physical address.
|
|
|
|
NumberOfBytes - Supplies a pointer to the number of bytes being removed.
|
|
|
|
Return Value:
|
|
|
|
NTSTATUS.
|
|
|
|
Environment:
|
|
|
|
Kernel mode. PASSIVE level. No locks held.
|
|
|
|
--*/
|
|
|
|
{
|
|
return MmRemovePhysicalMemoryEx (StartAddress, NumberOfBytes, 0);
|
|
}
|
|
|
|
NTSTATUS
|
|
MmRemovePhysicalMemoryEx (
|
|
IN PPHYSICAL_ADDRESS StartAddress,
|
|
IN OUT PLARGE_INTEGER NumberOfBytes,
|
|
IN ULONG Flags
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine attempts to remove the specified physical address range
|
|
from the system.
|
|
|
|
Arguments:
|
|
|
|
StartAddress - Supplies the starting physical address.
|
|
|
|
NumberOfBytes - Supplies a pointer to the number of bytes being removed.
|
|
|
|
Flags - Supplies relevant flags describing the memory range.
|
|
|
|
Return Value:
|
|
|
|
NTSTATUS.
|
|
|
|
Environment:
|
|
|
|
Kernel mode. PASSIVE level. No locks held.
|
|
|
|
--*/
|
|
|
|
{
|
|
NTSTATUS Status;
|
|
#if defined (_X86_)
|
|
BOOLEAN CachesFlushed;
|
|
#endif
|
|
#if defined(_IA64_)
|
|
PVOID VirtualAddress;
|
|
PVOID SingleVirtualAddress;
|
|
SIZE_T SizeInBytes;
|
|
SIZE_T MapSizeInBytes;
|
|
PFN_COUNT NumberOfPages;
|
|
PFN_COUNT i;
|
|
PFN_NUMBER StartPage;
|
|
#endif
|
|
|
|
PAGED_CODE();
|
|
|
|
#if defined (_MI_COMPRESSION_SUPPORTED_)
|
|
if (Flags & MM_PHYSICAL_MEMORY_PRODUCED_VIA_COMPRESSION) {
|
|
return STATUS_NOT_SUPPORTED;
|
|
}
|
|
#else
|
|
if (Flags != 0) {
|
|
return STATUS_INVALID_PARAMETER_3;
|
|
}
|
|
#endif
|
|
|
|
#if defined (_X86_)
|
|
|
|
//
|
|
// Issue a cache invalidation here just as a test to make sure the
|
|
// machine can support it. If not, then don't bother trying to remove
|
|
// any memory.
|
|
//
|
|
|
|
CachesFlushed = KeInvalidateAllCaches (TRUE);
|
|
if (CachesFlushed == FALSE) {
|
|
return STATUS_NOT_SUPPORTED;
|
|
}
|
|
#endif
|
|
|
|
#if defined(_IA64_)
|
|
|
|
//
|
|
// Pick up at least a single PTE mapping now as we do not want to fail this
|
|
// call if no PTEs are available after a successful remove. Resorting to
|
|
// actually using this PTE should be a very rare case indeed.
|
|
//
|
|
|
|
SingleVirtualAddress = (PMMPTE)MiMapSinglePage (NULL,
|
|
0,
|
|
MmCached,
|
|
HighPagePriority);
|
|
|
|
if (SingleVirtualAddress == NULL) {
|
|
return STATUS_INSUFFICIENT_RESOURCES;
|
|
}
|
|
|
|
#endif
|
|
|
|
Status = MiRemovePhysicalMemory (StartAddress, NumberOfBytes, TRUE, Flags);
|
|
|
|
if (NT_SUCCESS (Status)) {
|
|
|
|
#if defined (_X86_)
|
|
CachesFlushed = KeInvalidateAllCaches (TRUE);
|
|
ASSERT (CachesFlushed == TRUE);
|
|
#endif
|
|
|
|
#if defined(_IA64_)
|
|
SizeInBytes = (SIZE_T)NumberOfBytes->QuadPart;
|
|
|
|
//
|
|
// Flush the entire TB to remove any KSEG translations that may map the
|
|
// pages being removed. Otherwise hardware or software speculation
|
|
// can reference the memory speculatively which would crash the machine.
|
|
//
|
|
|
|
KeFlushEntireTb (TRUE, TRUE);
|
|
|
|
//
|
|
// Establish an uncached mapping to the pages being removed.
|
|
//
|
|
|
|
MapSizeInBytes = SizeInBytes;
|
|
|
|
//
|
|
// Initializing VirtualAddress is not needed for correctness
|
|
// but without it the compiler cannot compile this code
|
|
// W4 to check for use of uninitialized variables.
|
|
//
|
|
|
|
VirtualAddress = NULL;
|
|
|
|
while (MapSizeInBytes > PAGE_SIZE) {
|
|
|
|
VirtualAddress = MmMapIoSpace (*StartAddress,
|
|
MapSizeInBytes,
|
|
MmNonCached);
|
|
|
|
if (VirtualAddress != NULL) {
|
|
break;
|
|
}
|
|
|
|
MapSizeInBytes = MapSizeInBytes >> 1;
|
|
}
|
|
|
|
if (MapSizeInBytes <= PAGE_SIZE) {
|
|
|
|
StartPage = (PFN_NUMBER)(StartAddress->QuadPart >> PAGE_SHIFT);
|
|
|
|
NumberOfPages = (PFN_COUNT)(NumberOfBytes->QuadPart >> PAGE_SHIFT);
|
|
|
|
for (i = 0; i < NumberOfPages; i += 1) {
|
|
|
|
SingleVirtualAddress = (PMMPTE)MiMapSinglePage (SingleVirtualAddress,
|
|
StartPage,
|
|
MmCached,
|
|
HighPagePriority);
|
|
|
|
KeSweepCacheRangeWithDrain (TRUE,
|
|
SingleVirtualAddress,
|
|
PAGE_SIZE);
|
|
|
|
StartPage += 1;
|
|
}
|
|
}
|
|
else {
|
|
|
|
//
|
|
// Drain all pending transactions and prefetches and perform cache
|
|
// evictions. Only drain 4gb max at a time as this API takes a
|
|
// ULONG.
|
|
//
|
|
|
|
while (SizeInBytes > _4gb) {
|
|
KeSweepCacheRangeWithDrain (TRUE, VirtualAddress, _4gb - 1);
|
|
SizeInBytes -= (_4gb - 1);
|
|
}
|
|
|
|
KeSweepCacheRangeWithDrain (TRUE,
|
|
VirtualAddress,
|
|
(ULONG)SizeInBytes);
|
|
|
|
MmUnmapIoSpace (VirtualAddress, NumberOfBytes->QuadPart);
|
|
}
|
|
#endif
|
|
}
|
|
|
|
#if defined(_IA64_)
|
|
MiUnmapSinglePage (SingleVirtualAddress);
|
|
#endif
|
|
|
|
return Status;
|
|
}
|
|
|
|
NTSTATUS
|
|
MmMarkPhysicalMemoryAsBad (
|
|
IN PPHYSICAL_ADDRESS StartAddress,
|
|
IN OUT PLARGE_INTEGER NumberOfBytes
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine attempts to mark the specified physical address range
|
|
as bad so the system will not use it. This is generally done for pages
|
|
which contain ECC errors.
|
|
|
|
Note that this is different from removing pages permanently (ie: physically
|
|
removing the memory board) which should be done via the
|
|
MmRemovePhysicalMemory API.
|
|
|
|
The caller is responsible for maintaining a global table so that subsequent
|
|
boots can examine it and remove the ECC pages before loading the kernel.
|
|
|
|
Arguments:
|
|
|
|
StartAddress - Supplies the starting physical address.
|
|
|
|
NumberOfBytes - Supplies a pointer to the number of bytes being removed.
|
|
|
|
Return Value:
|
|
|
|
NTSTATUS.
|
|
|
|
Environment:
|
|
|
|
Kernel mode. PASSIVE level. No locks held.
|
|
|
|
--*/
|
|
|
|
{
|
|
PAGED_CODE();
|
|
|
|
return MiRemovePhysicalMemory (StartAddress, NumberOfBytes, FALSE, 0);
|
|
}
|
|
|
|
NTSTATUS
|
|
MmMarkPhysicalMemoryAsGood (
|
|
IN PPHYSICAL_ADDRESS StartAddress,
|
|
IN OUT PLARGE_INTEGER NumberOfBytes
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine attempts to mark the specified physical address range
|
|
as good so the system will use it. This is generally done for pages
|
|
which used to (but presumably no longer do) contain ECC errors.
|
|
|
|
Note that this is different from adding pages permanently (ie: physically
|
|
inserting a new memory board) which should be done via the
|
|
MmAddPhysicalMemory API.
|
|
|
|
The caller is responsible for removing these entries from a global table
|
|
so that subsequent boots will use the pages.
|
|
|
|
Arguments:
|
|
|
|
StartAddress - Supplies the starting physical address.
|
|
|
|
NumberOfBytes - Supplies a pointer to the number of bytes being removed.
|
|
|
|
Return Value:
|
|
|
|
NTSTATUS.
|
|
|
|
Environment:
|
|
|
|
Kernel mode. PASSIVE level. No locks held.
|
|
|
|
--*/
|
|
|
|
{
|
|
PMMPFN Pfn1;
|
|
KIRQL OldIrql;
|
|
PFN_NUMBER NumberOfPages;
|
|
PFN_NUMBER start;
|
|
PFN_NUMBER count;
|
|
PFN_NUMBER StartPage;
|
|
PFN_NUMBER EndPage;
|
|
PFN_NUMBER PageFrameIndex;
|
|
PFN_NUMBER Page;
|
|
PFN_NUMBER LastPage;
|
|
LOGICAL PfnDatabaseIsPhysical;
|
|
|
|
ASSERT (KeGetCurrentIrql() == PASSIVE_LEVEL);
|
|
|
|
ASSERT (BYTE_OFFSET(NumberOfBytes->LowPart) == 0);
|
|
ASSERT (BYTE_OFFSET(StartAddress->LowPart) == 0);
|
|
|
|
if (MI_IS_PHYSICAL_ADDRESS(MmPfnDatabase)) {
|
|
PfnDatabaseIsPhysical = TRUE;
|
|
}
|
|
else {
|
|
PfnDatabaseIsPhysical = FALSE;
|
|
}
|
|
|
|
StartPage = (PFN_NUMBER)(StartAddress->QuadPart >> PAGE_SHIFT);
|
|
NumberOfPages = (PFN_NUMBER)(NumberOfBytes->QuadPart >> PAGE_SHIFT);
|
|
|
|
EndPage = StartPage + NumberOfPages;
|
|
|
|
ExAcquireFastMutex (&MmDynamicMemoryMutex);
|
|
|
|
if (EndPage - 1 > MmHighestPhysicalPage) {
|
|
|
|
//
|
|
// Truncate the request into something that can be mapped by the PFN
|
|
// database.
|
|
//
|
|
|
|
EndPage = MmHighestPhysicalPage + 1;
|
|
NumberOfPages = EndPage - StartPage;
|
|
}
|
|
|
|
//
|
|
// The range cannot wrap.
|
|
//
|
|
|
|
if (StartPage >= EndPage) {
|
|
ExReleaseFastMutex (&MmDynamicMemoryMutex);
|
|
return STATUS_INVALID_PARAMETER_1;
|
|
}
|
|
|
|
//
|
|
// The request must lie within an already present range.
|
|
//
|
|
|
|
start = 0;
|
|
|
|
MmLockPagableSectionByHandle (ExPageLockHandle);
|
|
|
|
LOCK_PFN (OldIrql);
|
|
|
|
do {
|
|
|
|
count = MmPhysicalMemoryBlock->Run[start].PageCount;
|
|
Page = MmPhysicalMemoryBlock->Run[start].BasePage;
|
|
|
|
if (count != 0) {
|
|
|
|
LastPage = Page + count;
|
|
|
|
if ((StartPage >= Page) && (EndPage <= LastPage)) {
|
|
break;
|
|
}
|
|
}
|
|
|
|
start += 1;
|
|
|
|
} while (start != MmPhysicalMemoryBlock->NumberOfRuns);
|
|
|
|
if (start == MmPhysicalMemoryBlock->NumberOfRuns) {
|
|
UNLOCK_PFN (OldIrql);
|
|
MmUnlockPagableImageSection(ExPageLockHandle);
|
|
ExReleaseFastMutex (&MmDynamicMemoryMutex);
|
|
return STATUS_CONFLICTING_ADDRESSES;
|
|
}
|
|
|
|
//
|
|
// Walk through the range and add only pages previously removed to the
|
|
// free list in the PFN database.
|
|
//
|
|
|
|
PageFrameIndex = StartPage;
|
|
Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
|
|
NumberOfPages = 0;
|
|
|
|
while (PageFrameIndex < EndPage) {
|
|
|
|
if ((Pfn1->u3.e1.ParityError == 1) &&
|
|
(Pfn1->u3.e1.RemovalRequested == 1) &&
|
|
(Pfn1->u3.e1.PageLocation == BadPageList)) {
|
|
|
|
Pfn1->u3.e1.ParityError = 0;
|
|
Pfn1->u3.e1.RemovalRequested = 0;
|
|
MiUnlinkPageFromList (Pfn1);
|
|
MiInsertPageInFreeList (PageFrameIndex);
|
|
NumberOfPages += 1;
|
|
}
|
|
|
|
Pfn1 += 1;
|
|
PageFrameIndex += 1;
|
|
}
|
|
|
|
MmResidentAvailablePages += NumberOfPages;
|
|
|
|
UNLOCK_PFN (OldIrql);
|
|
|
|
//
|
|
// Increase all commit limits to reflect the additional memory.
|
|
//
|
|
|
|
InterlockedExchangeAddSizeT (&MmTotalCommitLimitMaximum, NumberOfPages);
|
|
|
|
InterlockedExchangeAddSizeT (&MmTotalCommitLimit, NumberOfPages);
|
|
|
|
MmUnlockPagableImageSection(ExPageLockHandle);
|
|
|
|
ExReleaseFastMutex (&MmDynamicMemoryMutex);
|
|
|
|
//
|
|
// Indicate number of bytes actually added to our caller.
|
|
//
|
|
|
|
NumberOfBytes->QuadPart = (ULONGLONG)NumberOfPages * PAGE_SIZE;
|
|
|
|
return STATUS_SUCCESS;
|
|
}
|
|
|
|
PPHYSICAL_MEMORY_RANGE
|
|
MmGetPhysicalMemoryRanges (
|
|
VOID
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine returns the virtual address of a nonpaged pool block which
|
|
contains the physical memory ranges in the system.
|
|
|
|
The returned block contains physical address and page count pairs.
|
|
The last entry contains zero for both.
|
|
|
|
The caller must understand that this block can change at any point before
|
|
or after this snapshot.
|
|
|
|
It is the caller's responsibility to free this block.
|
|
|
|
Arguments:
|
|
|
|
None.
|
|
|
|
Return Value:
|
|
|
|
NULL on failure.
|
|
|
|
Environment:
|
|
|
|
Kernel mode. PASSIVE level. No locks held.
|
|
|
|
--*/
|
|
|
|
{
|
|
ULONG i;
|
|
KIRQL OldIrql;
|
|
PPHYSICAL_MEMORY_RANGE p;
|
|
PPHYSICAL_MEMORY_RANGE PhysicalMemoryBlock;
|
|
|
|
ASSERT (KeGetCurrentIrql() == PASSIVE_LEVEL);
|
|
|
|
ExAcquireFastMutex (&MmDynamicMemoryMutex);
|
|
|
|
i = sizeof(PHYSICAL_MEMORY_RANGE) * (MmPhysicalMemoryBlock->NumberOfRuns + 1);
|
|
|
|
PhysicalMemoryBlock = ExAllocatePoolWithTag (NonPagedPool,
|
|
i,
|
|
'hPmM');
|
|
|
|
if (PhysicalMemoryBlock == NULL) {
|
|
ExReleaseFastMutex (&MmDynamicMemoryMutex);
|
|
return NULL;
|
|
}
|
|
|
|
p = PhysicalMemoryBlock;
|
|
|
|
MmLockPagableSectionByHandle (ExPageLockHandle);
|
|
|
|
LOCK_PFN (OldIrql);
|
|
|
|
ASSERT (i == (sizeof(PHYSICAL_MEMORY_RANGE) * (MmPhysicalMemoryBlock->NumberOfRuns + 1)));
|
|
|
|
for (i = 0; i < MmPhysicalMemoryBlock->NumberOfRuns; i += 1) {
|
|
p->BaseAddress.QuadPart = (LONGLONG)MmPhysicalMemoryBlock->Run[i].BasePage * PAGE_SIZE;
|
|
p->NumberOfBytes.QuadPart = (LONGLONG)MmPhysicalMemoryBlock->Run[i].PageCount * PAGE_SIZE;
|
|
p += 1;
|
|
}
|
|
|
|
p->BaseAddress.QuadPart = 0;
|
|
p->NumberOfBytes.QuadPart = 0;
|
|
|
|
UNLOCK_PFN (OldIrql);
|
|
|
|
MmUnlockPagableImageSection(ExPageLockHandle);
|
|
|
|
ExReleaseFastMutex (&MmDynamicMemoryMutex);
|
|
|
|
return PhysicalMemoryBlock;
|
|
}
|
|
|
|
PFN_COUNT
|
|
MiRemovePhysicalPages (
|
|
IN PFN_NUMBER StartPage,
|
|
IN PFN_NUMBER EndPage
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine searches the PFN database for free, zeroed or standby pages
|
|
that are marked for removal.
|
|
|
|
Arguments:
|
|
|
|
StartPage - Supplies the low physical frame number to remove.
|
|
|
|
EndPage - Supplies the last physical frame number to remove.
|
|
|
|
Return Value:
|
|
|
|
Returns the number of pages removed from the free, zeroed and standby lists.
|
|
|
|
Environment:
|
|
|
|
Kernel mode, PFN lock held. Since this routine is PAGELK, the caller is
|
|
responsible for locking it down and unlocking it on return.
|
|
|
|
--*/
|
|
|
|
{
|
|
PMMPFN Pfn1;
|
|
PMMPFN Pfn2;
|
|
PMMPFN PfnNextColored;
|
|
PMMPFN PfnNextFlink;
|
|
PMMPFN PfnLastColored;
|
|
PFN_NUMBER Page;
|
|
LOGICAL RemovePage;
|
|
ULONG Color;
|
|
PMMCOLOR_TABLES ColorHead;
|
|
PFN_NUMBER MovedPage;
|
|
MMLISTS MemoryList;
|
|
PFN_NUMBER PageNextColored;
|
|
PFN_NUMBER PageNextFlink;
|
|
PFN_NUMBER PageLastColored;
|
|
PFN_COUNT NumberOfPages;
|
|
PMMPFNLIST ListHead;
|
|
LOGICAL RescanNeeded;
|
|
|
|
MM_PFN_LOCK_ASSERT();
|
|
|
|
NumberOfPages = 0;
|
|
|
|
rescan:
|
|
|
|
//
|
|
// Grab all zeroed (and then free) pages first directly from the
|
|
// colored lists to avoid multiple walks down these singly linked lists.
|
|
// Handle transition pages last.
|
|
//
|
|
|
|
for (MemoryList = ZeroedPageList; MemoryList <= FreePageList; MemoryList += 1) {
|
|
|
|
ListHead = MmPageLocationList[MemoryList];
|
|
|
|
for (Color = 0; Color < MmSecondaryColors; Color += 1) {
|
|
ColorHead = &MmFreePagesByColor[MemoryList][Color];
|
|
|
|
MovedPage = (PFN_NUMBER) MM_EMPTY_LIST;
|
|
|
|
while (ColorHead->Flink != MM_EMPTY_LIST) {
|
|
|
|
Page = ColorHead->Flink;
|
|
|
|
Pfn1 = MI_PFN_ELEMENT(Page);
|
|
|
|
ASSERT ((MMLISTS)Pfn1->u3.e1.PageLocation == MemoryList);
|
|
|
|
//
|
|
// The Flink and Blink must be nonzero here for the page
|
|
// to be on the listhead. Only code that scans the
|
|
// MmPhysicalMemoryBlock has to check for the zero case.
|
|
//
|
|
|
|
ASSERT (Pfn1->u1.Flink != 0);
|
|
ASSERT (Pfn1->u2.Blink != 0);
|
|
|
|
//
|
|
// See if the page is desired by the caller.
|
|
//
|
|
// Systems utilizing memory compression may have more
|
|
// pages on the zero, free and standby lists than we
|
|
// want to give out. Explicitly check MmAvailablePages
|
|
// instead (and recheck whenever the PFN lock is
|
|
// released and reacquired).
|
|
//
|
|
|
|
if ((Pfn1->u3.e1.RemovalRequested == 1) &&
|
|
(MmAvailablePages != 0)) {
|
|
|
|
ASSERT (Pfn1->u3.e1.ReadInProgress == 0);
|
|
|
|
MiUnlinkFreeOrZeroedPage (Page);
|
|
|
|
MiInsertPageInList (&MmBadPageListHead, Page);
|
|
|
|
NumberOfPages += 1;
|
|
}
|
|
else {
|
|
|
|
//
|
|
// Unwanted so put the page on the end of list.
|
|
// If first time, save pfn.
|
|
//
|
|
|
|
if (MovedPage == MM_EMPTY_LIST) {
|
|
MovedPage = Page;
|
|
}
|
|
else if (Page == MovedPage) {
|
|
|
|
//
|
|
// No more pages available in this colored chain.
|
|
//
|
|
|
|
break;
|
|
}
|
|
|
|
//
|
|
// If the colored chain has more than one entry then
|
|
// put this page on the end.
|
|
//
|
|
|
|
PageNextColored = (PFN_NUMBER)Pfn1->OriginalPte.u.Long;
|
|
|
|
if (PageNextColored == MM_EMPTY_LIST) {
|
|
|
|
//
|
|
// No more pages available in this colored chain.
|
|
//
|
|
|
|
break;
|
|
}
|
|
|
|
ASSERT (Pfn1->u1.Flink != 0);
|
|
ASSERT (Pfn1->u1.Flink != MM_EMPTY_LIST);
|
|
ASSERT (Pfn1->u4.PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
|
|
|
PfnNextColored = MI_PFN_ELEMENT(PageNextColored);
|
|
ASSERT ((MMLISTS)PfnNextColored->u3.e1.PageLocation == MemoryList);
|
|
ASSERT (PfnNextColored->u4.PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
|
|
|
//
|
|
// Adjust the free page list so Page
|
|
// follows PageNextFlink.
|
|
//
|
|
|
|
PageNextFlink = Pfn1->u1.Flink;
|
|
PfnNextFlink = MI_PFN_ELEMENT(PageNextFlink);
|
|
|
|
ASSERT ((MMLISTS)PfnNextFlink->u3.e1.PageLocation == MemoryList);
|
|
ASSERT (PfnNextFlink->u4.PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
|
|
|
PfnLastColored = ColorHead->Blink;
|
|
ASSERT (PfnLastColored != (PMMPFN)MM_EMPTY_LIST);
|
|
ASSERT (PfnLastColored->OriginalPte.u.Long == MM_EMPTY_LIST);
|
|
ASSERT (PfnLastColored->u4.PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
|
ASSERT (PfnLastColored->u2.Blink != MM_EMPTY_LIST);
|
|
|
|
ASSERT ((MMLISTS)PfnLastColored->u3.e1.PageLocation == MemoryList);
|
|
PageLastColored = PfnLastColored - MmPfnDatabase;
|
|
|
|
if (ListHead->Flink == Page) {
|
|
|
|
ASSERT (Pfn1->u2.Blink == MM_EMPTY_LIST);
|
|
ASSERT (ListHead->Blink != Page);
|
|
|
|
ListHead->Flink = PageNextFlink;
|
|
|
|
PfnNextFlink->u2.Blink = MM_EMPTY_LIST;
|
|
}
|
|
else {
|
|
|
|
ASSERT (Pfn1->u2.Blink != MM_EMPTY_LIST);
|
|
ASSERT ((MMLISTS)(MI_PFN_ELEMENT((MI_PFN_ELEMENT(Pfn1->u2.Blink)->u1.Flink)))->u4.PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
|
ASSERT ((MMLISTS)(MI_PFN_ELEMENT((MI_PFN_ELEMENT(Pfn1->u2.Blink)->u1.Flink)))->u3.e1.PageLocation == MemoryList);
|
|
|
|
MI_PFN_ELEMENT(Pfn1->u2.Blink)->u1.Flink = PageNextFlink;
|
|
PfnNextFlink->u2.Blink = Pfn1->u2.Blink;
|
|
}
|
|
|
|
#if DBG
|
|
if (PfnLastColored->u1.Flink == MM_EMPTY_LIST) {
|
|
ASSERT (ListHead->Blink == PageLastColored);
|
|
}
|
|
#endif
|
|
|
|
Pfn1->u1.Flink = PfnLastColored->u1.Flink;
|
|
Pfn1->u2.Blink = PageLastColored;
|
|
|
|
if (ListHead->Blink == PageLastColored) {
|
|
ListHead->Blink = Page;
|
|
}
|
|
|
|
//
|
|
// Adjust the colored chains.
|
|
//
|
|
|
|
if (PfnLastColored->u1.Flink != MM_EMPTY_LIST) {
|
|
ASSERT (MI_PFN_ELEMENT(PfnLastColored->u1.Flink)->u4.PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
|
ASSERT ((MMLISTS)(MI_PFN_ELEMENT(PfnLastColored->u1.Flink)->u3.e1.PageLocation) == MemoryList);
|
|
MI_PFN_ELEMENT(PfnLastColored->u1.Flink)->u2.Blink = Page;
|
|
}
|
|
|
|
PfnLastColored->u1.Flink = Page;
|
|
|
|
ColorHead->Flink = PageNextColored;
|
|
Pfn1->OriginalPte.u.Long = MM_EMPTY_LIST;
|
|
|
|
ASSERT (PfnLastColored->OriginalPte.u.Long == MM_EMPTY_LIST);
|
|
PfnLastColored->OriginalPte.u.Long = Page;
|
|
ColorHead->Blink = Pfn1;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
RescanNeeded = FALSE;
|
|
Pfn1 = MI_PFN_ELEMENT (StartPage);
|
|
|
|
do {
|
|
|
|
if ((Pfn1->u3.e1.PageLocation == StandbyPageList) &&
|
|
(Pfn1->u1.Flink != 0) &&
|
|
(Pfn1->u2.Blink != 0) &&
|
|
(Pfn1->u3.e2.ReferenceCount == 0) &&
|
|
(MmAvailablePages != 0)) {
|
|
|
|
//
|
|
// Systems utilizing memory compression may have more
|
|
// pages on the zero, free and standby lists than we
|
|
// want to give out. Explicitly check MmAvailablePages
|
|
// above instead (and recheck whenever the PFN lock is
|
|
// released and reacquired).
|
|
//
|
|
|
|
ASSERT (Pfn1->u3.e1.ReadInProgress == 0);
|
|
|
|
RemovePage = TRUE;
|
|
|
|
if (Pfn1->u3.e1.RemovalRequested == 0) {
|
|
|
|
//
|
|
// This page is not directly needed for a hot remove - but if
|
|
// it contains a chunk of prototype PTEs (and this chunk is
|
|
// in a page that needs to be removed), then any pages
|
|
// referenced by transition prototype PTEs must also be removed
|
|
// before the desired page can be removed.
|
|
//
|
|
// The same analogy holds for page table, directory, parent
|
|
// and extended parent pages.
|
|
//
|
|
|
|
Pfn2 = MI_PFN_ELEMENT (Pfn1->u4.PteFrame);
|
|
if (Pfn2->u3.e1.RemovalRequested == 0) {
|
|
#if (_MI_PAGING_LEVELS >= 3)
|
|
Pfn2 = MI_PFN_ELEMENT (Pfn2->u4.PteFrame);
|
|
if (Pfn2->u3.e1.RemovalRequested == 0) {
|
|
RemovePage = FALSE;
|
|
}
|
|
else if (Pfn2->u2.ShareCount == 1) {
|
|
RescanNeeded = TRUE;
|
|
}
|
|
#if (_MI_PAGING_LEVELS >= 4)
|
|
Pfn2 = MI_PFN_ELEMENT (Pfn2->u4.PteFrame);
|
|
if (Pfn2->u3.e1.RemovalRequested == 0) {
|
|
RemovePage = FALSE;
|
|
}
|
|
else if (Pfn2->u2.ShareCount == 1) {
|
|
RescanNeeded = TRUE;
|
|
}
|
|
#endif
|
|
#else
|
|
RemovePage = FALSE;
|
|
#endif
|
|
}
|
|
else if (Pfn2->u2.ShareCount == 1) {
|
|
RescanNeeded = TRUE;
|
|
}
|
|
}
|
|
|
|
if (RemovePage == TRUE) {
|
|
|
|
//
|
|
// This page is in the desired range - grab it.
|
|
//
|
|
|
|
MiUnlinkPageFromList (Pfn1);
|
|
MiRestoreTransitionPte (StartPage);
|
|
MiInsertPageInList (&MmBadPageListHead, StartPage);
|
|
NumberOfPages += 1;
|
|
}
|
|
}
|
|
|
|
StartPage += 1;
|
|
Pfn1 += 1;
|
|
|
|
} while (StartPage < EndPage);
|
|
|
|
if (RescanNeeded == TRUE) {
|
|
|
|
//
|
|
// A page table, directory or parent was freed by removing a transition
|
|
// page from the cache. Rescan from the top to pick it up.
|
|
//
|
|
|
|
#if DBG
|
|
MiDynmemData[7] += 1;
|
|
#endif
|
|
|
|
goto rescan;
|
|
}
|
|
#if DBG
|
|
else {
|
|
MiDynmemData[8] += 1;
|
|
}
|
|
#endif
|
|
|
|
return NumberOfPages;
|
|
}
|