mirror of https://github.com/tongzx/nt5src
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
2758 lines
72 KiB
2758 lines
72 KiB
/*++
|
|
|
|
Copyright (c) 1989 Microsoft Corporation
|
|
|
|
Module Name:
|
|
|
|
pfnlist.c
|
|
|
|
Abstract:
|
|
|
|
This module contains the routines to manipulate pages within the
|
|
Page Frame Database.
|
|
|
|
Author:
|
|
|
|
Lou Perazzoli (loup) 4-Apr-1989
|
|
Landy Wang (landyw) 02-June-1997
|
|
|
|
Revision History:
|
|
|
|
--*/
|
|
#include "mi.h"
|
|
|
|
//
|
|
// The following line will generate an error is the number of colored
|
|
// free page lists is not 2, ie ZeroedPageList and FreePageList. If
|
|
// this number is changed, the size of the FreeCount array in the kernel
|
|
// node structure (KNODE) must be updated.
|
|
//
|
|
|
|
C_ASSERT(StandbyPageList == 2);
|
|
|
|
#define MM_LOW_LIMIT 2
|
|
|
|
KEVENT MmAvailablePagesEventHigh;
|
|
KEVENT MmAvailablePagesEventMedium;
|
|
|
|
PFN_NUMBER MmTransitionPrivatePages;
|
|
PFN_NUMBER MmTransitionSharedPages;
|
|
|
|
#define MI_TALLY_TRANSITION_PAGE_ADDITION(Pfn) \
|
|
if (Pfn->u3.e1.PrototypePte) { \
|
|
MmTransitionSharedPages += 1; \
|
|
} \
|
|
else { \
|
|
MmTransitionPrivatePages += 1; \
|
|
}
|
|
#if 0
|
|
ASSERT (MmTransitionPrivatePages + MmTransitionSharedPages == MmStandbyPageListHead.Total + MmModifiedPageListHead.Total + MmModifiedNoWritePageListHead.Total);
|
|
#endif
|
|
|
|
#define MI_TALLY_TRANSITION_PAGE_REMOVAL(Pfn) \
|
|
if (Pfn->u3.e1.PrototypePte) { \
|
|
MmTransitionSharedPages -= 1; \
|
|
} \
|
|
else { \
|
|
MmTransitionPrivatePages -= 1; \
|
|
}
|
|
#if 0
|
|
ASSERT (MmTransitionPrivatePages + MmTransitionSharedPages == MmStandbyPageListHead.Total + MmModifiedPageListHead.Total + MmModifiedNoWritePageListHead.Total);
|
|
#endif
|
|
|
|
//
|
|
// This counter is used to determine if standby pages are being cannibalized
|
|
// for use as free pages and therefore more aging should be attempted.
|
|
//
|
|
|
|
ULONG MiStandbyRemoved;
|
|
|
|
#if DBG
|
|
ULONG MiSaveStacks;
|
|
#endif
|
|
|
|
extern ULONG MmSystemShutdown;
|
|
|
|
PFN_NUMBER
|
|
FASTCALL
|
|
MiRemovePageByColor (
|
|
IN PFN_NUMBER Page,
|
|
IN ULONG PageColor
|
|
);
|
|
|
|
VOID
|
|
FASTCALL
|
|
MiLogPfnInformation (
|
|
IN PMMPFN Pfn1,
|
|
IN USHORT Reason
|
|
);
|
|
|
|
extern LOGICAL MiZeroingDisabled;
|
|
|
|
// #define _MI_DEBUG_PFN 1
|
|
|
|
#if defined (_MI_DEBUG_PFN)
|
|
|
|
#define MI_PFN_TRACE_MAX 0x8000
|
|
|
|
#define MI_PFN_BACKTRACE_LENGTH 6
|
|
|
|
typedef struct _MI_PFN_TRACES {
|
|
|
|
PFN_NUMBER PageFrameIndex;
|
|
MMLISTS Destination;
|
|
MMPTE PteContents;
|
|
PVOID Thread;
|
|
|
|
MMPFN Pfn;
|
|
|
|
PVOID StackTrace [MI_PFN_BACKTRACE_LENGTH];
|
|
|
|
} MI_PFN_TRACES, *PMI_PFN_TRACES;
|
|
|
|
LONG MiPfnIndex;
|
|
|
|
PMI_PFN_TRACES MiPfnTraces;
|
|
|
|
ULONG zpfn = 0x1;
|
|
|
|
VOID
|
|
FORCEINLINE
|
|
MiSnapPfn (
|
|
IN PMMPFN Pfn1,
|
|
IN MMLISTS Destination,
|
|
IN ULONG CallerId
|
|
)
|
|
{
|
|
MMPTE PteContents;
|
|
PMMPTE PointerPte;
|
|
PMI_PFN_TRACES Information;
|
|
ULONG Hash;
|
|
ULONG Index;
|
|
PEPROCESS Process;
|
|
|
|
if (MiPfnTraces == NULL) {
|
|
return;
|
|
}
|
|
|
|
if (zpfn & 0x1) {
|
|
|
|
if (Pfn1->PteAddress < MiGetPteAddress (MmPagedPoolStart)) {
|
|
return;
|
|
}
|
|
|
|
if (Pfn1->PteAddress > MiGetPteAddress (MmPagedPoolEnd)) {
|
|
return;
|
|
}
|
|
}
|
|
|
|
if (MmIsAddressValid (Pfn1->PteAddress)) {
|
|
PointerPte = Pfn1->PteAddress;
|
|
PointerPte = (PMMPTE)((ULONG_PTR)PointerPte & ~0x1);
|
|
PteContents = *PointerPte;
|
|
}
|
|
else {
|
|
|
|
//
|
|
// The containing page table page is not valid,
|
|
// map the page into hyperspace and reference it that way.
|
|
//
|
|
|
|
Process = PsGetCurrentProcess ();
|
|
PointerPte = MiMapPageInHyperSpaceAtDpc (Process, Pfn1->u4.PteFrame);
|
|
PointerPte = (PMMPTE)((PCHAR)PointerPte +
|
|
MiGetByteOffset(Pfn1->PteAddress));
|
|
PointerPte = (PMMPTE)((ULONG_PTR)PointerPte & ~0x1);
|
|
PteContents = *PointerPte;
|
|
MiUnmapPageInHyperSpaceFromDpc (Process, PointerPte);
|
|
}
|
|
|
|
Index = InterlockedIncrement(&MiPfnIndex);
|
|
Index &= (MI_PFN_TRACE_MAX - 1);
|
|
Information = &MiPfnTraces[Index];
|
|
Information->PageFrameIndex = (Pfn1 - MmPfnDatabase);
|
|
Information->Destination = Destination;
|
|
Information->PteContents = PteContents;
|
|
Information->Thread = (PVOID)((ULONG_PTR)KeGetCurrentThread() | (CallerId));
|
|
Information->Pfn = *Pfn1;
|
|
RtlCaptureStackBackTrace (0, MI_PFN_BACKTRACE_LENGTH, Information->StackTrace, &Hash);
|
|
}
|
|
|
|
#define MI_SNAP_PFN(_Pfn, dest, callerid) MiSnapPfn(_Pfn, dest, callerid)
|
|
|
|
#else
|
|
#define MI_SNAP_PFN(_Pfn, dest, callerid)
|
|
#endif
|
|
|
|
VOID
|
|
MiInitializePfnTracing (
|
|
VOID
|
|
)
|
|
{
|
|
#if defined (_MI_DEBUG_PFN)
|
|
MiPfnTraces = ExAllocatePoolWithTag (NonPagedPool,
|
|
MI_PFN_TRACE_MAX * sizeof (MI_PFN_TRACES),
|
|
'tCmM');
|
|
#endif
|
|
}
|
|
|
|
|
|
VOID
|
|
FASTCALL
|
|
MiInsertPageInFreeList (
|
|
IN PFN_NUMBER PageFrameIndex
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This procedure inserts a page at the end of the free list.
|
|
|
|
Arguments:
|
|
|
|
PageFrameIndex - Supplies the physical page number to insert in the list.
|
|
|
|
Return Value:
|
|
|
|
None.
|
|
|
|
Environment:
|
|
|
|
PFN lock held.
|
|
|
|
--*/
|
|
|
|
{
|
|
PFN_NUMBER last;
|
|
PMMPFN Pfn1;
|
|
PMMPFN Pfn2;
|
|
ULONG Color;
|
|
MMLISTS ListName;
|
|
PMMPFNLIST ListHead;
|
|
PMMCOLOR_TABLES ColorHead;
|
|
|
|
MM_PFN_LOCK_ASSERT();
|
|
ASSERT ((PageFrameIndex != 0) &&
|
|
(PageFrameIndex <= MmHighestPhysicalPage) &&
|
|
(PageFrameIndex >= MmLowestPhysicalPage));
|
|
|
|
Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
|
|
|
|
MI_SNAP_DATA (Pfn1, Pfn1->PteAddress, 8);
|
|
|
|
MI_SNAP_PFN(Pfn1, FreePageList, 0x1);
|
|
|
|
if (PERFINFO_IS_GROUP_ON(PERF_MEMORY)) {
|
|
MiLogPfnInformation (Pfn1, PERFINFO_LOG_TYPE_INSERTINFREELIST);
|
|
}
|
|
|
|
if (Pfn1->u3.e1.Rom == 1) {
|
|
|
|
//
|
|
// ROM pages do not go on free lists and are not counted towards
|
|
// MmAvailablePages as they are not reusable. Transform these pages
|
|
// into their pre-Phase 0 state and keep them off all lists.
|
|
//
|
|
|
|
ASSERT (XIPConfigured == TRUE);
|
|
|
|
ASSERT (Pfn1->u3.e1.CacheAttribute == MiCached);
|
|
ASSERT (Pfn1->u3.e1.Modified == 0);
|
|
ASSERT (Pfn1->u2.ShareCount == 0);
|
|
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
|
|
ASSERT (Pfn1->u4.InPageError == 0);
|
|
ASSERT (Pfn1->u3.e1.PrototypePte == 1);
|
|
|
|
Pfn1->u1.Flink = 0;
|
|
Pfn1->u3.e1.PageLocation = 0;
|
|
|
|
return;
|
|
}
|
|
|
|
if (Pfn1->u3.e1.RemovalRequested == 1) {
|
|
Pfn1->u3.e1.CacheAttribute = MiNotMapped;
|
|
MiInsertPageInList (&MmBadPageListHead, PageFrameIndex);
|
|
return;
|
|
}
|
|
|
|
ListHead = &MmFreePageListHead;
|
|
|
|
ASSERT (Pfn1->u3.e1.LockCharged == 0);
|
|
|
|
PERFINFO_INSERTINLIST(PageFrameIndex, ListHead);
|
|
|
|
ListName = FreePageList;
|
|
|
|
#if DBG
|
|
#if defined (_X86_)
|
|
if ((MiSaveStacks != 0) && (MmFirstReservedMappingPte != NULL)) {
|
|
|
|
ULONG_PTR StackPointer;
|
|
ULONG_PTR StackBytes;
|
|
PULONG_PTR DataPage;
|
|
PEPROCESS Process;
|
|
|
|
_asm {
|
|
mov StackPointer, esp
|
|
}
|
|
|
|
MiZeroingDisabled = TRUE;
|
|
|
|
Process = PsGetCurrentProcess ();
|
|
|
|
DataPage = MiMapPageInHyperSpaceAtDpc (Process, PageFrameIndex);
|
|
|
|
DataPage[0] = StackPointer;
|
|
|
|
//
|
|
// For now, don't get fancy with copying more than what's in the current
|
|
// stack page. To do so would require checking the thread stack limits,
|
|
// DPC stack limits, etc.
|
|
//
|
|
|
|
StackBytes = PAGE_SIZE - BYTE_OFFSET(StackPointer);
|
|
DataPage[1] = StackBytes;
|
|
|
|
if (StackBytes != 0) {
|
|
|
|
if (StackBytes > MI_STACK_BYTES) {
|
|
StackBytes = MI_STACK_BYTES;
|
|
}
|
|
|
|
RtlCopyMemory ((PVOID)&DataPage[2], (PVOID)StackPointer, StackBytes);
|
|
}
|
|
|
|
MiUnmapPageInHyperSpaceFromDpc (Process, DataPage);
|
|
}
|
|
#endif
|
|
#endif
|
|
|
|
ASSERT (Pfn1->u4.VerifierAllocation == 0);
|
|
|
|
//
|
|
// Check to ensure the reference count for the page is zero.
|
|
//
|
|
|
|
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
|
|
|
|
ListHead->Total += 1; // One more page on the list.
|
|
|
|
last = ListHead->Blink;
|
|
|
|
if (last != MM_EMPTY_LIST) {
|
|
Pfn2 = MI_PFN_ELEMENT (last);
|
|
Pfn2->u1.Flink = PageFrameIndex;
|
|
}
|
|
else {
|
|
|
|
//
|
|
// List is empty, add the page to the ListHead.
|
|
//
|
|
|
|
ListHead->Flink = PageFrameIndex;
|
|
}
|
|
|
|
ListHead->Blink = PageFrameIndex;
|
|
Pfn1->u1.Flink = MM_EMPTY_LIST;
|
|
Pfn1->u2.Blink = last;
|
|
|
|
Pfn1->u3.e1.PageLocation = ListName;
|
|
Pfn1->u3.e1.CacheAttribute = MiNotMapped;
|
|
Pfn1->u4.InPageError = 0;
|
|
|
|
//
|
|
// Update the count of usable pages in the system. If the count
|
|
// transitions from 0 to 1, the event associated with available
|
|
// pages should become true.
|
|
//
|
|
|
|
MmAvailablePages += 1;
|
|
|
|
//
|
|
// A page has just become available, check to see if the
|
|
// page wait events should be signaled.
|
|
//
|
|
|
|
if (MmAvailablePages <= MM_HIGH_LIMIT) {
|
|
if (MmAvailablePages == MM_HIGH_LIMIT) {
|
|
KeSetEvent (&MmAvailablePagesEventHigh, 0, FALSE);
|
|
}
|
|
else if (MmAvailablePages == MM_MEDIUM_LIMIT) {
|
|
KeSetEvent (&MmAvailablePagesEventMedium, 0, FALSE);
|
|
}
|
|
else if (MmAvailablePages == MM_LOW_LIMIT) {
|
|
KeSetEvent (&MmAvailablePagesEvent, 0, FALSE);
|
|
}
|
|
}
|
|
|
|
//
|
|
// Signal applications if the freed page crosses a threshold.
|
|
//
|
|
|
|
if (MmAvailablePages == MmLowMemoryThreshold) {
|
|
KeClearEvent (MiLowMemoryEvent);
|
|
}
|
|
else if (MmAvailablePages == MmHighMemoryThreshold) {
|
|
KeSetEvent (MiHighMemoryEvent, 0, FALSE);
|
|
}
|
|
|
|
#if defined(MI_MULTINODE)
|
|
|
|
//
|
|
// Increment the free page count for this node.
|
|
//
|
|
|
|
if (KeNumberNodes > 1) {
|
|
KeNodeBlock[Pfn1->u3.e1.PageColor]->FreeCount[ListName]++;
|
|
}
|
|
#endif
|
|
|
|
//
|
|
// We are adding a page to the free page list.
|
|
// Add the page to the end of the correct colored page list.
|
|
//
|
|
|
|
Color = MI_GET_COLOR_FROM_LIST_ENTRY(PageFrameIndex, Pfn1);
|
|
|
|
ColorHead = &MmFreePagesByColor[ListName][Color];
|
|
|
|
if (ColorHead->Flink == MM_EMPTY_LIST) {
|
|
|
|
//
|
|
// This list is empty, add this as the first and last
|
|
// entry.
|
|
//
|
|
|
|
ColorHead->Flink = PageFrameIndex;
|
|
ColorHead->Blink = Pfn1;
|
|
}
|
|
else {
|
|
Pfn2 = (PMMPFN)ColorHead->Blink;
|
|
Pfn2->OriginalPte.u.Long = PageFrameIndex;
|
|
ColorHead->Blink = Pfn1;
|
|
}
|
|
ColorHead->Count += 1;
|
|
Pfn1->OriginalPte.u.Long = MM_EMPTY_LIST;
|
|
|
|
if ((ListHead->Total >= MmMinimumFreePagesToZero) &&
|
|
(MmZeroingPageThreadActive == FALSE)) {
|
|
|
|
//
|
|
// There are enough pages on the free list, start
|
|
// the zeroing page thread.
|
|
//
|
|
|
|
MmZeroingPageThreadActive = TRUE;
|
|
KeSetEvent (&MmZeroingPageEvent, 0, FALSE);
|
|
}
|
|
|
|
return;
|
|
}
|
|
|
|
VOID
|
|
FASTCALL
|
|
MiInsertPageInList (
|
|
IN PMMPFNLIST ListHead,
|
|
IN PFN_NUMBER PageFrameIndex
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This procedure inserts a page at the end of the specified list (standby,
|
|
bad, zeroed, modified).
|
|
|
|
Arguments:
|
|
|
|
ListHead - Supplies the list of the list in which to insert the
|
|
specified physical page.
|
|
|
|
PageFrameIndex - Supplies the physical page number to insert in the list.
|
|
|
|
Return Value:
|
|
|
|
None.
|
|
|
|
Environment:
|
|
|
|
Must be holding the PFN database lock with APCs disabled.
|
|
|
|
--*/
|
|
|
|
{
|
|
PFN_NUMBER last;
|
|
PMMPFN Pfn1;
|
|
PMMPFN Pfn2;
|
|
ULONG Color;
|
|
MMLISTS ListName;
|
|
#if MI_BARRIER_SUPPORTED
|
|
ULONG BarrierStamp;
|
|
#endif
|
|
|
|
ASSERT (ListHead != &MmFreePageListHead);
|
|
|
|
MM_PFN_LOCK_ASSERT();
|
|
ASSERT ((PageFrameIndex != 0) &&
|
|
(PageFrameIndex <= MmHighestPhysicalPage) &&
|
|
(PageFrameIndex >= MmLowestPhysicalPage));
|
|
|
|
Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
|
|
|
|
ASSERT (Pfn1->u3.e1.LockCharged == 0);
|
|
|
|
PERFINFO_INSERTINLIST(PageFrameIndex, ListHead);
|
|
|
|
ListName = ListHead->ListName;
|
|
|
|
MI_SNAP_DATA (Pfn1, Pfn1->PteAddress, 0x20 + ListName);
|
|
|
|
MI_SNAP_PFN(Pfn1, ListName, 0x2);
|
|
|
|
#if DBG
|
|
if (MmDebug & MM_DBG_PAGE_REF_COUNT) {
|
|
|
|
if ((ListName == StandbyPageList) || (ListName == ModifiedPageList)) {
|
|
|
|
PMMPTE PointerPte;
|
|
PEPROCESS Process;
|
|
|
|
if ((Pfn1->u3.e1.PrototypePte == 1) &&
|
|
(MmIsAddressValid (Pfn1->PteAddress))) {
|
|
Process = NULL;
|
|
PointerPte = Pfn1->PteAddress;
|
|
}
|
|
else {
|
|
|
|
//
|
|
// The page containing the prototype PTE is not valid,
|
|
// map the page into hyperspace and reference it that way.
|
|
//
|
|
|
|
Process = PsGetCurrentProcess ();
|
|
PointerPte = MiMapPageInHyperSpaceAtDpc (Process, Pfn1->u4.PteFrame);
|
|
PointerPte = (PMMPTE)((PCHAR)PointerPte +
|
|
MiGetByteOffset(Pfn1->PteAddress));
|
|
}
|
|
|
|
ASSERT ((MI_GET_PAGE_FRAME_FROM_TRANSITION_PTE (PointerPte) == PageFrameIndex) ||
|
|
(MI_GET_PAGE_FRAME_FROM_PTE (PointerPte) == PageFrameIndex));
|
|
ASSERT (PointerPte->u.Soft.Transition == 1);
|
|
ASSERT (PointerPte->u.Soft.Prototype == 0);
|
|
if (Process != NULL) {
|
|
MiUnmapPageInHyperSpaceFromDpc (Process, PointerPte);
|
|
}
|
|
}
|
|
}
|
|
|
|
if ((ListName == StandbyPageList) || (ListName == ModifiedPageList)) {
|
|
if ((Pfn1->OriginalPte.u.Soft.Prototype == 0) &&
|
|
(Pfn1->OriginalPte.u.Soft.Transition == 1)) {
|
|
KeBugCheckEx (MEMORY_MANAGEMENT, 0x8888, 0,0,0);
|
|
}
|
|
}
|
|
#endif
|
|
|
|
//
|
|
// Check to ensure the reference count for the page is zero.
|
|
//
|
|
|
|
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
|
|
|
|
if (Pfn1->u3.e1.Rom == 1) {
|
|
|
|
//
|
|
// ROM pages do not go on transition lists and are not counted towards
|
|
// MmAvailablePages as they are not reusable. Migrate these pages
|
|
// into a separate list but keep the PageLocation as standby.
|
|
//
|
|
|
|
ASSERT (XIPConfigured == TRUE);
|
|
ASSERT (Pfn1->u3.e1.Modified == 0);
|
|
ASSERT (Pfn1->u3.e1.CacheAttribute == MiCached);
|
|
|
|
ListHead = &MmRomPageListHead;
|
|
ListHead->Total += 1; // One more page on the list.
|
|
last = ListHead->Blink;
|
|
|
|
if (last != MM_EMPTY_LIST) {
|
|
Pfn2 = MI_PFN_ELEMENT (last);
|
|
Pfn2->u1.Flink = PageFrameIndex;
|
|
}
|
|
else {
|
|
|
|
//
|
|
// List is empty, add the page to the ListHead.
|
|
//
|
|
|
|
ListHead->Flink = PageFrameIndex;
|
|
}
|
|
|
|
ListHead->Blink = PageFrameIndex;
|
|
Pfn1->u1.Flink = MM_EMPTY_LIST;
|
|
Pfn1->u2.Blink = last;
|
|
|
|
Pfn1->u3.e1.PageLocation = ListName;
|
|
|
|
return;
|
|
}
|
|
|
|
ListHead->Total += 1; // One more page on the list.
|
|
|
|
if (ListHead == &MmModifiedPageListHead) {
|
|
|
|
//
|
|
// Leave the page as cached as it may need to be written out at some
|
|
// point and presumably the driver stack will need to map it at that
|
|
// time.
|
|
//
|
|
|
|
ASSERT (Pfn1->u3.e1.CacheAttribute == MiCached);
|
|
|
|
//
|
|
// On MIPS R4000 modified pages destined for the paging file are
|
|
// kept on separate lists which group pages of the same color
|
|
// together.
|
|
//
|
|
|
|
if (Pfn1->OriginalPte.u.Soft.Prototype == 0) {
|
|
|
|
//
|
|
// This page is destined for the paging file (not
|
|
// a mapped file). Change the list head to the
|
|
// appropriate colored list head.
|
|
//
|
|
|
|
#if MM_MAXIMUM_NUMBER_OF_COLORS > 1
|
|
ListHead = &MmModifiedPageListByColor [Pfn1->u3.e1.PageColor];
|
|
#else
|
|
ListHead = &MmModifiedPageListByColor [0];
|
|
#endif
|
|
ASSERT (ListHead->ListName == ListName);
|
|
ListHead->Total += 1;
|
|
MmTotalPagesForPagingFile += 1;
|
|
}
|
|
else {
|
|
|
|
//
|
|
// This page is destined for a mapped file (not
|
|
// the paging file). If there are no other pages currently
|
|
// destined for the mapped file, start our timer so that we can
|
|
// ensure that these pages make it to disk even if we don't pile
|
|
// up enough of them to trigger the modified page writer or need
|
|
// the memory. If we don't do this here, then for this scenario,
|
|
// only an orderly system shutdown will write them out (days,
|
|
// weeks, months or years later) and any power out in between
|
|
// means we'll have lost the data.
|
|
//
|
|
|
|
if (ListHead->Total - MmTotalPagesForPagingFile == 1) {
|
|
|
|
//
|
|
// Start the DPC timer because we're the first on the list.
|
|
//
|
|
|
|
if (MiTimerPending == FALSE) {
|
|
MiTimerPending = TRUE;
|
|
|
|
KeSetTimerEx (&MiModifiedPageWriterTimer,
|
|
MiModifiedPageLife,
|
|
0,
|
|
&MiModifiedPageWriterTimerDpc);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
else if ((Pfn1->u3.e1.RemovalRequested == 1) &&
|
|
(ListName <= StandbyPageList)) {
|
|
|
|
ListHead->Total -= 1; // Undo previous increment
|
|
|
|
if (ListName == StandbyPageList) {
|
|
Pfn1->u3.e1.PageLocation = StandbyPageList;
|
|
MiRestoreTransitionPte (PageFrameIndex);
|
|
}
|
|
|
|
Pfn1->u3.e1.CacheAttribute = MiNotMapped;
|
|
|
|
ListHead = &MmBadPageListHead;
|
|
ASSERT (ListHead->ListName == BadPageList);
|
|
ListHead->Total += 1; // One more page on the list.
|
|
ListName = BadPageList;
|
|
}
|
|
|
|
last = ListHead->Blink;
|
|
|
|
if (last != MM_EMPTY_LIST) {
|
|
Pfn2 = MI_PFN_ELEMENT (last);
|
|
Pfn2->u1.Flink = PageFrameIndex;
|
|
}
|
|
else {
|
|
|
|
//
|
|
// List is empty, add the page to the ListHead.
|
|
//
|
|
|
|
ListHead->Flink = PageFrameIndex;
|
|
}
|
|
|
|
ListHead->Blink = PageFrameIndex;
|
|
Pfn1->u1.Flink = MM_EMPTY_LIST;
|
|
Pfn1->u2.Blink = last;
|
|
|
|
Pfn1->u3.e1.PageLocation = ListName;
|
|
|
|
//
|
|
// If the page was placed on the standby or zeroed list,
|
|
// update the count of usable pages in the system. If the count
|
|
// transitions from 0 to 1, the event associated with available
|
|
// pages should become true.
|
|
//
|
|
|
|
if (ListName <= StandbyPageList) {
|
|
|
|
MmAvailablePages += 1;
|
|
|
|
//
|
|
// A page has just become available, check to see if the
|
|
// page wait events should be signaled.
|
|
//
|
|
|
|
if (MmAvailablePages <= MM_HIGH_LIMIT) {
|
|
if (MmAvailablePages == MM_HIGH_LIMIT) {
|
|
KeSetEvent (&MmAvailablePagesEventHigh, 0, FALSE);
|
|
}
|
|
else if (MmAvailablePages == MM_MEDIUM_LIMIT) {
|
|
KeSetEvent (&MmAvailablePagesEventMedium, 0, FALSE);
|
|
}
|
|
else if (MmAvailablePages == MM_LOW_LIMIT) {
|
|
KeSetEvent (&MmAvailablePagesEvent, 0, FALSE);
|
|
}
|
|
}
|
|
|
|
//
|
|
// Signal applications if the freed page crosses a threshold.
|
|
//
|
|
|
|
if (MmAvailablePages == MmLowMemoryThreshold) {
|
|
KeClearEvent (MiLowMemoryEvent);
|
|
}
|
|
else if (MmAvailablePages == MmHighMemoryThreshold) {
|
|
KeSetEvent (MiHighMemoryEvent, 0, FALSE);
|
|
}
|
|
|
|
if (ListName <= FreePageList) {
|
|
|
|
PMMCOLOR_TABLES ColorHead;
|
|
|
|
ASSERT (ListName == ZeroedPageList);
|
|
ASSERT (Pfn1->u4.InPageError == 0);
|
|
|
|
ASSERT (Pfn1->u3.e1.CacheAttribute == MiNotMapped);
|
|
|
|
#if defined(MI_MULTINODE)
|
|
|
|
//
|
|
// Increment the zero page count for this node.
|
|
//
|
|
|
|
if (KeNumberNodes > 1) {
|
|
KeNodeBlock[Pfn1->u3.e1.PageColor]->FreeCount[ListName]++;
|
|
}
|
|
#endif
|
|
|
|
//
|
|
// We are adding a page to the zeroed page list.
|
|
// Add the page to the end of the correct colored page list.
|
|
//
|
|
|
|
Color = MI_GET_COLOR_FROM_LIST_ENTRY(PageFrameIndex, Pfn1);
|
|
|
|
ColorHead = &MmFreePagesByColor[ListName][Color];
|
|
|
|
if (ColorHead->Flink == MM_EMPTY_LIST) {
|
|
|
|
//
|
|
// This list is empty, add this as the first and last
|
|
// entry.
|
|
//
|
|
|
|
ColorHead->Flink = PageFrameIndex;
|
|
ColorHead->Blink = (PVOID)Pfn1;
|
|
}
|
|
else {
|
|
Pfn2 = (PMMPFN)ColorHead->Blink;
|
|
Pfn2->OriginalPte.u.Long = PageFrameIndex;
|
|
ColorHead->Blink = (PVOID)Pfn1;
|
|
}
|
|
Pfn1->OriginalPte.u.Long = MM_EMPTY_LIST;
|
|
|
|
ColorHead->Count += 1;
|
|
|
|
#if MI_BARRIER_SUPPORTED
|
|
if (ListName == ZeroedPageList) {
|
|
MI_BARRIER_STAMP_ZEROED_PAGE (&BarrierStamp);
|
|
Pfn1->u4.PteFrame = BarrierStamp;
|
|
}
|
|
#endif
|
|
}
|
|
else {
|
|
|
|
//
|
|
// Transition page list so tally it appropriately.
|
|
//
|
|
|
|
ASSERT (Pfn1->u3.e1.CacheAttribute == MiCached);
|
|
MI_TALLY_TRANSITION_PAGE_ADDITION (Pfn1);
|
|
}
|
|
|
|
return;
|
|
}
|
|
|
|
//
|
|
// Check to see if there are too many modified pages.
|
|
//
|
|
|
|
if (ListName == ModifiedPageList) {
|
|
|
|
ASSERT (Pfn1->u3.e1.CacheAttribute == MiCached);
|
|
|
|
//
|
|
// Transition page list so tally it appropriately.
|
|
//
|
|
|
|
MI_TALLY_TRANSITION_PAGE_ADDITION (Pfn1);
|
|
|
|
if (Pfn1->OriginalPte.u.Soft.Prototype == 0) {
|
|
ASSERT (Pfn1->OriginalPte.u.Soft.PageFileHigh == 0);
|
|
}
|
|
|
|
PsGetCurrentProcess()->ModifiedPageCount += 1;
|
|
|
|
if ((MmModifiedPageListHead.Total >= MmModifiedPageMaximum) &&
|
|
(MmAvailablePages < MmMoreThanEnoughFreePages)) {
|
|
|
|
//
|
|
// Start the modified page writer.
|
|
//
|
|
|
|
KeSetEvent (&MmModifiedPageWriterEvent, 0, FALSE);
|
|
}
|
|
}
|
|
else if (ListName == ModifiedNoWritePageList) {
|
|
ASSERT (Pfn1->u3.e1.CacheAttribute == MiCached);
|
|
MI_TALLY_TRANSITION_PAGE_ADDITION (Pfn1);
|
|
}
|
|
|
|
return;
|
|
}
|
|
|
|
|
|
VOID
|
|
FASTCALL
|
|
MiInsertStandbyListAtFront (
|
|
IN PFN_NUMBER PageFrameIndex
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This procedure inserts a page at the front of the standby list.
|
|
|
|
Arguments:
|
|
|
|
PageFrameIndex - Supplies the physical page number to insert in the list.
|
|
|
|
Return Value:
|
|
|
|
None.
|
|
|
|
Environment:
|
|
|
|
PFN lock held.
|
|
|
|
--*/
|
|
|
|
{
|
|
PFN_NUMBER first;
|
|
PFN_NUMBER last;
|
|
IN PMMPFNLIST ListHead;
|
|
PMMPFN Pfn1;
|
|
PMMPFN Pfn2;
|
|
|
|
MM_PFN_LOCK_ASSERT();
|
|
ASSERT ((PageFrameIndex != 0) && (PageFrameIndex <= MmHighestPhysicalPage) &&
|
|
(PageFrameIndex >= MmLowestPhysicalPage));
|
|
|
|
Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
|
|
|
|
MI_SNAP_DATA (Pfn1, Pfn1->PteAddress, 9);
|
|
|
|
PERFINFO_INSERT_FRONT_STANDBY(PageFrameIndex);
|
|
|
|
#if DBG
|
|
if (MmDebug & MM_DBG_PAGE_REF_COUNT) {
|
|
|
|
PMMPTE PointerPte;
|
|
PEPROCESS Process;
|
|
|
|
if ((Pfn1->u3.e1.PrototypePte == 1) &&
|
|
(MmIsAddressValid (Pfn1->PteAddress))) {
|
|
PointerPte = Pfn1->PteAddress;
|
|
Process = NULL;
|
|
}
|
|
else {
|
|
|
|
//
|
|
// The page containing the prototype PTE is not valid,
|
|
// map the page into hyperspace and reference it that way.
|
|
//
|
|
|
|
Process = PsGetCurrentProcess ();
|
|
PointerPte = MiMapPageInHyperSpaceAtDpc (Process, Pfn1->u4.PteFrame);
|
|
PointerPte = (PMMPTE)((PCHAR)PointerPte +
|
|
MiGetByteOffset(Pfn1->PteAddress));
|
|
}
|
|
|
|
ASSERT ((MI_GET_PAGE_FRAME_FROM_TRANSITION_PTE (PointerPte) == PageFrameIndex) ||
|
|
(MI_GET_PAGE_FRAME_FROM_PTE (PointerPte) == PageFrameIndex));
|
|
ASSERT (PointerPte->u.Soft.Transition == 1);
|
|
ASSERT (PointerPte->u.Soft.Prototype == 0);
|
|
if (Process != NULL) {
|
|
MiUnmapPageInHyperSpaceFromDpc (Process, PointerPte);
|
|
}
|
|
}
|
|
|
|
if ((Pfn1->OriginalPte.u.Soft.Prototype == 0) &&
|
|
(Pfn1->OriginalPte.u.Soft.Transition == 1)) {
|
|
KeBugCheckEx (MEMORY_MANAGEMENT, 0x8889, 0,0,0);
|
|
}
|
|
#endif
|
|
|
|
//
|
|
// Check to ensure the reference count for the page is zero.
|
|
//
|
|
|
|
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
|
|
ASSERT (Pfn1->u3.e1.PrototypePte == 1);
|
|
ASSERT (Pfn1->u3.e1.CacheAttribute == MiCached);
|
|
|
|
if (Pfn1->u3.e1.Rom == 1) {
|
|
|
|
//
|
|
// ROM pages do not go on transition lists and are not counted towards
|
|
// MmAvailablePages as they are not reusable. Migrate these pages
|
|
// into a separate list but keep the PageLocation as standby. Note
|
|
// it doesn't matter if the page is put at the head or the tail since
|
|
// it's not reusable.
|
|
//
|
|
|
|
ASSERT (XIPConfigured == TRUE);
|
|
ASSERT (Pfn1->u3.e1.Modified == 0);
|
|
|
|
ListHead = &MmRomPageListHead;
|
|
ListHead->Total += 1; // One more page on the list.
|
|
last = ListHead->Blink;
|
|
|
|
if (last != MM_EMPTY_LIST) {
|
|
Pfn2 = MI_PFN_ELEMENT (last);
|
|
Pfn2->u1.Flink = PageFrameIndex;
|
|
}
|
|
else {
|
|
|
|
//
|
|
// List is empty, add the page to the ListHead.
|
|
//
|
|
|
|
ListHead->Flink = PageFrameIndex;
|
|
}
|
|
|
|
ListHead->Blink = PageFrameIndex;
|
|
Pfn1->u1.Flink = MM_EMPTY_LIST;
|
|
Pfn1->u2.Blink = last;
|
|
|
|
Pfn1->u3.e1.PageLocation = StandbyPageList;
|
|
|
|
return;
|
|
}
|
|
|
|
MmTransitionSharedPages += 1;
|
|
|
|
MmStandbyPageListHead.Total += 1; // One more page on the list.
|
|
|
|
ASSERT (MmTransitionPrivatePages + MmTransitionSharedPages == MmStandbyPageListHead.Total + MmModifiedPageListHead.Total + MmModifiedNoWritePageListHead.Total);
|
|
|
|
ListHead = &MmStandbyPageListHead;
|
|
|
|
first = ListHead->Flink;
|
|
if (first == MM_EMPTY_LIST) {
|
|
|
|
//
|
|
// List is empty add the page to the ListHead.
|
|
//
|
|
|
|
ListHead->Blink = PageFrameIndex;
|
|
}
|
|
else {
|
|
Pfn2 = MI_PFN_ELEMENT (first);
|
|
Pfn2->u2.Blink = PageFrameIndex;
|
|
}
|
|
|
|
ListHead->Flink = PageFrameIndex;
|
|
Pfn1->u2.Blink = MM_EMPTY_LIST;
|
|
Pfn1->u1.Flink = first;
|
|
Pfn1->u3.e1.PageLocation = StandbyPageList;
|
|
|
|
//
|
|
// If the page was placed on the free, standby or zeroed list,
|
|
// update the count of usable pages in the system. If the count
|
|
// transitions from 0 to 1, the event associated with available
|
|
// pages should become true.
|
|
//
|
|
|
|
MmAvailablePages += 1;
|
|
|
|
//
|
|
// A page has just become available, check to see if the
|
|
// page wait events should be signalled.
|
|
//
|
|
|
|
if (MmAvailablePages <= MM_HIGH_LIMIT) {
|
|
if (MmAvailablePages == MM_HIGH_LIMIT) {
|
|
KeSetEvent (&MmAvailablePagesEventHigh, 0, FALSE);
|
|
}
|
|
else if (MmAvailablePages == MM_MEDIUM_LIMIT) {
|
|
KeSetEvent (&MmAvailablePagesEventMedium, 0, FALSE);
|
|
}
|
|
else if (MmAvailablePages == MM_LOW_LIMIT) {
|
|
KeSetEvent (&MmAvailablePagesEvent, 0, FALSE);
|
|
}
|
|
}
|
|
|
|
//
|
|
// Signal applications if the freed page crosses a threshold.
|
|
//
|
|
|
|
if (MmAvailablePages == MmLowMemoryThreshold) {
|
|
KeClearEvent (MiLowMemoryEvent);
|
|
}
|
|
else if (MmAvailablePages == MmHighMemoryThreshold) {
|
|
KeSetEvent (MiHighMemoryEvent, 0, FALSE);
|
|
}
|
|
|
|
return;
|
|
}
|
|
|
|
PFN_NUMBER
|
|
FASTCALL
|
|
MiRemovePageFromList (
|
|
IN PMMPFNLIST ListHead
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This procedure removes a page from the head of the specified list (free,
|
|
standby or zeroed).
|
|
|
|
This routine clears the flags word in the PFN database, hence the
|
|
PFN information for this page must be initialized.
|
|
|
|
Arguments:
|
|
|
|
ListHead - Supplies the list of the list in which to remove the
|
|
specified physical page.
|
|
|
|
Return Value:
|
|
|
|
The physical page number removed from the specified list.
|
|
|
|
Environment:
|
|
|
|
PFN lock held.
|
|
|
|
--*/
|
|
|
|
{
|
|
PMMCOLOR_TABLES ColorHead;
|
|
PFN_NUMBER PageFrameIndex;
|
|
PMMPFN Pfn1;
|
|
PMMPFN Pfn2;
|
|
ULONG Color;
|
|
MMLISTS ListName;
|
|
|
|
MM_PFN_LOCK_ASSERT();
|
|
|
|
//
|
|
// If the specified list is empty return MM_EMPTY_LIST.
|
|
//
|
|
|
|
if (ListHead->Total == 0) {
|
|
KeBugCheckEx (PFN_LIST_CORRUPT, 1, (ULONG_PTR)ListHead, MmAvailablePages, 0);
|
|
}
|
|
|
|
ListName = ListHead->ListName;
|
|
ASSERT (ListName != ModifiedPageList);
|
|
|
|
//
|
|
// Decrement the count of pages on the list and remove the first
|
|
// page from the list.
|
|
//
|
|
|
|
ListHead->Total -= 1;
|
|
PageFrameIndex = ListHead->Flink;
|
|
Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
|
|
|
|
if (PERFINFO_IS_GROUP_ON(PERF_MEMORY)) {
|
|
MiLogPfnInformation (Pfn1, PERFINFO_LOG_TYPE_REMOVEPAGEFROMLIST);
|
|
}
|
|
|
|
ListHead->Flink = Pfn1->u1.Flink;
|
|
|
|
//
|
|
// Zero the flink and blink in the PFN database element.
|
|
//
|
|
|
|
Pfn1->u1.Flink = 0; // Assumes Flink width is >= WsIndex width
|
|
Pfn1->u2.Blink = 0;
|
|
|
|
//
|
|
// If the last page was removed (the ListHead->Flink is now
|
|
// MM_EMPTY_LIST) make the Listhead->Blink MM_EMPTY_LIST as well.
|
|
//
|
|
|
|
if (ListHead->Flink != MM_EMPTY_LIST) {
|
|
|
|
//
|
|
// Make the PFN element blink point to MM_EMPTY_LIST signifying this
|
|
// is the first page in the list.
|
|
//
|
|
|
|
Pfn2 = MI_PFN_ELEMENT (ListHead->Flink);
|
|
Pfn2->u2.Blink = MM_EMPTY_LIST;
|
|
}
|
|
else {
|
|
ListHead->Blink = MM_EMPTY_LIST;
|
|
}
|
|
|
|
//
|
|
// Check to see if we now have one less page available.
|
|
//
|
|
|
|
if (ListName <= StandbyPageList) {
|
|
MmAvailablePages -= 1;
|
|
|
|
if (ListName == StandbyPageList) {
|
|
|
|
//
|
|
// This page is currently in transition, restore the PTE to
|
|
// its original contents so this page can be reused.
|
|
//
|
|
|
|
MI_TALLY_TRANSITION_PAGE_REMOVAL (Pfn1);
|
|
MiRestoreTransitionPte (PageFrameIndex);
|
|
}
|
|
|
|
if (MmAvailablePages < MmMinimumFreePages) {
|
|
|
|
//
|
|
// Obtain free pages.
|
|
//
|
|
|
|
MiObtainFreePages();
|
|
}
|
|
}
|
|
|
|
ASSERT ((PageFrameIndex != 0) &&
|
|
(PageFrameIndex <= MmHighestPhysicalPage) &&
|
|
(PageFrameIndex >= MmLowestPhysicalPage));
|
|
|
|
//
|
|
// Zero the PFN flags longword.
|
|
//
|
|
|
|
ASSERT (Pfn1->u3.e1.CacheAttribute == MiNotMapped);
|
|
Color = Pfn1->u3.e1.PageColor;
|
|
ASSERT (Pfn1->u3.e1.RemovalRequested == 0);
|
|
ASSERT (Pfn1->u3.e1.Rom == 0);
|
|
Pfn1->u3.e2.ShortFlags = 0;
|
|
Pfn1->u3.e1.PageColor = Color;
|
|
Pfn1->u3.e1.CacheAttribute = MiNotMapped;
|
|
|
|
if (ListName <= FreePageList) {
|
|
|
|
//
|
|
// Update the color lists.
|
|
//
|
|
|
|
Color = MI_GET_COLOR_FROM_LIST_ENTRY(PageFrameIndex, Pfn1);
|
|
ColorHead = &MmFreePagesByColor[ListName][Color];
|
|
ASSERT (ColorHead->Flink == PageFrameIndex);
|
|
ColorHead->Flink = (PFN_NUMBER) Pfn1->OriginalPte.u.Long;
|
|
ASSERT (ColorHead->Count >= 1);
|
|
ColorHead->Count -= 1;
|
|
}
|
|
|
|
return PageFrameIndex;
|
|
}
|
|
|
|
VOID
|
|
FASTCALL
|
|
MiUnlinkPageFromList (
|
|
IN PMMPFN Pfn
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This procedure removes a page from the middle of a list. This is
|
|
designed for the faulting of transition pages from the standby and
|
|
modified list and making them active and valid again.
|
|
|
|
Arguments:
|
|
|
|
Pfn - Supplies a pointer to the PFN database element for the physical
|
|
page to remove from the list.
|
|
|
|
Return Value:
|
|
|
|
none.
|
|
|
|
Environment:
|
|
|
|
Must be holding the PFN database lock with APCs disabled.
|
|
|
|
--*/
|
|
|
|
{
|
|
PMMPFNLIST ListHead;
|
|
PFN_NUMBER Previous;
|
|
PFN_NUMBER Next;
|
|
PMMPFN Pfn2;
|
|
MMLISTS ListName;
|
|
|
|
MM_PFN_LOCK_ASSERT();
|
|
|
|
PERFINFO_UNLINKPAGE((ULONG_PTR)(Pfn - MmPfnDatabase), Pfn->u3.e1.PageLocation);
|
|
|
|
//
|
|
// Page not on standby or modified list, check to see if the
|
|
// page is currently being written by the modified page
|
|
// writer, if so, just return this page. The reference
|
|
// count for the page will be incremented, so when the modified
|
|
// page write completes, the page will not be put back on
|
|
// the list, rather, it will remain active and valid.
|
|
//
|
|
|
|
if (Pfn->u3.e2.ReferenceCount > 0) {
|
|
|
|
//
|
|
// The page was not on any "transition lists", check to see
|
|
// if this has I/O in progress.
|
|
//
|
|
|
|
if (Pfn->u2.ShareCount == 0) {
|
|
#if DBG
|
|
if (MmDebug & MM_DBG_PAGE_IN_LIST) {
|
|
DbgPrint("unlinking page not in list...\n");
|
|
MiFormatPfn(Pfn);
|
|
}
|
|
#endif
|
|
return;
|
|
}
|
|
KdPrint(("MM:attempt to remove page from wrong page list\n"));
|
|
KeBugCheckEx (PFN_LIST_CORRUPT,
|
|
2,
|
|
Pfn - MmPfnDatabase,
|
|
MmHighestPhysicalPage,
|
|
Pfn->u3.e2.ReferenceCount);
|
|
}
|
|
|
|
ListHead = MmPageLocationList[Pfn->u3.e1.PageLocation];
|
|
|
|
//
|
|
// Must not remove pages from free or zeroed without updating
|
|
// the colored lists.
|
|
//
|
|
|
|
ListName = ListHead->ListName;
|
|
ASSERT (ListName >= StandbyPageList);
|
|
|
|
//
|
|
// If memory mirroring is in progress, any additions or removals to the
|
|
// free, zeroed, standby, modified or modified-no-write lists must
|
|
// update the bitmap.
|
|
//
|
|
|
|
if (MiMirroringActive == TRUE) {
|
|
RtlSetBit (MiMirrorBitMap2, (ULONG)(Pfn - MmPfnDatabase));
|
|
}
|
|
|
|
ASSERT (Pfn->u3.e1.CacheAttribute == MiCached);
|
|
|
|
if (ListHead == &MmStandbyPageListHead) {
|
|
if (Pfn->u3.e1.Rom == 1) {
|
|
ASSERT (XIPConfigured == TRUE);
|
|
ASSERT (Pfn->u3.e1.Modified == 0);
|
|
|
|
ListHead = &MmRomPageListHead;
|
|
|
|
//
|
|
// Deliberately switch to an invalid page listname so the checks
|
|
// below do not alter available pages, etc.
|
|
//
|
|
|
|
ListName = ActiveAndValid;
|
|
}
|
|
}
|
|
else if ((ListHead == &MmModifiedPageListHead) &&
|
|
(Pfn->OriginalPte.u.Soft.Prototype == 0)) {
|
|
|
|
//
|
|
// On MIPS R4000 modified pages destined for the paging file are
|
|
// kept on separate lists which group pages of the same color
|
|
// together.
|
|
//
|
|
|
|
//
|
|
// This page is destined for the paging file (not
|
|
// a mapped file). Change the list head to the
|
|
// appropriate colored list head.
|
|
//
|
|
|
|
ListHead->Total -= 1;
|
|
MmTotalPagesForPagingFile -= 1;
|
|
#if MM_MAXIMUM_NUMBER_OF_COLORS > 1
|
|
ListHead = &MmModifiedPageListByColor [Pfn->u3.e1.PageColor];
|
|
#else
|
|
ListHead = &MmModifiedPageListByColor [0];
|
|
#endif
|
|
ASSERT (ListHead->ListName == ListName);
|
|
}
|
|
|
|
ASSERT (Pfn->u3.e1.WriteInProgress == 0);
|
|
ASSERT (Pfn->u3.e1.ReadInProgress == 0);
|
|
ASSERT (ListHead->Total != 0);
|
|
|
|
Next = Pfn->u1.Flink;
|
|
Pfn->u1.Flink = 0; // Assumes Flink width is >= WsIndex width
|
|
Previous = Pfn->u2.Blink;
|
|
Pfn->u2.Blink = 0;
|
|
|
|
if (Next != MM_EMPTY_LIST) {
|
|
Pfn2 = MI_PFN_ELEMENT(Next);
|
|
Pfn2->u2.Blink = Previous;
|
|
}
|
|
else {
|
|
ListHead->Blink = Previous;
|
|
}
|
|
|
|
if (Previous != MM_EMPTY_LIST) {
|
|
Pfn2 = MI_PFN_ELEMENT(Previous);
|
|
Pfn2->u1.Flink = Next;
|
|
}
|
|
else {
|
|
ListHead->Flink = Next;
|
|
}
|
|
|
|
ListHead->Total -= 1;
|
|
|
|
//
|
|
// Check to see if we now have one less page available.
|
|
//
|
|
|
|
if (ListName <= StandbyPageList) {
|
|
MmAvailablePages -= 1;
|
|
|
|
if (ListName == StandbyPageList) {
|
|
MI_TALLY_TRANSITION_PAGE_REMOVAL (Pfn);
|
|
}
|
|
|
|
if (MmAvailablePages < MmMinimumFreePages) {
|
|
|
|
//
|
|
// Obtain free pages.
|
|
//
|
|
|
|
MiObtainFreePages();
|
|
|
|
}
|
|
}
|
|
else if (ListName == ModifiedPageList || ListName == ModifiedNoWritePageList) {
|
|
MI_TALLY_TRANSITION_PAGE_REMOVAL (Pfn);
|
|
}
|
|
|
|
return;
|
|
}
|
|
|
|
VOID
|
|
MiUnlinkFreeOrZeroedPage (
|
|
IN PFN_NUMBER Page
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This procedure removes a page from the middle of a list. This is
|
|
designed for the removing of free or zeroed pages from the middle of
|
|
their lists.
|
|
|
|
Arguments:
|
|
|
|
Page - Supplies a page frame index to remove from the list.
|
|
|
|
Return Value:
|
|
|
|
None.
|
|
|
|
Environment:
|
|
|
|
Must be holding the PFN database lock with APCs disabled.
|
|
|
|
--*/
|
|
|
|
{
|
|
PMMPFNLIST ListHead;
|
|
PFN_NUMBER Previous;
|
|
PFN_NUMBER Next;
|
|
PMMPFN Pfn2;
|
|
PMMPFN Pfn;
|
|
ULONG Color;
|
|
PMMCOLOR_TABLES ColorHead;
|
|
MMLISTS ListName;
|
|
|
|
Pfn = MI_PFN_ELEMENT (Page);
|
|
|
|
MM_PFN_LOCK_ASSERT();
|
|
|
|
ListHead = MmPageLocationList[Pfn->u3.e1.PageLocation];
|
|
ListName = ListHead->ListName;
|
|
ASSERT (ListHead->Total != 0);
|
|
ListHead->Total -= 1;
|
|
|
|
ASSERT (ListName <= FreePageList);
|
|
ASSERT (Pfn->u3.e1.WriteInProgress == 0);
|
|
ASSERT (Pfn->u3.e1.ReadInProgress == 0);
|
|
ASSERT (Pfn->u3.e1.CacheAttribute == MiNotMapped);
|
|
|
|
//
|
|
// If memory mirroring is in progress, any removals from the
|
|
// free, zeroed, standby, modified or modified-no-write lists that
|
|
// isn't immediately re-inserting into one of these 5 lists (WITHOUT
|
|
// modifying the page contents) must update the bitmap.
|
|
//
|
|
|
|
if (MiMirroringActive == TRUE) {
|
|
RtlSetBit (MiMirrorBitMap2, (ULONG)Page);
|
|
}
|
|
|
|
PERFINFO_UNLINKFREEPAGE (Page, Pfn->u3.e1.PageLocation);
|
|
|
|
Next = Pfn->u1.Flink;
|
|
Pfn->u1.Flink = 0; // Assumes Flink width is >= WsIndex width
|
|
Previous = Pfn->u2.Blink;
|
|
Pfn->u2.Blink = 0;
|
|
|
|
if (Next != MM_EMPTY_LIST) {
|
|
Pfn2 = MI_PFN_ELEMENT(Next);
|
|
Pfn2->u2.Blink = Previous;
|
|
}
|
|
else {
|
|
ListHead->Blink = Previous;
|
|
}
|
|
|
|
if (Previous == MM_EMPTY_LIST) {
|
|
ListHead->Flink = Next;
|
|
}
|
|
else {
|
|
Pfn2 = MI_PFN_ELEMENT(Previous);
|
|
Pfn2->u1.Flink = Next;
|
|
}
|
|
|
|
//
|
|
// We are removing a page from the middle of the free or zeroed page list.
|
|
// The secondary color tables must be updated at this time.
|
|
//
|
|
|
|
Color = MI_GET_COLOR_FROM_LIST_ENTRY(Page, Pfn);
|
|
|
|
//
|
|
// Walk down the list and remove the page.
|
|
//
|
|
|
|
ColorHead = &MmFreePagesByColor[ListName][Color];
|
|
Next = ColorHead->Flink;
|
|
if (Next == Page) {
|
|
ColorHead->Flink = (PFN_NUMBER) Pfn->OriginalPte.u.Long;
|
|
}
|
|
else {
|
|
|
|
//
|
|
// Walk the list to find the parent.
|
|
//
|
|
|
|
do {
|
|
Pfn2 = MI_PFN_ELEMENT (Next);
|
|
Next = (PFN_NUMBER) Pfn2->OriginalPte.u.Long;
|
|
if (Page == Next) {
|
|
Pfn2->OriginalPte.u.Long = Pfn->OriginalPte.u.Long;
|
|
if ((PFN_NUMBER) Pfn->OriginalPte.u.Long == MM_EMPTY_LIST) {
|
|
ColorHead->Blink = Pfn2;
|
|
}
|
|
break;
|
|
}
|
|
} while (TRUE);
|
|
}
|
|
|
|
ASSERT (ColorHead->Count >= 1);
|
|
ColorHead->Count -= 1;
|
|
|
|
//
|
|
// Decrement availability count.
|
|
//
|
|
|
|
#if defined(MI_MULTINODE)
|
|
|
|
if (KeNumberNodes > 1) {
|
|
MI_NODE_FROM_COLOR(Color)->FreeCount[ListName]--;
|
|
}
|
|
|
|
#endif
|
|
|
|
MmAvailablePages -= 1;
|
|
|
|
if (MmAvailablePages < MmMinimumFreePages) {
|
|
|
|
//
|
|
// Obtain free pages.
|
|
//
|
|
|
|
MiObtainFreePages();
|
|
}
|
|
|
|
return;
|
|
}
|
|
|
|
|
|
ULONG
|
|
FASTCALL
|
|
MiEnsureAvailablePageOrWait (
|
|
IN PEPROCESS Process,
|
|
IN PVOID VirtualAddress
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This procedure ensures that a physical page is available on
|
|
the zeroed, free or standby list such that the next call the remove a
|
|
page absolutely will not block. This is necessary as blocking would
|
|
require a wait which could cause a deadlock condition.
|
|
|
|
If a page is available the function returns immediately with a value
|
|
of FALSE indicating no wait operation was performed. If no physical
|
|
page is available, the thread enters a wait state and the function
|
|
returns the value TRUE when the wait operation completes.
|
|
|
|
Arguments:
|
|
|
|
Process - Supplies a pointer to the current process if, and only if,
|
|
the working set mutex is held currently held and should
|
|
be released if a wait operation is issued. Supplies
|
|
the value NULL otherwise.
|
|
|
|
VirtualAddress - Supplies the virtual address for the faulting page.
|
|
If the value is NULL, the page is treated as a
|
|
user mode address.
|
|
|
|
Return Value:
|
|
|
|
FALSE - if a page was immediately available.
|
|
TRUE - if a wait operation occurred before a page became available.
|
|
|
|
|
|
Environment:
|
|
|
|
Must be holding the PFN database lock with APCs disabled.
|
|
|
|
--*/
|
|
|
|
{
|
|
PVOID Event;
|
|
NTSTATUS Status;
|
|
KIRQL OldIrql;
|
|
KIRQL Ignore;
|
|
ULONG Limit;
|
|
ULONG Relock;
|
|
PFN_NUMBER StrandedPages;
|
|
LOGICAL WsHeldSafe;
|
|
PMMPFN Pfn1;
|
|
PMMPFN EndPfn;
|
|
LARGE_INTEGER WaitBegin;
|
|
LARGE_INTEGER WaitEnd;
|
|
|
|
MM_PFN_LOCK_ASSERT();
|
|
|
|
if (MmAvailablePages >= MM_HIGH_LIMIT) {
|
|
|
|
//
|
|
// Pages are available.
|
|
//
|
|
|
|
return FALSE;
|
|
}
|
|
|
|
//
|
|
// If this fault is for paged pool (or pagable kernel space,
|
|
// including page table pages), let it use the last page.
|
|
//
|
|
|
|
#if defined(_IA64_)
|
|
if (MI_IS_SYSTEM_ADDRESS(VirtualAddress) ||
|
|
(MI_IS_HYPER_SPACE_ADDRESS(VirtualAddress))) {
|
|
#else
|
|
if (((PMMPTE)VirtualAddress > MiGetPteAddress(HYPER_SPACE)) ||
|
|
((VirtualAddress > MM_HIGHEST_USER_ADDRESS) &&
|
|
(VirtualAddress < (PVOID)PTE_BASE))) {
|
|
#endif
|
|
|
|
//
|
|
// This fault is in the system, use 1 page as the limit.
|
|
//
|
|
|
|
if (MmAvailablePages >= MM_LOW_LIMIT) {
|
|
|
|
//
|
|
// Pages are available.
|
|
//
|
|
|
|
return FALSE;
|
|
}
|
|
|
|
Limit = MM_LOW_LIMIT;
|
|
Event = (PVOID)&MmAvailablePagesEvent;
|
|
}
|
|
else {
|
|
|
|
//
|
|
// If this thread has explicitly disabled APCs (FsRtlEnterFileSystem
|
|
// does this), then it may be holding resources or mutexes that may in
|
|
// turn be blocking memory making threads from making progress.
|
|
//
|
|
// Likewise give system threads a free pass as they may be worker
|
|
// threads processing potentially blocking items drivers have queued.
|
|
//
|
|
|
|
if ((IS_SYSTEM_THREAD(PsGetCurrentThread())) ||
|
|
(KeGetCurrentThread()->KernelApcDisable != 0)) {
|
|
|
|
if (MmAvailablePages >= MM_MEDIUM_LIMIT) {
|
|
|
|
//
|
|
// Pages are available.
|
|
//
|
|
|
|
return FALSE;
|
|
}
|
|
|
|
Limit = MM_MEDIUM_LIMIT;
|
|
Event = (PVOID) &MmAvailablePagesEventMedium;
|
|
}
|
|
else {
|
|
Limit = MM_HIGH_LIMIT;
|
|
Event = (PVOID) &MmAvailablePagesEventHigh;
|
|
}
|
|
}
|
|
|
|
//
|
|
// Initializing WsHeldSafe is not needed for
|
|
// correctness but without it the compiler cannot compile this code
|
|
// W4 to check for use of uninitialized variables.
|
|
//
|
|
|
|
WsHeldSafe = FALSE;
|
|
|
|
while (MmAvailablePages < Limit) {
|
|
|
|
KeClearEvent ((PKEVENT)Event);
|
|
|
|
UNLOCK_PFN (APC_LEVEL);
|
|
|
|
Relock = FALSE;
|
|
|
|
if (Process == HYDRA_PROCESS) {
|
|
UNLOCK_SESSION_SPACE_WS (APC_LEVEL);
|
|
}
|
|
else if (Process != NULL) {
|
|
|
|
//
|
|
// The working set lock may have been acquired safely or unsafely
|
|
// by our caller. Handle both cases here and below.
|
|
//
|
|
|
|
UNLOCK_WS_REGARDLESS (Process, WsHeldSafe);
|
|
}
|
|
else {
|
|
if (MmSystemLockOwner == PsGetCurrentThread()) {
|
|
UNLOCK_SYSTEM_WS (APC_LEVEL);
|
|
Relock = TRUE;
|
|
}
|
|
}
|
|
|
|
KiQueryInterruptTime(&WaitBegin);
|
|
|
|
//
|
|
// Wait 7 minutes for pages to become available.
|
|
//
|
|
|
|
Status = KeWaitForSingleObject(Event,
|
|
WrFreePage,
|
|
KernelMode,
|
|
FALSE,
|
|
(PLARGE_INTEGER)&MmSevenMinutes);
|
|
|
|
if (Status == STATUS_TIMEOUT) {
|
|
|
|
KiQueryInterruptTime(&WaitEnd);
|
|
|
|
if (MmSystemShutdown != 0) {
|
|
|
|
//
|
|
// Because applications are not terminated and drivers are
|
|
// not unloaded, they can continue to access pages even after
|
|
// the modified writer has terminated. This can cause the
|
|
// system to run out of pages since the pagefile(s) cannot be
|
|
// used.
|
|
//
|
|
|
|
KeBugCheckEx (DISORDERLY_SHUTDOWN,
|
|
MmModifiedPageListHead.Total,
|
|
MmTotalPagesForPagingFile,
|
|
(MmMaximumNonPagedPoolInBytes >> PAGE_SHIFT) - MmAllocatedNonPagedPool,
|
|
MmSystemShutdown);
|
|
}
|
|
|
|
//
|
|
// See how many transition pages have nonzero reference counts as
|
|
// these indicate drivers that aren't unlocking the pages in their
|
|
// MDLs.
|
|
//
|
|
|
|
Limit = 0;
|
|
StrandedPages = 0;
|
|
|
|
do {
|
|
|
|
Pfn1 = MI_PFN_ELEMENT (MmPhysicalMemoryBlock->Run[Limit].BasePage);
|
|
EndPfn = Pfn1 + MmPhysicalMemoryBlock->Run[Limit].PageCount;
|
|
|
|
while (Pfn1 < EndPfn) {
|
|
if ((Pfn1->u3.e1.PageLocation == TransitionPage) &&
|
|
(Pfn1->u3.e2.ReferenceCount != 0)) {
|
|
StrandedPages += 1;
|
|
}
|
|
Pfn1 += 1;
|
|
}
|
|
Limit += 1;
|
|
|
|
} while (Limit != MmPhysicalMemoryBlock->NumberOfRuns);
|
|
|
|
//
|
|
// This bugcheck can occur for the following reasons:
|
|
//
|
|
// A driver has blocked, deadlocking the modified or mapped
|
|
// page writers. Examples of this include mutex deadlocks or
|
|
// accesses to paged out memory in filesystem drivers, filter
|
|
// drivers, etc. This indicates a driver bug.
|
|
//
|
|
// The storage driver(s) are not processing requests. Examples
|
|
// of this are stranded queues, non-responding drives, etc. This
|
|
// indicates a driver bug.
|
|
//
|
|
// Not enough pool is available for the storage stack to write out
|
|
// modified pages. This indicates a driver bug.
|
|
//
|
|
// A high priority realtime thread has starved the balance set
|
|
// manager from trimming pages and/or starved the modified writer
|
|
// from writing them out. This indicates a bug in the component
|
|
// that created this thread.
|
|
//
|
|
|
|
StrandedPages &= ~3;
|
|
StrandedPages |= MmSystemShutdown;
|
|
|
|
if (KdDebuggerNotPresent) {
|
|
if (MmTotalPagesForPagingFile >= (MmModifiedPageListHead.Total >> 2)) {
|
|
KeBugCheckEx (NO_PAGES_AVAILABLE,
|
|
MmModifiedPageListHead.Total,
|
|
MmTotalPagesForPagingFile,
|
|
(MmMaximumNonPagedPoolInBytes >> PAGE_SHIFT) - MmAllocatedNonPagedPool,
|
|
StrandedPages);
|
|
}
|
|
KeBugCheckEx (DIRTY_MAPPED_PAGES_CONGESTION,
|
|
MmModifiedPageListHead.Total,
|
|
MmTotalPagesForPagingFile,
|
|
(MmMaximumNonPagedPoolInBytes >> PAGE_SHIFT) - MmAllocatedNonPagedPool,
|
|
StrandedPages);
|
|
}
|
|
|
|
DbgPrint ("MmEnsureAvailablePageOrWait: 7 min timeout %x %x %x %x\n", WaitEnd.HighPart, WaitEnd.LowPart, WaitBegin.HighPart, WaitBegin.LowPart);
|
|
|
|
DbgPrint ("Without a debugger attached, the following bugcheck would have occurred.\n");
|
|
if (MmTotalPagesForPagingFile >= (MmModifiedPageListHead.Total >> 2)) {
|
|
DbgPrint ("%3lx ", NO_PAGES_AVAILABLE);
|
|
}
|
|
else {
|
|
DbgPrint ("%3lxp ", DIRTY_MAPPED_PAGES_CONGESTION);
|
|
}
|
|
|
|
DbgPrint ("%p %p %p %p\n",
|
|
MmModifiedPageListHead.Total,
|
|
MmTotalPagesForPagingFile,
|
|
(MmMaximumNonPagedPoolInBytes >> PAGE_SHIFT) - MmAllocatedNonPagedPool,
|
|
StrandedPages);
|
|
|
|
//
|
|
// Pop into the debugger (even on free builds) to determine
|
|
// the cause of the starvation and march on.
|
|
//
|
|
|
|
DbgBreakPoint ();
|
|
}
|
|
|
|
if (Process == HYDRA_PROCESS) {
|
|
LOCK_SESSION_SPACE_WS (Ignore, PsGetCurrentThread ());
|
|
}
|
|
else if (Process != NULL) {
|
|
|
|
//
|
|
// The working set lock may have been acquired safely or unsafely
|
|
// by our caller. Reacquire it in the same manner our caller did.
|
|
//
|
|
|
|
LOCK_WS_REGARDLESS (Process, WsHeldSafe);
|
|
}
|
|
else {
|
|
if (Relock) {
|
|
LOCK_SYSTEM_WS (Ignore, PsGetCurrentThread ());
|
|
}
|
|
}
|
|
|
|
LOCK_PFN (OldIrql);
|
|
}
|
|
|
|
return TRUE;
|
|
}
|
|
|
|
|
|
PFN_NUMBER
|
|
FASTCALL
|
|
MiRemoveZeroPage (
|
|
IN ULONG Color
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This procedure removes a zero page from either the zeroed, free
|
|
or standby lists (in that order). If no pages exist on the zeroed
|
|
or free list a transition page is removed from the standby list
|
|
and the PTE (may be a prototype PTE) which refers to this page is
|
|
changed from transition back to its original contents.
|
|
|
|
If the page is not obtained from the zeroed list, it is zeroed.
|
|
|
|
Arguments:
|
|
|
|
Color - Supplies the page color for which this page is destined.
|
|
This is used for checking virtual address alignments to
|
|
determine if the D cache needs flushing before the page
|
|
can be reused.
|
|
|
|
The above was true when we were concerned about caches
|
|
which are virtually indexed (ie MIPS). Today we
|
|
are more concerned that we get a good usage spread across
|
|
the L2 caches of most machines. These caches are physically
|
|
indexed. By gathering pages that would have the same
|
|
index to the same color, then maximizing the color spread,
|
|
we maximize the effective use of the caches.
|
|
|
|
This has been extended for NUMA machines. The high part
|
|
of the color gives the node color (basically node number).
|
|
If we cannot allocate a page of the requested color, we
|
|
try to allocate a page on the same node before taking a
|
|
page from a different node.
|
|
|
|
Return Value:
|
|
|
|
The physical page number removed from the specified list.
|
|
|
|
Environment:
|
|
|
|
Must be holding the PFN database lock with APCs disabled.
|
|
|
|
--*/
|
|
|
|
{
|
|
PFN_NUMBER Page;
|
|
PMMPFN Pfn1;
|
|
PMMCOLOR_TABLES FreePagesByColor;
|
|
#if MI_BARRIER_SUPPORTED
|
|
ULONG BarrierStamp;
|
|
#endif
|
|
#if defined(MI_MULTINODE)
|
|
PKNODE Node;
|
|
ULONG NodeColor;
|
|
ULONG OriginalColor;
|
|
#endif
|
|
|
|
MM_PFN_LOCK_ASSERT();
|
|
ASSERT(MmAvailablePages != 0);
|
|
|
|
FreePagesByColor = MmFreePagesByColor[ZeroedPageList];
|
|
|
|
#if defined(MI_MULTINODE)
|
|
|
|
//
|
|
// Initializing Node is not needed for correctness, but without it
|
|
// the compiler cannot compile this code W4 to check for use of
|
|
// uninitialized variables.
|
|
//
|
|
|
|
Node = NULL;
|
|
|
|
NodeColor = Color & ~MmSecondaryColorMask;
|
|
OriginalColor = Color;
|
|
|
|
if (KeNumberNodes > 1) {
|
|
Node = MI_NODE_FROM_COLOR(Color);
|
|
}
|
|
|
|
do {
|
|
|
|
#endif
|
|
|
|
//
|
|
// Attempt to remove a page from the zeroed page list. If a page
|
|
// is available, then remove it and return its page frame index.
|
|
// Otherwise, attempt to remove a page from the free page list or
|
|
// the standby list.
|
|
//
|
|
// N.B. It is not necessary to change page colors even if the old
|
|
// color is not equal to the new color. The zero page thread
|
|
// ensures that all zeroed pages are removed from all caches.
|
|
//
|
|
|
|
ASSERT (Color < MmSecondaryColors);
|
|
Page = FreePagesByColor[Color].Flink;
|
|
|
|
if (Page != MM_EMPTY_LIST) {
|
|
|
|
//
|
|
// Remove the first entry on the zeroed by color list.
|
|
//
|
|
|
|
#if DBG
|
|
Pfn1 = MI_PFN_ELEMENT(Page);
|
|
ASSERT (Pfn1->u3.e1.CacheAttribute == MiNotMapped);
|
|
#endif
|
|
ASSERT ((Pfn1->u3.e1.PageLocation == ZeroedPageList) ||
|
|
((Pfn1->u3.e1.PageLocation == FreePageList) &&
|
|
(FreePagesByColor == MmFreePagesByColor[FreePageList])));
|
|
|
|
ASSERT (Pfn1->u4.PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
|
|
|
Page = MiRemovePageByColor (Page, Color);
|
|
|
|
ASSERT (Pfn1 == MI_PFN_ELEMENT(Page));
|
|
ASSERT (Pfn1->u3.e1.CacheAttribute == MiNotMapped);
|
|
|
|
#if defined(MI_MULTINODE)
|
|
|
|
if (FreePagesByColor != MmFreePagesByColor[ZeroedPageList]) {
|
|
goto ZeroPage;
|
|
}
|
|
|
|
#endif
|
|
|
|
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
|
|
ASSERT (Pfn1->u2.ShareCount == 0);
|
|
ASSERT (Pfn1->u4.PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
|
|
|
return Page;
|
|
|
|
}
|
|
|
|
#if defined(MI_MULTINODE)
|
|
|
|
//
|
|
// If this is a multinode machine and there are zero
|
|
// pages on this node, select another color on this
|
|
// node in preference to random selection.
|
|
//
|
|
|
|
if (KeNumberNodes > 1) {
|
|
if (Node->FreeCount[ZeroedPageList] != 0) {
|
|
Color = ((Color + 1) & MmSecondaryColorMask) | NodeColor;
|
|
ASSERT(Color != OriginalColor);
|
|
continue;
|
|
}
|
|
|
|
//
|
|
// No previously zeroed page with the specified secondary
|
|
// color exists. Since this is a multinode machine, zero
|
|
// an available local free page now instead of allocating a
|
|
// zeroed page from another node below.
|
|
//
|
|
|
|
if (Node->FreeCount[FreePageList] != 0) {
|
|
if (FreePagesByColor != MmFreePagesByColor[FreePageList]) {
|
|
FreePagesByColor = MmFreePagesByColor[FreePageList];
|
|
Color = OriginalColor;
|
|
}
|
|
else {
|
|
Color = ((Color + 1) & MmSecondaryColorMask) | NodeColor;
|
|
ASSERT(Color != OriginalColor);
|
|
}
|
|
continue;
|
|
}
|
|
}
|
|
|
|
break;
|
|
} while (TRUE);
|
|
|
|
#endif
|
|
|
|
//
|
|
// No previously zeroed page with the specified secondary color exists.
|
|
// Try a zeroed page of any color.
|
|
//
|
|
|
|
Page = MmZeroedPageListHead.Flink;
|
|
if (Page != MM_EMPTY_LIST) {
|
|
#if DBG
|
|
Pfn1 = MI_PFN_ELEMENT(Page);
|
|
#endif
|
|
ASSERT (Pfn1->u3.e1.CacheAttribute == MiNotMapped);
|
|
ASSERT (Pfn1->u3.e1.PageLocation == ZeroedPageList);
|
|
ASSERT (Pfn1->u4.PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
|
|
|
Color = MI_GET_COLOR_FROM_LIST_ENTRY(Page, MI_PFN_ELEMENT(Page));
|
|
|
|
Page = MiRemovePageByColor (Page, Color);
|
|
|
|
ASSERT (Pfn1 == MI_PFN_ELEMENT(Page));
|
|
ASSERT (Pfn1->u3.e1.CacheAttribute == MiNotMapped);
|
|
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
|
|
ASSERT (Pfn1->u2.ShareCount == 0);
|
|
ASSERT (Pfn1->u4.PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
|
|
|
return Page;
|
|
}
|
|
|
|
//
|
|
// No zeroed page of the primary color exists, try a free page of the
|
|
// secondary color. Note in the multinode case this has already been done
|
|
// above.
|
|
//
|
|
|
|
#if defined(MI_MULTINODE)
|
|
if (KeNumberNodes <= 1) {
|
|
#endif
|
|
FreePagesByColor = MmFreePagesByColor[FreePageList];
|
|
|
|
Page = FreePagesByColor[Color].Flink;
|
|
if (Page != MM_EMPTY_LIST) {
|
|
|
|
//
|
|
// Remove the first entry on the free list by color.
|
|
//
|
|
|
|
#if DBG
|
|
Pfn1 = MI_PFN_ELEMENT(Page);
|
|
#endif
|
|
ASSERT (Pfn1->u3.e1.PageLocation == FreePageList);
|
|
ASSERT (Pfn1->u4.PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
|
ASSERT (Pfn1->u3.e1.CacheAttribute == MiNotMapped);
|
|
|
|
Page = MiRemovePageByColor (Page, Color);
|
|
|
|
ASSERT (Pfn1 == MI_PFN_ELEMENT(Page));
|
|
ASSERT (Pfn1->u3.e1.CacheAttribute == MiNotMapped);
|
|
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
|
|
ASSERT (Pfn1->u2.ShareCount == 0);
|
|
ASSERT (Pfn1->u4.PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
|
|
|
goto ZeroPage;
|
|
}
|
|
#if defined(MI_MULTINODE)
|
|
}
|
|
#endif
|
|
|
|
Page = MmFreePageListHead.Flink;
|
|
if (Page != MM_EMPTY_LIST) {
|
|
|
|
Color = MI_GET_COLOR_FROM_LIST_ENTRY(Page, MI_PFN_ELEMENT(Page));
|
|
#if DBG
|
|
Pfn1 = MI_PFN_ELEMENT(Page);
|
|
ASSERT (Pfn1->u3.e1.CacheAttribute == MiNotMapped);
|
|
#endif
|
|
Page = MiRemovePageByColor (Page, Color);
|
|
|
|
ASSERT (Pfn1 == MI_PFN_ELEMENT(Page));
|
|
ASSERT (Pfn1->u3.e1.CacheAttribute == MiNotMapped);
|
|
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
|
|
ASSERT (Pfn1->u2.ShareCount == 0);
|
|
ASSERT (Pfn1->u4.PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
|
goto ZeroPage;
|
|
}
|
|
|
|
ASSERT (MmZeroedPageListHead.Total == 0);
|
|
ASSERT (MmFreePageListHead.Total == 0);
|
|
|
|
//
|
|
// Remove a page from the standby list and restore the original
|
|
// contents of the PTE to free the last reference to the physical
|
|
// page.
|
|
//
|
|
|
|
ASSERT (MmStandbyPageListHead.Total != 0);
|
|
|
|
Page = MiRemovePageFromList (&MmStandbyPageListHead);
|
|
ASSERT ((MI_PFN_ELEMENT(Page))->u4.PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
|
ASSERT (MI_PFN_ELEMENT(Page)->u3.e1.CacheAttribute == MiNotMapped);
|
|
MiStandbyRemoved += 1;
|
|
|
|
//
|
|
// If memory mirroring is in progress, any removals from the
|
|
// free, zeroed, standby, modified or modified-no-write lists that
|
|
// isn't immediately re-inserting into one of these 5 lists (WITHOUT
|
|
// modifying the page contents) must update the bitmap.
|
|
//
|
|
|
|
if (MiMirroringActive == TRUE) {
|
|
RtlSetBit (MiMirrorBitMap2, (ULONG)Page);
|
|
}
|
|
|
|
//
|
|
// Zero the page removed from the free or standby list.
|
|
//
|
|
|
|
ZeroPage:
|
|
|
|
Pfn1 = MI_PFN_ELEMENT(Page);
|
|
|
|
MiZeroPhysicalPage (Page, 0);
|
|
|
|
#if MI_BARRIER_SUPPORTED
|
|
|
|
//
|
|
// Note the stamping must occur after the page is zeroed.
|
|
//
|
|
|
|
MI_BARRIER_STAMP_ZEROED_PAGE (&BarrierStamp);
|
|
Pfn1->u4.PteFrame = BarrierStamp;
|
|
|
|
#endif
|
|
|
|
ASSERT (Pfn1->u3.e1.CacheAttribute == MiNotMapped);
|
|
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
|
|
ASSERT (Pfn1->u2.ShareCount == 0);
|
|
ASSERT (Pfn1->u4.PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
|
|
|
return Page;
|
|
}
|
|
|
|
PFN_NUMBER
|
|
FASTCALL
|
|
MiRemoveAnyPage (
|
|
IN ULONG Color
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This procedure removes a page from either the free, zeroed,
|
|
or standby lists (in that order). If no pages exist on the zeroed
|
|
or free list a transition page is removed from the standby list
|
|
and the PTE (may be a prototype PTE) which refers to this page is
|
|
changed from transition back to its original contents.
|
|
|
|
Note pages MUST exist to satisfy this request. The caller ensures this
|
|
by first calling MiEnsureAvailablePageOrWait.
|
|
|
|
Arguments:
|
|
|
|
Color - Supplies the page color for which this page is destined.
|
|
This is used for checking virtual address alignments to
|
|
determine if the D cache needs flushing before the page
|
|
can be reused.
|
|
|
|
The above was true when we were concerned about caches
|
|
which are virtually indexed. (eg MIPS). Today we
|
|
are more concerned that we get a good usage spread across
|
|
the L2 caches of most machines. These caches are physically
|
|
indexed. By gathering pages that would have the same
|
|
index to the same color, then maximizing the color spread,
|
|
we maximize the effective use of the caches.
|
|
|
|
This has been extended for NUMA machines. The high part
|
|
of the color gives the node color (basically node number).
|
|
If we cannot allocate a page of the requested color, we
|
|
try to allocate a page on the same node before taking a
|
|
page from a different node.
|
|
|
|
Return Value:
|
|
|
|
The physical page number removed from the specified list.
|
|
|
|
Environment:
|
|
|
|
Must be holding the PFN database lock with APCs disabled.
|
|
|
|
--*/
|
|
|
|
{
|
|
PFN_NUMBER Page;
|
|
#if DBG
|
|
PMMPFN Pfn1;
|
|
#endif
|
|
#if defined(MI_MULTINODE)
|
|
PKNODE Node;
|
|
ULONG NodeColor;
|
|
ULONG OriginalColor;
|
|
PFN_NUMBER LocalNodePagesAvailable;
|
|
#endif
|
|
|
|
MM_PFN_LOCK_ASSERT();
|
|
ASSERT(MmAvailablePages != 0);
|
|
|
|
#if defined(MI_MULTINODE)
|
|
|
|
//
|
|
// Bias color to memory node. The assumption is that if memory
|
|
// of the correct color is not available on this node, it is
|
|
// better to choose memory of a different color if you can stay
|
|
// on this node.
|
|
//
|
|
|
|
LocalNodePagesAvailable = 0;
|
|
NodeColor = Color & ~MmSecondaryColorMask;
|
|
OriginalColor = Color;
|
|
|
|
if (KeNumberNodes > 1) {
|
|
Node = MI_NODE_FROM_COLOR(Color);
|
|
LocalNodePagesAvailable = (Node->FreeCount[ZeroedPageList] | Node->FreeCount[ZeroedPageList]);
|
|
}
|
|
|
|
do {
|
|
|
|
#endif
|
|
|
|
//
|
|
// Check the free page list, and if a page is available
|
|
// remove it and return its value.
|
|
//
|
|
|
|
ASSERT (Color < MmSecondaryColors);
|
|
if (MmFreePagesByColor[FreePageList][Color].Flink != MM_EMPTY_LIST) {
|
|
|
|
//
|
|
// Remove the first entry on the free by color list.
|
|
//
|
|
|
|
Page = MmFreePagesByColor[FreePageList][Color].Flink;
|
|
#if DBG
|
|
Pfn1 = MI_PFN_ELEMENT(Page);
|
|
#endif
|
|
ASSERT (Pfn1->u3.e1.CacheAttribute == MiNotMapped);
|
|
ASSERT (Pfn1->u3.e1.PageLocation == FreePageList);
|
|
ASSERT (Pfn1->u4.PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
|
|
|
Page = MiRemovePageByColor (Page, Color);
|
|
|
|
ASSERT (Pfn1 == MI_PFN_ELEMENT(Page));
|
|
ASSERT (Pfn1->u3.e1.CacheAttribute == MiNotMapped);
|
|
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
|
|
ASSERT (Pfn1->u2.ShareCount == 0);
|
|
ASSERT (Pfn1->u4.PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
|
|
|
return Page;
|
|
}
|
|
|
|
//
|
|
// Try the zero page list by primary color.
|
|
//
|
|
|
|
if (MmFreePagesByColor[ZeroedPageList][Color].Flink
|
|
!= MM_EMPTY_LIST) {
|
|
|
|
//
|
|
// Remove the first entry on the zeroed by color list.
|
|
//
|
|
|
|
Page = MmFreePagesByColor[ZeroedPageList][Color].Flink;
|
|
#if DBG
|
|
Pfn1 = MI_PFN_ELEMENT(Page);
|
|
#endif
|
|
ASSERT (Pfn1->u3.e1.PageLocation == ZeroedPageList);
|
|
ASSERT (Pfn1->u4.PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
|
ASSERT (Pfn1->u3.e1.CacheAttribute == MiNotMapped);
|
|
|
|
Page = MiRemovePageByColor (Page, Color);
|
|
|
|
ASSERT (Pfn1 == MI_PFN_ELEMENT(Page));
|
|
ASSERT (Pfn1->u4.PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
|
ASSERT (Pfn1->u3.e1.CacheAttribute == MiNotMapped);
|
|
|
|
return Page;
|
|
}
|
|
|
|
//
|
|
// If this is a multinode machine and there are free
|
|
// pages on this node, select another color on this
|
|
// node in preference to random selection.
|
|
//
|
|
|
|
#if defined(MI_MULTINODE)
|
|
|
|
if (LocalNodePagesAvailable != 0) {
|
|
Color = ((Color + 1) & MmSecondaryColorMask) | NodeColor;
|
|
ASSERT(Color != OriginalColor);
|
|
continue;
|
|
}
|
|
|
|
break;
|
|
} while (TRUE);
|
|
|
|
#endif
|
|
|
|
//
|
|
// Check the free page list, and if a page is available
|
|
// remove it and return its value.
|
|
//
|
|
|
|
if (MmFreePageListHead.Flink != MM_EMPTY_LIST) {
|
|
Page = MmFreePageListHead.Flink;
|
|
Color = MI_GET_COLOR_FROM_LIST_ENTRY(Page, MI_PFN_ELEMENT(Page));
|
|
|
|
#if DBG
|
|
Pfn1 = MI_PFN_ELEMENT(Page);
|
|
#endif
|
|
ASSERT (Pfn1->u3.e1.PageLocation == FreePageList);
|
|
ASSERT (Pfn1->u4.PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
|
ASSERT (Pfn1->u3.e1.CacheAttribute == MiNotMapped);
|
|
|
|
Page = MiRemovePageByColor (Page, Color);
|
|
|
|
ASSERT (Pfn1 == MI_PFN_ELEMENT(Page));
|
|
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
|
|
ASSERT (Pfn1->u2.ShareCount == 0);
|
|
ASSERT (Pfn1->u4.PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
|
ASSERT (Pfn1->u3.e1.CacheAttribute == MiNotMapped);
|
|
|
|
return Page;
|
|
}
|
|
ASSERT (MmFreePageListHead.Total == 0);
|
|
|
|
//
|
|
// Check the zeroed page list, and if a page is available
|
|
// remove it and return its value.
|
|
//
|
|
|
|
if (MmZeroedPageListHead.Flink != MM_EMPTY_LIST) {
|
|
Page = MmZeroedPageListHead.Flink;
|
|
Color = MI_GET_COLOR_FROM_LIST_ENTRY(Page, MI_PFN_ELEMENT(Page));
|
|
|
|
#if DBG
|
|
Pfn1 = MI_PFN_ELEMENT(Page);
|
|
#endif
|
|
ASSERT (Pfn1->u3.e1.CacheAttribute == MiNotMapped);
|
|
Page = MiRemovePageByColor (Page, Color);
|
|
|
|
ASSERT (Pfn1 == MI_PFN_ELEMENT(Page));
|
|
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
|
|
ASSERT (Pfn1->u2.ShareCount == 0);
|
|
ASSERT (Pfn1->u4.PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
|
ASSERT (Pfn1->u3.e1.CacheAttribute == MiNotMapped);
|
|
|
|
return Page;
|
|
}
|
|
ASSERT (MmZeroedPageListHead.Total == 0);
|
|
|
|
//
|
|
// No pages exist on the free or zeroed list, use the standby list.
|
|
//
|
|
|
|
ASSERT(MmStandbyPageListHead.Total != 0);
|
|
|
|
Page = MiRemovePageFromList (&MmStandbyPageListHead);
|
|
ASSERT ((MI_PFN_ELEMENT(Page))->u4.PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
|
ASSERT ((MI_PFN_ELEMENT(Page))->u3.e1.CacheAttribute == MiNotMapped);
|
|
MiStandbyRemoved += 1;
|
|
|
|
//
|
|
// If memory mirroring is in progress, any removals from the
|
|
// free, zeroed, standby, modified or modified-no-write lists that
|
|
// isn't immediately re-inserting into one of these 5 lists (WITHOUT
|
|
// modifying the page contents) must update the bitmap.
|
|
//
|
|
|
|
if (MiMirroringActive == TRUE) {
|
|
RtlSetBit (MiMirrorBitMap2, (ULONG)Page);
|
|
}
|
|
|
|
MI_CHECK_PAGE_ALIGNMENT(Page, Color & MM_COLOR_MASK);
|
|
#if DBG
|
|
Pfn1 = MI_PFN_ELEMENT (Page);
|
|
#endif
|
|
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
|
|
ASSERT (Pfn1->u2.ShareCount == 0);
|
|
|
|
return Page;
|
|
}
|
|
|
|
|
|
PFN_NUMBER
|
|
FASTCALL
|
|
MiRemovePageByColor (
|
|
IN PFN_NUMBER Page,
|
|
IN ULONG Color
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This procedure removes a page from the middle of the free or
|
|
zeroed page list.
|
|
|
|
Arguments:
|
|
|
|
Page - Supplies the physical page number to unlink from the list.
|
|
|
|
Color - Supplies the page color for which this page is destined.
|
|
This is used for checking virtual address alignments to
|
|
determine if the D cache needs flushing before the page
|
|
can be reused.
|
|
|
|
Return Value:
|
|
|
|
The page frame number that was unlinked (always equal to the one
|
|
passed in, but returned so the caller's fastcall sequences save
|
|
extra register pushes and pops.
|
|
|
|
Environment:
|
|
|
|
Must be holding the PFN database lock with APCs disabled.
|
|
|
|
--*/
|
|
|
|
{
|
|
PMMPFNLIST ListHead;
|
|
PMMPFNLIST PrimaryListHead;
|
|
PFN_NUMBER Previous;
|
|
PFN_NUMBER Next;
|
|
PMMPFN Pfn1;
|
|
PMMPFN Pfn2;
|
|
ULONG NodeColor;
|
|
MMLISTS ListName;
|
|
PMMCOLOR_TABLES ColorHead;
|
|
|
|
MM_PFN_LOCK_ASSERT();
|
|
|
|
Pfn1 = MI_PFN_ELEMENT (Page);
|
|
NodeColor = Pfn1->u3.e1.PageColor;
|
|
|
|
#if defined(MI_MULTINODE)
|
|
|
|
ASSERT (NodeColor == (Color >> MmSecondaryColorNodeShift));
|
|
|
|
#endif
|
|
|
|
if (PERFINFO_IS_GROUP_ON(PERF_MEMORY)) {
|
|
MiLogPfnInformation (Pfn1, PERFINFO_LOG_TYPE_REMOVEPAGEBYCOLOR);
|
|
}
|
|
|
|
//
|
|
// If memory mirroring is in progress, any additions or removals to the
|
|
// free, zeroed, standby, modified or modified-no-write lists must
|
|
// update the bitmap.
|
|
//
|
|
|
|
if (MiMirroringActive == TRUE) {
|
|
RtlSetBit (MiMirrorBitMap2, (ULONG)Page);
|
|
}
|
|
|
|
ListHead = MmPageLocationList[Pfn1->u3.e1.PageLocation];
|
|
ListName = ListHead->ListName;
|
|
|
|
ListHead->Total -= 1;
|
|
|
|
PrimaryListHead = ListHead;
|
|
|
|
Next = Pfn1->u1.Flink;
|
|
Pfn1->u1.Flink = 0; // Assumes Flink width is >= WsIndex width
|
|
Previous = Pfn1->u2.Blink;
|
|
Pfn1->u2.Blink = 0;
|
|
|
|
if (Next == MM_EMPTY_LIST) {
|
|
PrimaryListHead->Blink = Previous;
|
|
}
|
|
else {
|
|
Pfn2 = MI_PFN_ELEMENT(Next);
|
|
Pfn2->u2.Blink = Previous;
|
|
}
|
|
|
|
if (Previous == MM_EMPTY_LIST) {
|
|
PrimaryListHead->Flink = Next;
|
|
}
|
|
else {
|
|
Pfn2 = MI_PFN_ELEMENT(Previous);
|
|
Pfn2->u1.Flink = Next;
|
|
}
|
|
|
|
ASSERT (Pfn1->u3.e1.RemovalRequested == 0);
|
|
|
|
//
|
|
// Zero the flags longword, but keep the color and cache information.
|
|
//
|
|
|
|
ASSERT (Pfn1->u3.e1.CacheAttribute == MiNotMapped);
|
|
ASSERT (Pfn1->u3.e1.Rom == 0);
|
|
Pfn1->u3.e2.ShortFlags = 0;
|
|
Pfn1->u3.e1.PageColor = NodeColor;
|
|
Pfn1->u3.e1.CacheAttribute = MiNotMapped;
|
|
|
|
//
|
|
// Update the color lists.
|
|
//
|
|
|
|
ASSERT (Color < MmSecondaryColors);
|
|
ColorHead = &MmFreePagesByColor[ListName][Color];
|
|
ColorHead->Flink = (PFN_NUMBER) Pfn1->OriginalPte.u.Long;
|
|
ASSERT (ColorHead->Count >= 1);
|
|
ColorHead->Count -= 1;
|
|
|
|
//
|
|
// Note that we now have one less page available.
|
|
//
|
|
|
|
#if defined(MI_MULTINODE)
|
|
if (KeNumberNodes > 1) {
|
|
KeNodeBlock[NodeColor]->FreeCount[ListName]--;
|
|
}
|
|
#endif
|
|
|
|
MmAvailablePages -= 1;
|
|
|
|
if (MmAvailablePages < MmMinimumFreePages) {
|
|
|
|
//
|
|
// Obtain free pages.
|
|
//
|
|
|
|
MiObtainFreePages();
|
|
}
|
|
|
|
return Page;
|
|
}
|
|
|
|
|
|
VOID
|
|
FASTCALL
|
|
MiInsertFrontModifiedNoWrite (
|
|
IN PFN_NUMBER PageFrameIndex
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This procedure inserts a page at the FRONT of the modified no write list.
|
|
|
|
Arguments:
|
|
|
|
PageFrameIndex - Supplies the physical page number to insert in the list.
|
|
|
|
Return Value:
|
|
|
|
None.
|
|
|
|
Environment:
|
|
|
|
Must be holding the PFN database lock with APCs disabled.
|
|
|
|
--*/
|
|
|
|
{
|
|
PFN_NUMBER first;
|
|
PMMPFN Pfn1;
|
|
PMMPFN Pfn2;
|
|
|
|
MM_PFN_LOCK_ASSERT();
|
|
ASSERT ((PageFrameIndex != 0) && (PageFrameIndex <= MmHighestPhysicalPage) &&
|
|
(PageFrameIndex >= MmLowestPhysicalPage));
|
|
|
|
//
|
|
// Check to ensure the reference count for the page is zero.
|
|
//
|
|
|
|
Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
|
|
|
|
MI_SNAP_DATA (Pfn1, Pfn1->PteAddress, 0xA);
|
|
|
|
ASSERT (Pfn1->u3.e1.CacheAttribute == MiCached);
|
|
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
|
|
|
|
MmModifiedNoWritePageListHead.Total += 1; // One more page on the list.
|
|
|
|
MI_TALLY_TRANSITION_PAGE_ADDITION (Pfn1);
|
|
|
|
first = MmModifiedNoWritePageListHead.Flink;
|
|
if (first == MM_EMPTY_LIST) {
|
|
|
|
//
|
|
// List is empty add the page to the ListHead.
|
|
//
|
|
|
|
MmModifiedNoWritePageListHead.Blink = PageFrameIndex;
|
|
}
|
|
else {
|
|
Pfn2 = MI_PFN_ELEMENT (first);
|
|
Pfn2->u2.Blink = PageFrameIndex;
|
|
}
|
|
|
|
MmModifiedNoWritePageListHead.Flink = PageFrameIndex;
|
|
Pfn1->u1.Flink = first;
|
|
Pfn1->u2.Blink = MM_EMPTY_LIST;
|
|
Pfn1->u3.e1.PageLocation = ModifiedNoWritePageList;
|
|
return;
|
|
}
|
|
|
|
PFN_NUMBER
|
|
MiAllocatePfn (
|
|
IN PMMPTE PointerPte,
|
|
IN ULONG Protection
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This procedure allocates and initializes a page of memory.
|
|
|
|
Arguments:
|
|
|
|
PointerPte - Supplies the PTE to initialize.
|
|
|
|
Return Value:
|
|
|
|
The page frame index allocated.
|
|
|
|
Environment:
|
|
|
|
Kernel mode.
|
|
|
|
--*/
|
|
{
|
|
KIRQL OldIrql;
|
|
PFN_NUMBER PageFrameIndex;
|
|
|
|
LOCK_PFN (OldIrql);
|
|
|
|
MiEnsureAvailablePageOrWait (NULL, NULL);
|
|
|
|
PageFrameIndex = MiRemoveAnyPage (MI_GET_PAGE_COLOR_FROM_PTE (PointerPte));
|
|
|
|
PointerPte->u.Long = MM_KERNEL_DEMAND_ZERO_PTE;
|
|
|
|
PointerPte->u.Soft.Protection |= Protection;
|
|
|
|
MiInitializePfn (PageFrameIndex, PointerPte, 1);
|
|
|
|
UNLOCK_PFN (OldIrql);
|
|
|
|
return PageFrameIndex;
|
|
}
|
|
|
|
VOID
|
|
FASTCALL
|
|
MiLogPfnInformation (
|
|
IN PMMPFN Pfn1,
|
|
IN USHORT Reason
|
|
)
|
|
{
|
|
MMPFN_IDENTITY PfnIdentity;
|
|
|
|
RtlZeroMemory (&PfnIdentity, sizeof(PfnIdentity));
|
|
|
|
if (Reason == PERFINFO_LOG_TYPE_INSERTINFREELIST) {
|
|
MI_MARK_PFN_UNDELETED (Pfn1);
|
|
}
|
|
|
|
MiIdentifyPfn(Pfn1, &PfnIdentity);
|
|
|
|
PerfInfoLogBytes (Reason, &PfnIdentity, sizeof(PfnIdentity));
|
|
|
|
if (Reason == PERFINFO_LOG_TYPE_INSERTINFREELIST) {
|
|
MI_SET_PFN_DELETED (Pfn1);
|
|
}
|
|
}
|
|
|
|
VOID
|
|
MiPurgeTransitionList (
|
|
VOID
|
|
)
|
|
{
|
|
PMMPFN Pfn1;
|
|
KIRQL OldIrql;
|
|
PFN_NUMBER PageFrameIndex;
|
|
|
|
//
|
|
// Run the transition list and free all the entries so transition
|
|
// faults are not satisfied for any of the non modified pages that were
|
|
// freed.
|
|
//
|
|
|
|
LOCK_PFN (OldIrql);
|
|
|
|
while (MmStandbyPageListHead.Total != 0) {
|
|
|
|
PageFrameIndex = MiRemovePageFromList (&MmStandbyPageListHead);
|
|
|
|
Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
|
|
|
|
ASSERT (Pfn1->u2.ShareCount == 0);
|
|
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
|
|
|
|
Pfn1->u3.e2.ReferenceCount += 1;
|
|
Pfn1->OriginalPte = ZeroPte;
|
|
|
|
MI_SET_PFN_DELETED (Pfn1);
|
|
|
|
MiDecrementReferenceCount (PageFrameIndex);
|
|
|
|
//
|
|
// If memory mirroring is in progress, any removal from
|
|
// the standby, modified or modified-no-write lists that isn't
|
|
// immediately re-inserting in one of these 3 lists must
|
|
// update the bitmap.
|
|
//
|
|
|
|
if (MiMirroringActive == TRUE) {
|
|
RtlSetBit (MiMirrorBitMap2, (ULONG)PageFrameIndex);
|
|
}
|
|
}
|
|
|
|
UNLOCK_PFN (OldIrql);
|
|
return;
|
|
}
|