Leaked source code of windows server 2003
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 
 

629 lines
15 KiB

/*++
Copyright (c) 1989 Microsoft Corporation
Module Name:
zeropage.c
Abstract:
This module contains the zero page thread for memory management.
Author:
Lou Perazzoli (loup) 6-Apr-1991
Landy Wang (landyw) 02-June-1997
Revision History:
--*/
#include "mi.h"
#define MM_ZERO_PAGE_OBJECT 0
#define PO_SYS_IDLE_OBJECT 1
#define NUMBER_WAIT_OBJECTS 2
#define MACHINE_ZERO_PAGE(ZeroBase,NumberOfBytes) KeZeroPagesFromIdleThread(ZeroBase,NumberOfBytes)
LOGICAL MiZeroingDisabled = FALSE;
#if !defined(NT_UP)
LONG MiNextZeroProcessor = (LONG)-1;
#ifdef ALLOC_PRAGMA
#pragma alloc_text(INIT,MiStartZeroPageWorkers)
#endif
#endif
VOID
MmZeroPageThread (
VOID
)
/*++
Routine Description:
Implements the NT zeroing page thread. This thread runs
at priority zero and removes a page from the free list,
zeroes it, and places it on the zeroed page list.
Arguments:
None.
Return Value:
None.
Environment:
Kernel mode.
--*/
{
KIRQL OldIrql;
PFN_NUMBER PageFrame1;
PFN_NUMBER PageFrame;
PMMPFN Pfn1;
PKTHREAD Thread;
PVOID ZeroBase;
PVOID WaitObjects[NUMBER_WAIT_OBJECTS];
NTSTATUS Status;
PVOID StartVa;
PVOID EndVa;
PFN_COUNT PagesToZero;
PFN_COUNT MaximumPagesToZero;
ULONG Color;
ULONG StartColor;
PMMPFN PfnAllocation;
#if defined(MI_MULTINODE)
ULONG i;
ULONG n;
ULONG LastNodeZeroing;
n = 0;
LastNodeZeroing = 0;
#endif
//
// Before this becomes the zero page thread, free the kernel
// initialization code.
//
MiFindInitializationCode (&StartVa, &EndVa);
if (StartVa != NULL) {
MiFreeInitializationCode (StartVa, EndVa);
}
MaximumPagesToZero = 1;
#if !defined(NT_UP)
//
// Zero groups of pages at once to reduce PFN lock contention.
// Charge commitment as well as resident available up front since
// zeroing may get starved priority-wise.
//
// Note using MmSecondaryColors here would be excessively wasteful
// on NUMA systems. MmSecondaryColorMask + 1 is correct for all platforms.
//
PagesToZero = MmSecondaryColorMask + 1;
if (PagesToZero > NUMBER_OF_ZEROING_PTES) {
PagesToZero = NUMBER_OF_ZEROING_PTES;
}
if (MiChargeCommitment (PagesToZero, NULL) == TRUE) {
LOCK_PFN (OldIrql);
//
// Check to make sure the physical pages are available.
//
if (MI_NONPAGABLE_MEMORY_AVAILABLE() > (SPFN_NUMBER)(PagesToZero)) {
MI_DECREMENT_RESIDENT_AVAILABLE (PagesToZero,
MM_RESAVAIL_ALLOCATE_ZERO_PAGE_CLUSTERS);
MaximumPagesToZero = PagesToZero;
}
UNLOCK_PFN (OldIrql);
}
#endif
//
// The following code sets the current thread's base priority to zero
// and then sets its current priority to zero. This ensures that the
// thread always runs at a priority of zero.
//
Thread = KeGetCurrentThread ();
Thread->BasePriority = 0;
KeSetPriorityThread (Thread, 0);
//
// Initialize wait object array for multiple wait
//
WaitObjects[MM_ZERO_PAGE_OBJECT] = &MmZeroingPageEvent;
WaitObjects[PO_SYS_IDLE_OBJECT] = &PoSystemIdleTimer;
Color = 0;
PfnAllocation = (PMMPFN) MM_EMPTY_LIST;
//
// Loop forever zeroing pages.
//
do {
//
// Wait until there are at least MmZeroPageMinimum pages
// on the free list.
//
Status = KeWaitForMultipleObjects (NUMBER_WAIT_OBJECTS,
WaitObjects,
WaitAny,
WrFreePage,
KernelMode,
FALSE,
NULL,
NULL);
if (Status == PO_SYS_IDLE_OBJECT) {
PoSystemIdleWorker (TRUE);
continue;
}
PagesToZero = 0;
LOCK_PFN (OldIrql);
do {
if (MmFreePageListHead.Total == 0) {
//
// No pages on the free list at this time, wait for
// some more.
//
MmZeroingPageThreadActive = FALSE;
UNLOCK_PFN (OldIrql);
break;
}
if (MiZeroingDisabled == TRUE) {
MmZeroingPageThreadActive = FALSE;
UNLOCK_PFN (OldIrql);
KeDelayExecutionThread (KernelMode,
FALSE,
(PLARGE_INTEGER)&MmHalfSecond);
break;
}
#if defined(MI_MULTINODE)
//
// In a multinode system, zero pages by node. Resume on
// the last node examined, find a node with free pages that
// need to be zeroed.
//
if (KeNumberNodes > 1) {
n = LastNodeZeroing;
for (i = 0; i < KeNumberNodes; i += 1) {
if (KeNodeBlock[n]->FreeCount[FreePageList] != 0) {
break;
}
n = (n + 1) % KeNumberNodes;
}
ASSERT (i != KeNumberNodes);
ASSERT (KeNodeBlock[n]->FreeCount[FreePageList] != 0);
if (n != LastNodeZeroing) {
Color = KeNodeBlock[n]->MmShiftedColor;
}
}
#endif
ASSERT (PagesToZero == 0);
StartColor = Color;
do {
PageFrame = MmFreePagesByColor[FreePageList][Color].Flink;
if (PageFrame != MM_EMPTY_LIST) {
PageFrame1 = MiRemoveAnyPage (Color);
ASSERT (PageFrame == PageFrame1);
Pfn1 = MI_PFN_ELEMENT (PageFrame);
Pfn1->u1.Flink = (PFN_NUMBER) PfnAllocation;
//
// Temporarily mark the page as bad so that contiguous
// memory allocators won't steal it when we release
// the PFN lock below. This also prevents the
// MiIdentifyPfn code from trying to identify it as
// we haven't filled in all the fields yet.
//
Pfn1->u3.e1.PageLocation = BadPageList;
PfnAllocation = Pfn1;
PagesToZero += 1;
}
//
// March to the next color - this will be used to finish
// filling the current chunk or to start the next one.
//
Color = (Color & ~MmSecondaryColorMask) |
((Color + 1) & MmSecondaryColorMask);
if (PagesToZero == MaximumPagesToZero) {
break;
}
if (Color == StartColor) {
break;
}
} while (TRUE);
ASSERT (PfnAllocation != (PMMPFN) MM_EMPTY_LIST);
UNLOCK_PFN (OldIrql);
ZeroBase = MiMapPagesToZeroInHyperSpace (PfnAllocation, PagesToZero);
#if defined(MI_MULTINODE)
//
// If a node switch is in order, do it now that the PFN
// lock has been released.
//
if ((KeNumberNodes > 1) && (n != LastNodeZeroing)) {
LastNodeZeroing = n;
KeFindFirstSetLeftAffinity (KeNodeBlock[n]->ProcessorMask, &i);
KeSetIdealProcessorThread (Thread, (UCHAR)i);
}
#endif
MACHINE_ZERO_PAGE (ZeroBase, PagesToZero << PAGE_SHIFT);
#if 0
ASSERT (RtlCompareMemoryUlong (ZeroBase, PagesToZero << PAGE_SHIFT, 0) == PagesToZero << PAGE_SHIFT);
#endif
MiUnmapPagesInZeroSpace (ZeroBase, PagesToZero);
PagesToZero = 0;
Pfn1 = PfnAllocation;
LOCK_PFN (OldIrql);
do {
PageFrame = MI_PFN_ELEMENT_TO_INDEX (Pfn1);
Pfn1 = (PMMPFN) Pfn1->u1.Flink;
MiInsertPageInList (&MmZeroedPageListHead, PageFrame);
} while (Pfn1 != (PMMPFN) MM_EMPTY_LIST);
//
// We just finished processing a cluster of pages - briefly
// release the PFN lock to allow other threads to make progress.
//
UNLOCK_PFN (OldIrql);
PfnAllocation = (PMMPFN) MM_EMPTY_LIST;
LOCK_PFN (OldIrql);
} while (TRUE);
} while (TRUE);
}
#if !defined(NT_UP)
VOID
MiZeroPageWorker (
IN PVOID Context
)
/*++
Routine Description:
This routine is the worker routine executed by all processors so that
initial page zeroing occurs in parallel.
Arguments:
Context - Supplies a pointer to the workitem.
Return Value:
None.
Environment:
Kernel mode initialization time, PASSIVE_LEVEL. Because this is INIT
only code, don't bother charging commit for the pages.
--*/
{
MMPTE TempPte;
PMMPTE PointerPte;
KAFFINITY Affinity;
KIRQL OldIrql;
PVOID ZeroBase;
PKTHREAD Thread;
CCHAR OldProcessor;
SCHAR OldBasePriority;
KPRIORITY OldPriority;
PWORK_QUEUE_ITEM WorkItem;
PMMPFN Pfn1;
PFN_NUMBER NewPage;
PFN_NUMBER PageFrame;
#if defined(MI_MULTINODE)
PKNODE Node;
ULONG Color;
ULONG FinalColor;
#endif
WorkItem = (PWORK_QUEUE_ITEM) Context;
ExFreePool (WorkItem);
TempPte = ValidKernelPte;
//
// The following code sets the current thread's base and current priorities
// to one so all other code (except the zero page thread) can preempt it.
//
Thread = KeGetCurrentThread ();
OldBasePriority = Thread->BasePriority;
Thread->BasePriority = 1;
OldPriority = KeSetPriorityThread (Thread, 1);
//
// Dispatch each worker thread to the next processor in line.
//
OldProcessor = (CCHAR) InterlockedIncrement (&MiNextZeroProcessor);
Affinity = AFFINITY_MASK (OldProcessor);
Affinity = KeSetAffinityThread (Thread, Affinity);
//
// Zero all local pages.
//
#if defined(MI_MULTINODE)
if (KeNumberNodes > 1) {
Node = KeGetCurrentNode ();
Color = Node->MmShiftedColor;
FinalColor = Color + MmSecondaryColorMask + 1;
}
else {
SATISFY_OVERZEALOUS_COMPILER (Node = NULL);
SATISFY_OVERZEALOUS_COMPILER (Color = 0);
SATISFY_OVERZEALOUS_COMPILER (FinalColor = 0);
}
#endif
LOCK_PFN (OldIrql);
do {
#if defined(MI_MULTINODE)
//
// In a multinode system, zero pages by node.
//
if (KeNumberNodes > 1) {
if (Node->FreeCount[FreePageList] == 0) {
//
// No pages on the free list at this time, bail.
//
UNLOCK_PFN (OldIrql);
break;
}
//
// Must start with a color MiRemoveAnyPage will
// satisfy from the free list otherwise it will
// return an already zeroed page.
//
while (MmFreePagesByColor[FreePageList][Color].Flink == MM_EMPTY_LIST) {
//
// No pages on this free list color, march to the next one.
//
Color += 1;
if (Color == FinalColor) {
UNLOCK_PFN (OldIrql);
goto ZeroingFinished;
}
}
PageFrame = MiRemoveAnyPage (Color);
}
else {
#endif
if (MmFreePageListHead.Total == 0) {
//
// No pages on the free list at this time, bail.
//
UNLOCK_PFN (OldIrql);
break;
}
PageFrame = MmFreePageListHead.Flink;
ASSERT (PageFrame != MM_EMPTY_LIST);
Pfn1 = MI_PFN_ELEMENT(PageFrame);
NewPage = MiRemoveAnyPage (MI_GET_COLOR_FROM_LIST_ENTRY(PageFrame, Pfn1));
if (NewPage != PageFrame) {
//
// Someone has removed a page from the colored lists
// chain without updating the freelist chain.
//
KeBugCheckEx (PFN_LIST_CORRUPT,
0x8F,
NewPage,
PageFrame,
0);
}
#if defined(MI_MULTINODE)
}
#endif
//
// Use system PTEs instead of hyperspace to zero the page so that
// a spinlock (ie: interrupts blocked) is not held while zeroing.
// Since system PTE acquisition is lock free and the TB lazy flushed,
// this is perhaps the best path regardless.
//
UNLOCK_PFN (OldIrql);
PointerPte = MiReserveSystemPtes (1, SystemPteSpace);
if (PointerPte == NULL) {
//
// Put this page back on the freelist.
//
LOCK_PFN (OldIrql);
MiInsertPageInFreeList (PageFrame);
UNLOCK_PFN (OldIrql);
break;
}
ASSERT (PointerPte->u.Hard.Valid == 0);
ZeroBase = MiGetVirtualAddressMappedByPte (PointerPte);
TempPte.u.Hard.PageFrameNumber = PageFrame;
MI_WRITE_VALID_PTE (PointerPte, TempPte);
KeZeroPages (ZeroBase, PAGE_SIZE);
MiReleaseSystemPtes (PointerPte, 1, SystemPteSpace);
LOCK_PFN (OldIrql);
MiInsertPageInList (&MmZeroedPageListHead, PageFrame);
} while (TRUE);
#if defined(MI_MULTINODE)
ZeroingFinished:
#endif
//
// Restore the entry thread priority and processor affinity.
//
KeSetAffinityThread (Thread, Affinity);
KeSetPriorityThread (Thread, OldPriority);
Thread->BasePriority = OldBasePriority;
}
VOID
MiStartZeroPageWorkers (
VOID
)
/*++
Routine Description:
This routine starts the zero page worker threads.
Arguments:
None.
Return Value:
None.
Environment:
Kernel mode initialization phase 1, PASSIVE_LEVEL.
--*/
{
ULONG i;
PWORK_QUEUE_ITEM WorkItem;
for (i = 0; i < (ULONG) KeNumberProcessors; i += 1) {
WorkItem = ExAllocatePoolWithTag (NonPagedPool,
sizeof (WORK_QUEUE_ITEM),
'wZmM');
if (WorkItem == NULL) {
break;
}
ExInitializeWorkItem (WorkItem, MiZeroPageWorker, (PVOID) WorkItem);
ExQueueWorkItem (WorkItem, CriticalWorkQueue);
}
}
#endif