mirror of https://github.com/tongzx/nt5src
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
639 lines
16 KiB
639 lines
16 KiB
/*++
|
|
|
|
Copyright (c) 1998 Microsoft Corporation
|
|
|
|
Module Name:
|
|
|
|
paesup.c
|
|
|
|
Abstract:
|
|
|
|
This module contains the machine dependent support for the x86 PAE
|
|
architecture.
|
|
|
|
Author:
|
|
|
|
Landy Wang (landyw) 15-Nov-1998
|
|
|
|
Revision History:
|
|
|
|
--*/
|
|
|
|
#include "mi.h"
|
|
|
|
#if defined (_X86PAE_)
|
|
|
|
#define PAES_PER_PAGE (PAGE_SIZE / sizeof(PAE_ENTRY))
|
|
|
|
#define MINIMUM_PAE_SLIST_THRESHOLD (PAES_PER_PAGE * 1)
|
|
#define MINIMUM_PAE_THRESHOLD (PAES_PER_PAGE * 4)
|
|
#define REPLENISH_PAE_SIZE (PAES_PER_PAGE * 16)
|
|
#define EXCESS_PAE_THRESHOLD (PAES_PER_PAGE * 20)
|
|
|
|
#define MM_HIGHEST_PAE_PAGE 0xFFFFF
|
|
|
|
ULONG MiFreePaeEntries;
|
|
PAE_ENTRY MiFirstFreePae;
|
|
|
|
LONG MmAllocatedPaePages;
|
|
KSPIN_LOCK MiPaeLock;
|
|
|
|
SLIST_HEADER MiPaeEntrySList;
|
|
|
|
PAE_ENTRY MiSystemPaeVa;
|
|
|
|
LONG
|
|
MiPaeAllocatePages (
|
|
VOID
|
|
);
|
|
|
|
VOID
|
|
MiPaeFreePages (
|
|
PVOID VirtualAddress
|
|
);
|
|
|
|
#pragma alloc_text(INIT,MiPaeInitialize)
|
|
#pragma alloc_text(PAGE,MiPaeFreePages)
|
|
|
|
VOID
|
|
MiMarkMdlPageAttributes (
|
|
IN PMDL Mdl,
|
|
IN PFN_NUMBER NumberOfPages,
|
|
IN MI_PFN_CACHE_ATTRIBUTE CacheAttribute
|
|
);
|
|
|
|
VOID
|
|
MiPaeInitialize (
|
|
VOID
|
|
)
|
|
{
|
|
InitializeSListHead (&MiPaeEntrySList);
|
|
KeInitializeSpinLock (&MiPaeLock);
|
|
InitializeListHead (&MiFirstFreePae.PaeEntry.ListHead);
|
|
}
|
|
|
|
ULONG
|
|
MiPaeAllocate (
|
|
OUT PPAE_ENTRY *Va
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine allocates the top level page directory pointer structure.
|
|
This structure will contain 4 PDPTEs.
|
|
|
|
Arguments:
|
|
|
|
Va - Supplies a place to put the virtual address this page can be accessed
|
|
at.
|
|
|
|
Return Value:
|
|
|
|
Returns a virtual and physical address suitable for use as a top
|
|
level page directory pointer page. The page returned must be below
|
|
physical 4GB as required by the processor.
|
|
|
|
Returns 0 if no page was allocated.
|
|
|
|
Environment:
|
|
|
|
Kernel mode. No locks may be held.
|
|
|
|
--*/
|
|
|
|
{
|
|
LOGICAL FlushedOnce;
|
|
PPAE_ENTRY Pae2;
|
|
PPAE_ENTRY Pae3;
|
|
PPAE_ENTRY Pae3Base;
|
|
PPAE_ENTRY Pae;
|
|
PPAE_ENTRY PaeBase;
|
|
PFN_NUMBER PageFrameIndex;
|
|
PSINGLE_LIST_ENTRY SingleListEntry;
|
|
ULONG j;
|
|
ULONG Entries;
|
|
KLOCK_QUEUE_HANDLE LockHandle;
|
|
#if DBG
|
|
PMMPFN Pfn1;
|
|
#endif
|
|
|
|
FlushedOnce = FALSE;
|
|
|
|
ASSERT (KeGetCurrentIrql() <= APC_LEVEL);
|
|
|
|
do {
|
|
|
|
//
|
|
// Pop an entry from the freelist.
|
|
//
|
|
|
|
SingleListEntry = InterlockedPopEntrySList (&MiPaeEntrySList);
|
|
|
|
if (SingleListEntry != NULL) {
|
|
Pae = CONTAINING_RECORD (SingleListEntry,
|
|
PAE_ENTRY,
|
|
NextPae);
|
|
|
|
PaeBase = (PPAE_ENTRY)PAGE_ALIGN(Pae);
|
|
|
|
*Va = Pae;
|
|
|
|
PageFrameIndex = PaeBase->PaeEntry.PageFrameNumber;
|
|
ASSERT (PageFrameIndex <= MM_HIGHEST_PAE_PAGE);
|
|
|
|
return (PageFrameIndex << PAGE_SHIFT) + BYTE_OFFSET (Pae);
|
|
}
|
|
|
|
KeAcquireInStackQueuedSpinLock (&MiPaeLock, &LockHandle);
|
|
|
|
if (MiFreePaeEntries != 0) {
|
|
|
|
ASSERT (IsListEmpty (&MiFirstFreePae.PaeEntry.ListHead) == 0);
|
|
|
|
Pae = (PPAE_ENTRY) RemoveHeadList (&MiFirstFreePae.PaeEntry.ListHead);
|
|
|
|
PaeBase = (PPAE_ENTRY)PAGE_ALIGN(Pae);
|
|
PaeBase->PaeEntry.EntriesInUse += 1;
|
|
#if DBG
|
|
RtlZeroMemory ((PVOID)Pae, sizeof(PAE_ENTRY));
|
|
|
|
Pfn1 = MI_PFN_ELEMENT (PaeBase->PaeEntry.PageFrameNumber);
|
|
ASSERT (Pfn1->u2.ShareCount == 1);
|
|
ASSERT (Pfn1->u3.e2.ReferenceCount == 1);
|
|
ASSERT (Pfn1->u3.e1.PageLocation == ActiveAndValid);
|
|
ASSERT (Pfn1->u3.e1.CacheAttribute == MiCached);
|
|
#endif
|
|
|
|
MiFreePaeEntries -= 1;
|
|
|
|
//
|
|
// Since we're holding the spinlock, dequeue a chain of entries
|
|
// for the SLIST.
|
|
//
|
|
|
|
Entries = MiFreePaeEntries;
|
|
|
|
if (Entries != 0) {
|
|
if (Entries > MINIMUM_PAE_SLIST_THRESHOLD) {
|
|
Entries = MINIMUM_PAE_SLIST_THRESHOLD;
|
|
}
|
|
|
|
ASSERT (IsListEmpty (&MiFirstFreePae.PaeEntry.ListHead) == 0);
|
|
|
|
Pae2 = (PPAE_ENTRY) RemoveHeadList (&MiFirstFreePae.PaeEntry.ListHead);
|
|
Pae2->NextPae.Next = NULL;
|
|
Pae3 = Pae2;
|
|
Pae3Base = (PPAE_ENTRY)PAGE_ALIGN(Pae3);
|
|
Pae3Base->PaeEntry.EntriesInUse += 1;
|
|
|
|
for (j = 1; j < Entries; j += 1) {
|
|
ASSERT (IsListEmpty (&MiFirstFreePae.PaeEntry.ListHead) == 0);
|
|
|
|
Pae3->NextPae.Next = (PSINGLE_LIST_ENTRY) RemoveHeadList (&MiFirstFreePae.PaeEntry.ListHead);
|
|
|
|
Pae3 = (PPAE_ENTRY) Pae3->NextPae.Next;
|
|
Pae3Base = (PPAE_ENTRY)PAGE_ALIGN(Pae3);
|
|
Pae3Base->PaeEntry.EntriesInUse += 1;
|
|
}
|
|
|
|
MiFreePaeEntries -= Entries;
|
|
|
|
KeReleaseInStackQueuedSpinLock (&LockHandle);
|
|
|
|
Pae3->NextPae.Next = NULL;
|
|
|
|
InterlockedPushListSList (&MiPaeEntrySList,
|
|
(PSINGLE_LIST_ENTRY) Pae2,
|
|
(PSINGLE_LIST_ENTRY) Pae3,
|
|
Entries);
|
|
}
|
|
else {
|
|
KeReleaseInStackQueuedSpinLock (&LockHandle);
|
|
}
|
|
|
|
ASSERT (KeGetCurrentIrql() <= APC_LEVEL);
|
|
*Va = Pae;
|
|
|
|
PageFrameIndex = PaeBase->PaeEntry.PageFrameNumber;
|
|
ASSERT (PageFrameIndex <= MM_HIGHEST_PAE_PAGE);
|
|
|
|
return (PageFrameIndex << PAGE_SHIFT) + BYTE_OFFSET (Pae);
|
|
}
|
|
|
|
KeReleaseInStackQueuedSpinLock (&LockHandle);
|
|
|
|
if (FlushedOnce == TRUE) {
|
|
break;
|
|
}
|
|
|
|
//
|
|
// No free pages in the cachelist, replenish the list now.
|
|
//
|
|
|
|
if (MiPaeAllocatePages () == 0) {
|
|
|
|
InterlockedIncrement (&MiDelayPageFaults);
|
|
|
|
//
|
|
// Attempt to move pages to the standby list.
|
|
//
|
|
|
|
MmEmptyAllWorkingSets ();
|
|
MiFlushAllPages();
|
|
|
|
KeDelayExecutionThread (KernelMode,
|
|
FALSE,
|
|
(PLARGE_INTEGER)&MmHalfSecond);
|
|
|
|
InterlockedDecrement (&MiDelayPageFaults);
|
|
|
|
FlushedOnce = TRUE;
|
|
|
|
//
|
|
// Since all the working sets have been trimmed, check whether
|
|
// another thread has replenished our list. If not, then attempt
|
|
// to do so since the working set pain has already been absorbed.
|
|
//
|
|
|
|
if (MiFreePaeEntries < MINIMUM_PAE_THRESHOLD) {
|
|
MiPaeAllocatePages ();
|
|
}
|
|
}
|
|
|
|
} while (TRUE);
|
|
|
|
ASSERT (KeGetCurrentIrql() <= APC_LEVEL);
|
|
|
|
return 0;
|
|
}
|
|
|
|
VOID
|
|
MiPaeFree (
|
|
PPAE_ENTRY Pae
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine releases the top level page directory pointer page.
|
|
|
|
Arguments:
|
|
|
|
PageFrameIndex - Supplies the top level page directory pointer page.
|
|
|
|
Return Value:
|
|
|
|
None.
|
|
|
|
Environment:
|
|
|
|
Kernel mode. No locks may be held.
|
|
|
|
--*/
|
|
|
|
{
|
|
ULONG i;
|
|
PLIST_ENTRY NextEntry;
|
|
PPAE_ENTRY PaeBase;
|
|
KLOCK_QUEUE_HANDLE LockHandle;
|
|
|
|
#if DBG
|
|
PMMPTE PointerPte;
|
|
PFN_NUMBER PageFrameIndex;
|
|
PMMPFN Pfn1;
|
|
|
|
PointerPte = MiGetPteAddress (Pae);
|
|
PageFrameIndex = MI_GET_PAGE_FRAME_FROM_PTE (PointerPte);
|
|
|
|
//
|
|
// This page must be in the first 4GB of RAM.
|
|
//
|
|
|
|
ASSERT (PageFrameIndex <= MM_HIGHEST_PAE_PAGE);
|
|
|
|
Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
|
|
|
|
ASSERT (Pfn1->u2.ShareCount == 1);
|
|
ASSERT (Pfn1->u3.e2.ReferenceCount == 1);
|
|
ASSERT (Pfn1->u3.e1.PageLocation == ActiveAndValid);
|
|
ASSERT (Pfn1->u3.e1.CacheAttribute == MiCached);
|
|
#endif
|
|
|
|
if (ExQueryDepthSList (&MiPaeEntrySList) < MINIMUM_PAE_SLIST_THRESHOLD) {
|
|
InterlockedPushEntrySList (&MiPaeEntrySList, &Pae->NextPae);
|
|
return;
|
|
}
|
|
|
|
PaeBase = (PPAE_ENTRY)PAGE_ALIGN(Pae);
|
|
|
|
KeAcquireInStackQueuedSpinLock (&MiPaeLock, &LockHandle);
|
|
|
|
PaeBase->PaeEntry.EntriesInUse -= 1;
|
|
|
|
if ((PaeBase->PaeEntry.EntriesInUse == 0) &&
|
|
(MiFreePaeEntries > EXCESS_PAE_THRESHOLD)) {
|
|
|
|
//
|
|
// Free the entire page.
|
|
//
|
|
|
|
i = 1;
|
|
NextEntry = MiFirstFreePae.PaeEntry.ListHead.Flink;
|
|
while (NextEntry != &MiFirstFreePae.PaeEntry.ListHead) {
|
|
|
|
Pae = CONTAINING_RECORD (NextEntry,
|
|
PAE_ENTRY,
|
|
PaeEntry.ListHead);
|
|
|
|
if (PAGE_ALIGN(Pae) == PaeBase) {
|
|
RemoveEntryList (NextEntry);
|
|
i += 1;
|
|
}
|
|
NextEntry = Pae->PaeEntry.ListHead.Flink;
|
|
}
|
|
ASSERT (i == PAES_PER_PAGE - 1);
|
|
MiFreePaeEntries -= (PAES_PER_PAGE - 1);
|
|
KeReleaseInStackQueuedSpinLock (&LockHandle);
|
|
|
|
MiPaeFreePages (PaeBase);
|
|
}
|
|
else {
|
|
|
|
InsertTailList (&MiFirstFreePae.PaeEntry.ListHead,
|
|
&Pae->PaeEntry.ListHead);
|
|
MiFreePaeEntries += 1;
|
|
KeReleaseInStackQueuedSpinLock (&LockHandle);
|
|
}
|
|
|
|
return;
|
|
}
|
|
|
|
LONG
|
|
MiPaeAllocatePages (
|
|
VOID
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine replenishes the PAE top level mapping list.
|
|
|
|
Arguments:
|
|
|
|
None.
|
|
|
|
Return Value:
|
|
|
|
The number of pages allocated.
|
|
|
|
Environment:
|
|
|
|
Kernel mode, IRQL of APC_LEVEL or below.
|
|
|
|
--*/
|
|
{
|
|
PMDL MemoryDescriptorList;
|
|
LONG AllocatedPaePages;
|
|
ULONG i;
|
|
ULONG j;
|
|
PPFN_NUMBER SlidePage;
|
|
PPFN_NUMBER Page;
|
|
PFN_NUMBER PageFrameIndex;
|
|
ULONG_PTR ActualPages;
|
|
PMMPTE PointerPte;
|
|
PVOID BaseAddress;
|
|
PPAE_ENTRY Pae;
|
|
ULONG NumberOfPages;
|
|
MMPTE TempPte;
|
|
PHYSICAL_ADDRESS HighAddress;
|
|
PHYSICAL_ADDRESS LowAddress;
|
|
PHYSICAL_ADDRESS SkipBytes;
|
|
KLOCK_QUEUE_HANDLE LockHandle;
|
|
|
|
#if defined (_MI_MORE_THAN_4GB_)
|
|
if (MiNoLowMemory != 0) {
|
|
BaseAddress = MiAllocateLowMemory (PAGE_SIZE,
|
|
0,
|
|
MiNoLowMemory - 1,
|
|
0,
|
|
(PVOID)0x123,
|
|
MmCached,
|
|
'DeaP');
|
|
if (BaseAddress == NULL) {
|
|
return 0;
|
|
}
|
|
|
|
PageFrameIndex = MI_GET_PAGE_FRAME_FROM_PTE (MiGetPteAddress(BaseAddress));
|
|
|
|
Pae = (PPAE_ENTRY) BaseAddress;
|
|
Pae->PaeEntry.EntriesInUse = 0;
|
|
Pae->PaeEntry.PageFrameNumber = PageFrameIndex;
|
|
Pae += 1;
|
|
|
|
KeAcquireInStackQueuedSpinLock (&MiPaeLock, &LockHandle);
|
|
|
|
for (i = 1; i < PAES_PER_PAGE; i += 1) {
|
|
InsertTailList (&MiFirstFreePae.PaeEntry.ListHead,
|
|
&Pae->PaeEntry.ListHead);
|
|
Pae += 1;
|
|
}
|
|
MiFreePaeEntries += (PAES_PER_PAGE - 1);
|
|
|
|
KeReleaseInStackQueuedSpinLock (&LockHandle);
|
|
|
|
InterlockedIncrement (&MmAllocatedPaePages);
|
|
return 1;
|
|
}
|
|
#endif
|
|
|
|
NumberOfPages = REPLENISH_PAE_SIZE / PAES_PER_PAGE;
|
|
AllocatedPaePages = 0;
|
|
|
|
HighAddress.QuadPart = (ULONGLONG)_4gb - 1;
|
|
LowAddress.QuadPart = 0;
|
|
SkipBytes.QuadPart = 0;
|
|
|
|
//
|
|
// This is a potentially expensive call so pick up a chunk of pages
|
|
// at once to amortize the cost.
|
|
//
|
|
|
|
MemoryDescriptorList = MmAllocatePagesForMdl (LowAddress,
|
|
HighAddress,
|
|
SkipBytes,
|
|
NumberOfPages << PAGE_SHIFT);
|
|
|
|
if (MemoryDescriptorList == NULL) {
|
|
return 0;
|
|
}
|
|
|
|
ActualPages = MemoryDescriptorList->ByteCount >> PAGE_SHIFT;
|
|
|
|
MiMarkMdlPageAttributes (MemoryDescriptorList, ActualPages, MiCached);
|
|
|
|
TempPte = ValidKernelPte;
|
|
Page = (PPFN_NUMBER)(MemoryDescriptorList + 1);
|
|
|
|
//
|
|
// Map each page individually as they may need to be freed individually
|
|
// later.
|
|
//
|
|
|
|
for (i = 0; i < ActualPages; i += 1) {
|
|
PageFrameIndex = *Page;
|
|
|
|
PointerPte = MiReserveSystemPtes (1, SystemPteSpace);
|
|
|
|
if (PointerPte == NULL) {
|
|
|
|
//
|
|
// Free any remaining pages in the MDL as they are not mapped.
|
|
// Slide the MDL pages forward so the mapped ones are kept.
|
|
//
|
|
|
|
MmInitializeMdl (MemoryDescriptorList,
|
|
0,
|
|
(ActualPages - i) << PAGE_SHIFT);
|
|
|
|
SlidePage = (PPFN_NUMBER)(MemoryDescriptorList + 1);
|
|
|
|
while (i < ActualPages) {
|
|
i += 1;
|
|
*SlidePage = *Page;
|
|
SlidePage += 1;
|
|
Page += 1;
|
|
}
|
|
|
|
MmFreePagesFromMdl (MemoryDescriptorList);
|
|
|
|
break;
|
|
}
|
|
|
|
TempPte.u.Hard.PageFrameNumber = PageFrameIndex;
|
|
MI_WRITE_VALID_PTE (PointerPte, TempPte);
|
|
|
|
BaseAddress = MiGetVirtualAddressMappedByPte (PointerPte);
|
|
|
|
Pae = (PPAE_ENTRY) BaseAddress;
|
|
|
|
Pae->PaeEntry.EntriesInUse = 0;
|
|
Pae->PaeEntry.PageFrameNumber = PageFrameIndex;
|
|
Pae += 1;
|
|
|
|
//
|
|
// Put the first chunk into the SLIST if it's still low, and just
|
|
// enqueue all the other entries normally.
|
|
//
|
|
|
|
if ((i == 0) &&
|
|
(ExQueryDepthSList (&MiPaeEntrySList) < MINIMUM_PAE_SLIST_THRESHOLD)) {
|
|
|
|
(Pae - 1)->PaeEntry.EntriesInUse = PAES_PER_PAGE - 1;
|
|
|
|
for (j = 1; j < PAES_PER_PAGE - 1; j += 1) {
|
|
Pae->NextPae.Next = (PSINGLE_LIST_ENTRY) (Pae + 1);
|
|
Pae += 1;
|
|
}
|
|
|
|
Pae->NextPae.Next = NULL;
|
|
|
|
InterlockedPushListSList (&MiPaeEntrySList,
|
|
(PSINGLE_LIST_ENTRY)((PPAE_ENTRY) BaseAddress + 1),
|
|
(PSINGLE_LIST_ENTRY) Pae,
|
|
PAES_PER_PAGE - 1);
|
|
}
|
|
else {
|
|
|
|
KeAcquireInStackQueuedSpinLock (&MiPaeLock, &LockHandle);
|
|
|
|
for (j = 1; j < PAES_PER_PAGE; j += 1) {
|
|
InsertTailList (&MiFirstFreePae.PaeEntry.ListHead,
|
|
&Pae->PaeEntry.ListHead);
|
|
Pae += 1;
|
|
}
|
|
|
|
MiFreePaeEntries += (PAES_PER_PAGE - 1);
|
|
|
|
KeReleaseInStackQueuedSpinLock (&LockHandle);
|
|
}
|
|
|
|
AllocatedPaePages += 1;
|
|
|
|
Page += 1;
|
|
}
|
|
|
|
ExFreePool (MemoryDescriptorList);
|
|
|
|
InterlockedExchangeAdd (&MmAllocatedPaePages, AllocatedPaePages);
|
|
|
|
return AllocatedPaePages;
|
|
}
|
|
|
|
VOID
|
|
MiPaeFreePages (
|
|
PVOID VirtualAddress
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine releases a single page that previously contained top level
|
|
page directory pointer pages.
|
|
|
|
Arguments:
|
|
|
|
VirtualAddress - Supplies the virtual address of the page that contained
|
|
top level page directory pointer pages.
|
|
|
|
Return Value:
|
|
|
|
None.
|
|
|
|
Environment:
|
|
|
|
Kernel mode. No locks held.
|
|
|
|
--*/
|
|
|
|
{
|
|
ULONG MdlPages;
|
|
PFN_NUMBER PageFrameIndex;
|
|
PMMPTE PointerPte;
|
|
PFN_NUMBER MdlHack[(sizeof(MDL) / sizeof(PFN_NUMBER)) + 1];
|
|
PPFN_NUMBER MdlPage;
|
|
PMDL MemoryDescriptorList;
|
|
|
|
#if defined (_MI_MORE_THAN_4GB_)
|
|
if (MiNoLowMemory != 0) {
|
|
if (MiFreeLowMemory (VirtualAddress, 'DeaP') == TRUE) {
|
|
InterlockedDecrement (&MmAllocatedPaePages);
|
|
return;
|
|
}
|
|
}
|
|
#endif
|
|
|
|
MemoryDescriptorList = (PMDL)&MdlHack[0];
|
|
MdlPages = 1;
|
|
MmInitializeMdl (MemoryDescriptorList, 0, MdlPages << PAGE_SHIFT);
|
|
|
|
MdlPage = (PPFN_NUMBER)(MemoryDescriptorList + 1);
|
|
|
|
PointerPte = MiGetPteAddress (VirtualAddress);
|
|
PageFrameIndex = MI_GET_PAGE_FRAME_FROM_PTE (PointerPte);
|
|
*MdlPage = PageFrameIndex;
|
|
|
|
ASSERT ((MI_PFN_ELEMENT(PageFrameIndex))->u3.e1.CacheAttribute == MiCached);
|
|
|
|
MiReleaseSystemPtes (PointerPte, 1, SystemPteSpace);
|
|
|
|
MmFreePagesFromMdl (MemoryDescriptorList);
|
|
|
|
InterlockedDecrement (&MmAllocatedPaePages);
|
|
}
|
|
|
|
#endif
|