mirror of https://github.com/tongzx/nt5src
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
7483 lines
186 KiB
7483 lines
186 KiB
/*++
|
|
|
|
Copyright (c) 1989 Microsoft Corporation
|
|
|
|
Module Name:
|
|
|
|
mi.h
|
|
|
|
Abstract:
|
|
|
|
This module contains the private data structures and procedure
|
|
prototypes for the memory management system.
|
|
|
|
Author:
|
|
|
|
Lou Perazzoli (loup) 20-Mar-1989
|
|
Landy Wang (landyw) 02-Jun-1997
|
|
|
|
Revision History:
|
|
|
|
--*/
|
|
|
|
#ifndef _MI_
|
|
#define _MI_
|
|
|
|
#pragma warning(disable:4214) // bit field types other than int
|
|
#pragma warning(disable:4201) // nameless struct/union
|
|
#pragma warning(disable:4324) // alignment sensitive to declspec
|
|
#pragma warning(disable:4127) // condition expression is constant
|
|
#pragma warning(disable:4115) // named type definition in parentheses
|
|
#pragma warning(disable:4232) // dllimport not static
|
|
#pragma warning(disable:4206) // translation unit empty
|
|
|
|
#include "ntos.h"
|
|
#include "ntimage.h"
|
|
#include "ki.h"
|
|
#include "fsrtl.h"
|
|
#include "zwapi.h"
|
|
#include "pool.h"
|
|
#include "stdio.h"
|
|
#include "string.h"
|
|
#include "safeboot.h"
|
|
#include "triage.h"
|
|
#include "xip.h"
|
|
|
|
#if defined(_X86_)
|
|
#include "..\mm\i386\mi386.h"
|
|
|
|
#elif defined(_AMD64_)
|
|
#include "..\mm\amd64\miamd.h"
|
|
|
|
#elif defined(_IA64_)
|
|
#include "..\mm\ia64\miia64.h"
|
|
|
|
#else
|
|
#error "mm: a target architecture must be defined."
|
|
#endif
|
|
|
|
#if defined (_WIN64)
|
|
#define ASSERT32(exp)
|
|
#define ASSERT64(exp) ASSERT(exp)
|
|
#else
|
|
#define ASSERT32(exp) ASSERT(exp)
|
|
#define ASSERT64(exp)
|
|
#endif
|
|
|
|
//
|
|
// Special pool constants
|
|
//
|
|
#define MI_SPECIAL_POOL_PAGABLE 0x8000
|
|
#define MI_SPECIAL_POOL_VERIFIER 0x4000
|
|
#define MI_SPECIAL_POOL_IN_SESSION 0x2000
|
|
#define MI_SPECIAL_POOL_PTE_PAGABLE 0x0002
|
|
#define MI_SPECIAL_POOL_PTE_NONPAGABLE 0x0004
|
|
|
|
|
|
#define _2gb 0x80000000 // 2 gigabytes
|
|
#define _4gb 0x100000000 // 4 gigabytes
|
|
|
|
#define MM_FLUSH_COUNTER_MASK (0xFFFFF)
|
|
|
|
#define MM_FREE_WSLE_SHIFT 4
|
|
|
|
#define WSLE_NULL_INDEX ((((WSLE_NUMBER)-1) >> MM_FREE_WSLE_SHIFT))
|
|
|
|
#define MM_FREE_POOL_SIGNATURE (0x50554F4C)
|
|
|
|
#define MM_MINIMUM_PAGED_POOL_NTAS ((SIZE_T)(48*1024*1024))
|
|
|
|
#define MM_ALLOCATION_FILLS_VAD ((PMMPTE)(ULONG_PTR)~3)
|
|
|
|
#define MM_WORKING_SET_LIST_SEARCH 17
|
|
|
|
#define MM_FLUID_WORKING_SET 8
|
|
|
|
#define MM_FLUID_PHYSICAL_PAGES 32 //see MmResidentPages below.
|
|
|
|
#define MM_USABLE_PAGES_FREE 32
|
|
|
|
#define X64K (ULONG)65536
|
|
|
|
#define MM_HIGHEST_VAD_ADDRESS ((PVOID)((ULONG_PTR)MM_HIGHEST_USER_ADDRESS - (64 * 1024)))
|
|
|
|
|
|
#define MM_NO_WS_EXPANSION ((PLIST_ENTRY)0)
|
|
#define MM_WS_EXPANSION_IN_PROGRESS ((PLIST_ENTRY)35)
|
|
#define MM_WS_SWAPPED_OUT ((PLIST_ENTRY)37)
|
|
#define MM_IO_IN_PROGRESS ((PLIST_ENTRY)97) // MUST HAVE THE HIGHEST VALUE
|
|
|
|
#define MM4K_SHIFT 12 //MUST BE LESS THAN OR EQUAL TO PAGE_SHIFT
|
|
#define MM4K_MASK 0xfff
|
|
|
|
#define MMSECTOR_SHIFT 9 //MUST BE LESS THAN OR EQUAL TO PAGE_SHIFT
|
|
|
|
#define MMSECTOR_MASK 0x1ff
|
|
|
|
#define MM_LOCK_BY_REFCOUNT 0
|
|
|
|
#define MM_LOCK_BY_NONPAGE 1
|
|
|
|
#define MM_FORCE_TRIM 6
|
|
|
|
#define MM_GROW_WSLE_HASH 20
|
|
|
|
#define MM_MAXIMUM_WRITE_CLUSTER (MM_MAXIMUM_DISK_IO_SIZE / PAGE_SIZE)
|
|
|
|
//
|
|
// Number of PTEs to flush singularly before flushing the entire TB.
|
|
//
|
|
|
|
#define MM_MAXIMUM_FLUSH_COUNT (FLUSH_MULTIPLE_MAXIMUM-1)
|
|
|
|
//
|
|
// Page protections
|
|
//
|
|
|
|
#define MM_ZERO_ACCESS 0 // this value is not used.
|
|
#define MM_READONLY 1
|
|
#define MM_EXECUTE 2
|
|
#define MM_EXECUTE_READ 3
|
|
#define MM_READWRITE 4 // bit 2 is set if this is writable.
|
|
#define MM_WRITECOPY 5
|
|
#define MM_EXECUTE_READWRITE 6
|
|
#define MM_EXECUTE_WRITECOPY 7
|
|
|
|
#define MM_NOCACHE 0x8
|
|
#define MM_GUARD_PAGE 0x10
|
|
#define MM_DECOMMIT 0x10 //NO_ACCESS, Guard page
|
|
#define MM_NOACCESS 0x18 //NO_ACCESS, Guard_page, nocache.
|
|
#define MM_UNKNOWN_PROTECTION 0x100 //bigger than 5 bits!
|
|
#define MM_LARGE_PAGES 0x111
|
|
|
|
#define MM_INVALID_PROTECTION ((ULONG)-1) //bigger than 5 bits!
|
|
|
|
#define MM_KSTACK_OUTSWAPPED 0x1F // Denotes outswapped kernel stack pages.
|
|
|
|
#define MM_PROTECTION_WRITE_MASK 4
|
|
#define MM_PROTECTION_COPY_MASK 1
|
|
#define MM_PROTECTION_OPERATION_MASK 7 // mask off guard page and nocache.
|
|
#define MM_PROTECTION_EXECUTE_MASK 2
|
|
|
|
#define MM_SECURE_DELETE_CHECK 0x55
|
|
|
|
//
|
|
// Debug flags
|
|
//
|
|
|
|
#define MM_DBG_WRITEFAULT 0x1
|
|
#define MM_DBG_PTE_UPDATE 0x2
|
|
#define MM_DBG_DUMP_WSL 0x4
|
|
#define MM_DBG_PAGEFAULT 0x8
|
|
#define MM_DBG_WS_EXPANSION 0x10
|
|
#define MM_DBG_MOD_WRITE 0x20
|
|
#define MM_DBG_CHECK_PTE 0x40
|
|
#define MM_DBG_VAD_CONFLICT 0x80
|
|
#define MM_DBG_SECTIONS 0x100
|
|
#define MM_DBG_SYS_PTES 0x400
|
|
#define MM_DBG_CLEAN_PROCESS 0x800
|
|
#define MM_DBG_COLLIDED_PAGE 0x1000
|
|
#define MM_DBG_DUMP_BOOT_PTES 0x2000
|
|
#define MM_DBG_FORK 0x4000
|
|
#define MM_DBG_DIR_BASE 0x8000
|
|
#define MM_DBG_FLUSH_SECTION 0x10000
|
|
#define MM_DBG_PRINTS_MODWRITES 0x20000
|
|
#define MM_DBG_PAGE_IN_LIST 0x40000
|
|
#define MM_DBG_CHECK_PFN_LOCK 0x80000
|
|
#define MM_DBG_PRIVATE_PAGES 0x100000
|
|
#define MM_DBG_WALK_VAD_TREE 0x200000
|
|
#define MM_DBG_SWAP_PROCESS 0x400000
|
|
#define MM_DBG_LOCK_CODE 0x800000
|
|
#define MM_DBG_STOP_ON_ACCVIO 0x1000000
|
|
#define MM_DBG_PAGE_REF_COUNT 0x2000000
|
|
#define MM_DBG_SHOW_FAULTS 0x40000000
|
|
#define MM_DBG_SESSIONS 0x80000000
|
|
|
|
//
|
|
// If the PTE.protection & MM_COPY_ON_WRITE_MASK == MM_COPY_ON_WRITE_MASK
|
|
// then the PTE is copy on write.
|
|
//
|
|
|
|
#define MM_COPY_ON_WRITE_MASK 5
|
|
|
|
extern ULONG MmProtectToValue[32];
|
|
|
|
extern
|
|
#if (defined(_WIN64) || defined(_X86PAE_))
|
|
ULONGLONG
|
|
#else
|
|
ULONG
|
|
#endif
|
|
MmProtectToPteMask[32];
|
|
extern ULONG MmMakeProtectNotWriteCopy[32];
|
|
extern ACCESS_MASK MmMakeSectionAccess[8];
|
|
extern ACCESS_MASK MmMakeFileAccess[8];
|
|
|
|
|
|
//
|
|
// Time constants
|
|
//
|
|
|
|
extern const LARGE_INTEGER MmSevenMinutes;
|
|
extern LARGE_INTEGER MmWorkingSetProtectionTime;
|
|
const extern LARGE_INTEGER MmOneSecond;
|
|
const extern LARGE_INTEGER MmTwentySeconds;
|
|
const extern LARGE_INTEGER MmShortTime;
|
|
const extern LARGE_INTEGER MmHalfSecond;
|
|
const extern LARGE_INTEGER Mm30Milliseconds;
|
|
extern LARGE_INTEGER MmCriticalSectionTimeout;
|
|
|
|
//
|
|
// A month's worth
|
|
//
|
|
|
|
extern ULONG MmCritsectTimeoutSeconds;
|
|
|
|
//
|
|
// this is the csrss process !
|
|
//
|
|
|
|
extern PEPROCESS ExpDefaultErrorPortProcess;
|
|
|
|
extern SIZE_T MmExtendedCommit;
|
|
|
|
extern SIZE_T MmTotalProcessCommit;
|
|
|
|
//
|
|
// The total number of pages needed for the loader to successfully hibernate.
|
|
//
|
|
|
|
extern PFN_NUMBER MmHiberPages;
|
|
|
|
//
|
|
// The counters and reasons to retry IO to protect against verifier induced
|
|
// failures and temporary conditions.
|
|
//
|
|
|
|
extern ULONG MiIoRetryMask;
|
|
extern ULONG MiFaultRetryMask;
|
|
extern ULONG MiUserFaultRetryMask;
|
|
|
|
#define MmIsRetryIoStatus(S) (((S) == STATUS_INSUFFICIENT_RESOURCES) || \
|
|
((S) == STATUS_WORKING_SET_QUOTA) || \
|
|
((S) == STATUS_NO_MEMORY))
|
|
|
|
#if defined (_MI_MORE_THAN_4GB_)
|
|
|
|
extern PFN_NUMBER MiNoLowMemory;
|
|
|
|
#if defined (_WIN64)
|
|
#define MI_MAGIC_4GB_RECLAIM 0xffffedf0
|
|
#else
|
|
#define MI_MAGIC_4GB_RECLAIM 0xffedf0
|
|
#endif
|
|
|
|
#define MI_LOWMEM_MAGIC_BIT (0x80000000)
|
|
|
|
extern PRTL_BITMAP MiLowMemoryBitMap;
|
|
#endif
|
|
|
|
//
|
|
// This is a version of COMPUTE_PAGES_SPANNED that works for 32 and 64 ranges.
|
|
//
|
|
|
|
#define MI_COMPUTE_PAGES_SPANNED(Va, Size) \
|
|
((((ULONG_PTR)(Va) & (PAGE_SIZE -1)) + (Size) + (PAGE_SIZE - 1)) >> PAGE_SHIFT)
|
|
|
|
//++
|
|
//
|
|
// ULONG
|
|
// MI_CONVERT_FROM_PTE_PROTECTION (
|
|
// IN ULONG PROTECTION_MASK
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// This routine converts a PTE protection into a Protect value.
|
|
//
|
|
// Arguments:
|
|
//
|
|
//
|
|
// Return Value:
|
|
//
|
|
// Returns the
|
|
//
|
|
//--
|
|
|
|
#define MI_CONVERT_FROM_PTE_PROTECTION(PROTECTION_MASK) \
|
|
(MmProtectToValue[PROTECTION_MASK])
|
|
|
|
#define MI_MASK_TO_PTE(PROTECTION_MASK) MmProtectToPteMask[PROTECTION_MASK]
|
|
|
|
|
|
#define MI_IS_PTE_PROTECTION_COPY_WRITE(PROTECTION_MASK) \
|
|
(((PROTECTION_MASK) & MM_COPY_ON_WRITE_MASK) == MM_COPY_ON_WRITE_MASK)
|
|
|
|
//++
|
|
//
|
|
// ULONG
|
|
// MI_ROUND_TO_64K (
|
|
// IN ULONG LENGTH
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
//
|
|
// The ROUND_TO_64k macro takes a LENGTH in bytes and rounds it up to a multiple
|
|
// of 64K.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// LENGTH - LENGTH in bytes to round up to 64k.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// Returns the LENGTH rounded up to a multiple of 64k.
|
|
//
|
|
//--
|
|
|
|
#define MI_ROUND_TO_64K(LENGTH) (((LENGTH) + X64K - 1) & ~((ULONG_PTR)X64K - 1))
|
|
|
|
extern ULONG MiLastVadBit;
|
|
|
|
//++
|
|
//
|
|
// ULONG
|
|
// MI_ROUND_TO_SIZE (
|
|
// IN ULONG LENGTH,
|
|
// IN ULONG ALIGNMENT
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
//
|
|
// The ROUND_TO_SIZE macro takes a LENGTH in bytes and rounds it up to a
|
|
// multiple of the alignment.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// LENGTH - LENGTH in bytes to round up to.
|
|
//
|
|
// ALIGNMENT - alignment to round to, must be a power of 2, e.g, 2**n.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// Returns the LENGTH rounded up to a multiple of the alignment.
|
|
//
|
|
//--
|
|
|
|
#define MI_ROUND_TO_SIZE(LENGTH,ALIGNMENT) \
|
|
(((LENGTH) + ((ALIGNMENT) - 1)) & ~((ALIGNMENT) - 1))
|
|
|
|
//++
|
|
//
|
|
// PVOID
|
|
// MI_64K_ALIGN (
|
|
// IN PVOID VA
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
//
|
|
// The MI_64K_ALIGN macro takes a virtual address and returns a 64k-aligned
|
|
// virtual address for that page.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// VA - Virtual address.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// Returns the 64k aligned virtual address.
|
|
//
|
|
//--
|
|
|
|
#define MI_64K_ALIGN(VA) ((PVOID)((ULONG_PTR)(VA) & ~((LONG)X64K - 1)))
|
|
|
|
|
|
//++
|
|
//
|
|
// PVOID
|
|
// MI_ALIGN_TO_SIZE (
|
|
// IN PVOID VA
|
|
// IN ULONG ALIGNMENT
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
//
|
|
// The MI_ALIGN_TO_SIZE macro takes a virtual address and returns a
|
|
// virtual address for that page with the specified alignment.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// VA - Virtual address.
|
|
//
|
|
// ALIGNMENT - alignment to round to, must be a power of 2, e.g, 2**n.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// Returns the aligned virtual address.
|
|
//
|
|
//--
|
|
|
|
#define MI_ALIGN_TO_SIZE(VA,ALIGNMENT) ((PVOID)((ULONG_PTR)(VA) & ~((ULONG_PTR) ALIGNMENT - 1)))
|
|
|
|
//++
|
|
//
|
|
// LONGLONG
|
|
// MI_STARTING_OFFSET (
|
|
// IN PSUBSECTION SUBSECT
|
|
// IN PMMPTE PTE
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// This macro takes a pointer to a PTE within a subsection and a pointer
|
|
// to that subsection and calculates the offset for that PTE within the
|
|
// file.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// PTE - PTE within subsection.
|
|
//
|
|
// SUBSECT - Subsection
|
|
//
|
|
// Return Value:
|
|
//
|
|
// Offset for issuing I/O from.
|
|
//
|
|
//--
|
|
|
|
#define MI_STARTING_OFFSET(SUBSECT,PTE) \
|
|
(((LONGLONG)((ULONG_PTR)((PTE) - ((SUBSECT)->SubsectionBase))) << PAGE_SHIFT) + \
|
|
((LONGLONG)((SUBSECT)->StartingSector) << MMSECTOR_SHIFT));
|
|
|
|
|
|
// NTSTATUS
|
|
// MiFindEmptyAddressRangeDown (
|
|
// IN ULONG_PTR SizeOfRange,
|
|
// IN PVOID HighestAddressToEndAt,
|
|
// IN ULONG_PTR Alignment,
|
|
// OUT PVOID *Base
|
|
// )
|
|
//
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// The function examines the virtual address descriptors to locate
|
|
// an unused range of the specified size and returns the starting
|
|
// address of the range. This routine looks from the top down.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// SizeOfRange - Supplies the size in bytes of the range to locate.
|
|
//
|
|
// HighestAddressToEndAt - Supplies the virtual address to begin looking
|
|
// at.
|
|
//
|
|
// Alignment - Supplies the alignment for the address. Must be
|
|
// a power of 2 and greater than the page_size.
|
|
//
|
|
//Return Value:
|
|
//
|
|
// Returns the starting address of a suitable range.
|
|
//
|
|
|
|
#define MiFindEmptyAddressRangeDown(Root,SizeOfRange,HighestAddressToEndAt,Alignment,Base) \
|
|
(MiFindEmptyAddressRangeDownTree( \
|
|
(SizeOfRange), \
|
|
(HighestAddressToEndAt), \
|
|
(Alignment), \
|
|
(PMMADDRESS_NODE)(Root), \
|
|
(Base)))
|
|
|
|
// PMMVAD
|
|
// MiGetPreviousVad (
|
|
// IN PMMVAD Vad
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// This function locates the virtual address descriptor which contains
|
|
// the address range which logically precedes the specified virtual
|
|
// address descriptor.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// Vad - Supplies a pointer to a virtual address descriptor.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// Returns a pointer to the virtual address descriptor containing the
|
|
// next address range, NULL if none.
|
|
//
|
|
//
|
|
|
|
#define MiGetPreviousVad(VAD) ((PMMVAD)MiGetPreviousNode((PMMADDRESS_NODE)(VAD)))
|
|
|
|
|
|
// PMMVAD
|
|
// MiGetNextVad (
|
|
// IN PMMVAD Vad
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// This function locates the virtual address descriptor which contains
|
|
// the address range which logically follows the specified address range.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// VAD - Supplies a pointer to a virtual address descriptor.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// Returns a pointer to the virtual address descriptor containing the
|
|
// next address range, NULL if none.
|
|
//
|
|
|
|
#define MiGetNextVad(VAD) ((PMMVAD)MiGetNextNode((PMMADDRESS_NODE)(VAD)))
|
|
|
|
|
|
|
|
// PMMVAD
|
|
// MiGetFirstVad (
|
|
// Process
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// This function locates the virtual address descriptor which contains
|
|
// the address range which logically is first within the address space.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// Process - Specifies the process in which to locate the VAD.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// Returns a pointer to the virtual address descriptor containing the
|
|
// first address range, NULL if none.
|
|
|
|
#define MiGetFirstVad(Process) \
|
|
((PMMVAD)MiGetFirstNode((PMMADDRESS_NODE)(Process->VadRoot)))
|
|
|
|
|
|
LOGICAL
|
|
MiCheckForConflictingVadExistence (
|
|
IN PEPROCESS Process,
|
|
IN PVOID StartingAddress,
|
|
IN PVOID EndingAddress
|
|
);
|
|
|
|
// PMMVAD
|
|
// MiCheckForConflictingVad (
|
|
// IN PVOID StartingAddress,
|
|
// IN PVOID EndingAddress
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// The function determines if any addresses between a given starting and
|
|
// ending address is contained within a virtual address descriptor.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// StartingAddress - Supplies the virtual address to locate a containing
|
|
// descriptor.
|
|
//
|
|
// EndingAddress - Supplies the virtual address to locate a containing
|
|
// descriptor.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// Returns a pointer to the first conflicting virtual address descriptor
|
|
// if one is found, otherwise a NULL value is returned.
|
|
//
|
|
|
|
#define MiCheckForConflictingVad(CurrentProcess,StartingAddress,EndingAddress) \
|
|
((PMMVAD)MiCheckForConflictingNode( \
|
|
MI_VA_TO_VPN(StartingAddress), \
|
|
MI_VA_TO_VPN(EndingAddress), \
|
|
(PMMADDRESS_NODE)(CurrentProcess->VadRoot)))
|
|
|
|
// PMMCLONE_DESCRIPTOR
|
|
// MiGetNextClone (
|
|
// IN PMMCLONE_DESCRIPTOR Clone
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// This function locates the virtual address descriptor which contains
|
|
// the address range which logically follows the specified address range.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// Clone - Supplies a pointer to a virtual address descriptor.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// Returns a pointer to the virtual address descriptor containing the
|
|
// next address range, NULL if none.
|
|
//
|
|
//
|
|
|
|
#define MiGetNextClone(CLONE) \
|
|
((PMMCLONE_DESCRIPTOR)MiGetNextNode((PMMADDRESS_NODE)(CLONE)))
|
|
|
|
|
|
|
|
// PMMCLONE_DESCRIPTOR
|
|
// MiGetPreviousClone (
|
|
// IN PMMCLONE_DESCRIPTOR Clone
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// This function locates the virtual address descriptor which contains
|
|
// the address range which logically precedes the specified virtual
|
|
// address descriptor.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// Clone - Supplies a pointer to a virtual address descriptor.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// Returns a pointer to the virtual address descriptor containing the
|
|
// next address range, NULL if none.
|
|
|
|
|
|
#define MiGetPreviousClone(CLONE) \
|
|
((PMMCLONE_DESCRIPTOR)MiGetPreviousNode((PMMADDRESS_NODE)(CLONE)))
|
|
|
|
|
|
|
|
// PMMCLONE_DESCRIPTOR
|
|
// MiGetFirstClone (
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// This function locates the virtual address descriptor which contains
|
|
// the address range which logically is first within the address space.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// None.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// Returns a pointer to the virtual address descriptor containing the
|
|
// first address range, NULL if none.
|
|
//
|
|
|
|
|
|
#define MiGetFirstClone(_CurrentProcess) \
|
|
((PMMCLONE_DESCRIPTOR)MiGetFirstNode((PMMADDRESS_NODE)(_CurrentProcess->CloneRoot)))
|
|
|
|
|
|
|
|
// VOID
|
|
// MiInsertClone (
|
|
// IN PMMCLONE_DESCRIPTOR Clone
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// This function inserts a virtual address descriptor into the tree and
|
|
// reorders the splay tree as appropriate.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// Clone - Supplies a pointer to a virtual address descriptor
|
|
//
|
|
//
|
|
// Return Value:
|
|
//
|
|
// None.
|
|
//
|
|
|
|
#define MiInsertClone(_CurrentProcess, CLONE) \
|
|
{ \
|
|
ASSERT ((CLONE)->NumberOfPtes != 0); \
|
|
MiInsertNode(((PMMADDRESS_NODE)(CLONE)),(PMMADDRESS_NODE *)&(_CurrentProcess->CloneRoot)); \
|
|
}
|
|
|
|
|
|
|
|
|
|
// VOID
|
|
// MiRemoveClone (
|
|
// IN PMMCLONE_DESCRIPTOR Clone
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// This function removes a virtual address descriptor from the tree and
|
|
// reorders the splay tree as appropriate.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// Clone - Supplies a pointer to a virtual address descriptor.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// None.
|
|
//
|
|
|
|
#define MiRemoveClone(_CurrentProcess, CLONE) \
|
|
MiRemoveNode((PMMADDRESS_NODE)(CLONE),(PMMADDRESS_NODE *)&(_CurrentProcess->CloneRoot));
|
|
|
|
|
|
|
|
// PMMCLONE_DESCRIPTOR
|
|
// MiLocateCloneAddress (
|
|
// IN PVOID VirtualAddress
|
|
// )
|
|
//
|
|
// /*++
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// The function locates the virtual address descriptor which describes
|
|
// a given address.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// VirtualAddress - Supplies the virtual address to locate a descriptor
|
|
// for.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// Returns a pointer to the virtual address descriptor which contains
|
|
// the supplied virtual address or NULL if none was located.
|
|
//
|
|
|
|
#define MiLocateCloneAddress(_CurrentProcess, VA) \
|
|
(_CurrentProcess->CloneRoot ? \
|
|
((PMMCLONE_DESCRIPTOR)MiLocateAddressInTree(((ULONG_PTR)VA), \
|
|
(PMMADDRESS_NODE *)&(_CurrentProcess->CloneRoot))) : \
|
|
NULL)
|
|
|
|
|
|
#define MI_VA_TO_PAGE(va) ((ULONG_PTR)(va) >> PAGE_SHIFT)
|
|
|
|
#define MI_VA_TO_VPN(va) ((ULONG_PTR)(va) >> PAGE_SHIFT)
|
|
|
|
#define MI_VPN_TO_VA(vpn) (PVOID)((vpn) << PAGE_SHIFT)
|
|
|
|
#define MI_VPN_TO_VA_ENDING(vpn) (PVOID)(((vpn) << PAGE_SHIFT) | (PAGE_SIZE - 1))
|
|
|
|
#define MiGetByteOffset(va) ((ULONG_PTR)(va) & (PAGE_SIZE - 1))
|
|
|
|
#define MI_PFN_ELEMENT(index) (&MmPfnDatabase[index])
|
|
|
|
//
|
|
// Make a write-copy PTE, only writable.
|
|
//
|
|
|
|
#define MI_MAKE_PROTECT_NOT_WRITE_COPY(PROTECT) \
|
|
(MmMakeProtectNotWriteCopy[PROTECT])
|
|
|
|
//
|
|
// Define macros to lock and unlock the PFN database.
|
|
//
|
|
|
|
#define MiLockPfnDatabase(OldIrql) \
|
|
OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock)
|
|
|
|
#define MiUnlockPfnDatabase(OldIrql) \
|
|
KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql)
|
|
|
|
#define MiLockPfnDatabaseAtDpcLevel() \
|
|
KeAcquireQueuedSpinLockAtDpcLevel(&KeGetCurrentPrcb()->LockQueue[LockQueuePfnLock])
|
|
|
|
#define MiUnlockPfnDatabaseFromDpcLevel() \
|
|
KeReleaseQueuedSpinLockFromDpcLevel(&KeGetCurrentPrcb()->LockQueue[LockQueuePfnLock])
|
|
|
|
#define MiTryToLockPfnDatabase(OldIrql) \
|
|
KeTryToAcquireQueuedSpinLock(LockQueuePfnLock, &OldIrql)
|
|
|
|
#define MiReleasePfnLock() \
|
|
KeReleaseQueuedSpinLockFromDpcLevel(&KeGetCurrentPrcb()->LockQueue[LockQueuePfnLock])
|
|
|
|
#define MiLockSystemSpace(OldIrql) \
|
|
OldIrql = KeAcquireQueuedSpinLock(LockQueueSystemSpaceLock)
|
|
|
|
#define MiUnlockSystemSpace(OldIrql) \
|
|
KeReleaseQueuedSpinLock(LockQueueSystemSpaceLock, OldIrql)
|
|
|
|
#define MiLockSystemSpaceAtDpcLevel() \
|
|
KeAcquireQueuedSpinLockAtDpcLevel(&KeGetCurrentPrcb()->LockQueue[LockQueueSystemSpaceLock])
|
|
|
|
#define MiUnlockSystemSpaceFromDpcLevel() \
|
|
KeReleaseQueuedSpinLockFromDpcLevel(&KeGetCurrentPrcb()->LockQueue[LockQueueSystemSpaceLock])
|
|
|
|
// #define _MI_INSTRUMENT_PFN 1
|
|
|
|
#if defined (_MI_INSTRUMENT_PFN)
|
|
|
|
#define MI_MAX_PFN_CALLERS 500
|
|
|
|
typedef struct _MMPFNTIMINGS {
|
|
LARGE_INTEGER HoldTime; // Low bit is set if another processor waited
|
|
ULONG_PTR AcquiredAddress;
|
|
ULONG_PTR ReleasedAddress;
|
|
} MMPFNTIMINGS, *PMMPFNTIMINGS;
|
|
|
|
extern ULONG MiPfnTimings;
|
|
extern ULONG_PTR MiPfnAcquiredAddress;
|
|
extern MMPFNTIMINGS MiPfnSorted[];
|
|
extern LARGE_INTEGER MiPfnAcquired;
|
|
extern LARGE_INTEGER MiPfnThreshold;
|
|
|
|
ULONG_PTR
|
|
MiGetExecutionAddress(
|
|
VOID
|
|
);
|
|
|
|
#if defined(_X86_)
|
|
#define MI_GET_EXECUTION_ADDRESS(varname) varname = MiGetExecutionAddress();
|
|
#else
|
|
#define MI_GET_EXECUTION_ADDRESS(varname) varname = 0;
|
|
#endif
|
|
|
|
#define LOCK_PFN_TIMESTAMP() \
|
|
{ \
|
|
MiPfnAcquired = KeQueryPerformanceCounter (NULL);\
|
|
MI_GET_EXECUTION_ADDRESS(MiPfnAcquiredAddress); \
|
|
}
|
|
|
|
#define UNLOCK_PFN_TIMESTAMP() \
|
|
{ \
|
|
ULONG i; \
|
|
ULONG_PTR ExecutionAddress; \
|
|
LARGE_INTEGER PfnReleased; \
|
|
LARGE_INTEGER PfnHoldTime; \
|
|
PfnReleased = KeQueryPerformanceCounter (NULL); \
|
|
MI_GET_EXECUTION_ADDRESS(ExecutionAddress); \
|
|
PfnHoldTime.QuadPart = (PfnReleased.QuadPart - MiPfnAcquired.QuadPart) & ~0x1; \
|
|
i = MI_MAX_PFN_CALLERS - 1; \
|
|
do { \
|
|
if (PfnHoldTime.QuadPart < MiPfnSorted[i].HoldTime.QuadPart) { \
|
|
break; \
|
|
} \
|
|
i -= 1; \
|
|
} while (i != (ULONG)-1); \
|
|
if (i != MI_MAX_PFN_CALLERS - 1) { \
|
|
i += 1; \
|
|
if (i != MI_MAX_PFN_CALLERS - 1) { \
|
|
RtlMoveMemory (&MiPfnSorted[i+1], &MiPfnSorted[i], (MI_MAX_PFN_CALLERS-(i+1)) * sizeof(MMPFNTIMINGS)); \
|
|
} \
|
|
MiPfnSorted[i].HoldTime = PfnHoldTime; \
|
|
if (KeTestForWaitersQueuedSpinLock(LockQueuePfnLock) == TRUE) {\
|
|
MiPfnSorted[i].HoldTime.LowPart |= 0x1; \
|
|
} \
|
|
MiPfnSorted[i].AcquiredAddress = MiPfnAcquiredAddress; \
|
|
MiPfnSorted[i].ReleasedAddress = ExecutionAddress; \
|
|
} \
|
|
if ((MiPfnTimings & 0x2) && (PfnHoldTime.QuadPart >= MiPfnThreshold.QuadPart)) { \
|
|
DbgBreakPoint (); \
|
|
} \
|
|
if (MiPfnTimings & 0x1) { \
|
|
MiPfnTimings &= ~0x1; \
|
|
RtlZeroMemory (&MiPfnSorted[0], MI_MAX_PFN_CALLERS * sizeof(MMPFNTIMINGS)); \
|
|
} \
|
|
}
|
|
#else
|
|
#define LOCK_PFN_TIMESTAMP()
|
|
#define UNLOCK_PFN_TIMESTAMP()
|
|
#endif
|
|
|
|
#define LOCK_PFN(OLDIRQL) ASSERT (KeGetCurrentIrql() <= APC_LEVEL); \
|
|
MiLockPfnDatabase(OLDIRQL); \
|
|
LOCK_PFN_TIMESTAMP();
|
|
|
|
#define LOCK_PFN_WITH_TRY(OLDIRQL) \
|
|
ASSERT (KeGetCurrentIrql() <= APC_LEVEL); \
|
|
do { \
|
|
} while (MiTryToLockPfnDatabase(OLDIRQL) == FALSE); \
|
|
LOCK_PFN_TIMESTAMP();
|
|
|
|
#define UNLOCK_PFN(OLDIRQL) \
|
|
UNLOCK_PFN_TIMESTAMP(); \
|
|
MiUnlockPfnDatabase(OLDIRQL); \
|
|
ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
|
|
|
|
#define LOCK_PFN2(OLDIRQL) ASSERT (KeGetCurrentIrql() <= DISPATCH_LEVEL); \
|
|
MiLockPfnDatabase(OLDIRQL); \
|
|
LOCK_PFN_TIMESTAMP();
|
|
|
|
#define UNLOCK_PFN2(OLDIRQL) \
|
|
UNLOCK_PFN_TIMESTAMP(); \
|
|
MiUnlockPfnDatabase(OLDIRQL); \
|
|
ASSERT(KeGetCurrentIrql() <= DISPATCH_LEVEL);
|
|
|
|
#define LOCK_PFN_AT_DPC() ASSERT (KeGetCurrentIrql() == DISPATCH_LEVEL); \
|
|
MiLockPfnDatabaseAtDpcLevel(); \
|
|
LOCK_PFN_TIMESTAMP();
|
|
|
|
#define UNLOCK_PFN_FROM_DPC() \
|
|
UNLOCK_PFN_TIMESTAMP(); \
|
|
MiUnlockPfnDatabaseFromDpcLevel(); \
|
|
ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
|
|
|
|
#define UNLOCK_PFN_AND_THEN_WAIT(OLDIRQL) \
|
|
{ \
|
|
KIRQL XXX; \
|
|
ASSERT (KeGetCurrentIrql() == DISPATCH_LEVEL); \
|
|
ASSERT (OLDIRQL <= APC_LEVEL); \
|
|
KiLockDispatcherDatabase (&XXX); \
|
|
UNLOCK_PFN_TIMESTAMP(); \
|
|
MiReleasePfnLock(); \
|
|
(KeGetCurrentThread())->WaitIrql = OLDIRQL; \
|
|
(KeGetCurrentThread())->WaitNext = TRUE; \
|
|
}
|
|
|
|
extern KMUTANT MmSystemLoadLock;
|
|
|
|
#if DBG
|
|
#define SYSLOAD_LOCK_OWNED_BY_ME() ((PETHREAD)MmSystemLoadLock.OwnerThread == PsGetCurrentThread())
|
|
#else
|
|
#define SYSLOAD_LOCK_OWNED_BY_ME()
|
|
#endif
|
|
|
|
#if DBG
|
|
|
|
#if defined (_MI_COMPRESSION)
|
|
|
|
extern KIRQL MiCompressionIrql;
|
|
|
|
#define MM_PFN_LOCK_ASSERT() \
|
|
if (MmDebug & 0x80000) { \
|
|
KIRQL _OldIrql; \
|
|
_OldIrql = KeGetCurrentIrql(); \
|
|
ASSERT ((_OldIrql == DISPATCH_LEVEL) || \
|
|
((MiCompressionIrql != 0) && (_OldIrql == MiCompressionIrql))); \
|
|
}
|
|
|
|
#else
|
|
|
|
#define MM_PFN_LOCK_ASSERT() \
|
|
if (MmDebug & 0x80000) { \
|
|
KIRQL _OldIrql; \
|
|
_OldIrql = KeGetCurrentIrql(); \
|
|
ASSERT (_OldIrql == DISPATCH_LEVEL); \
|
|
}
|
|
|
|
#endif
|
|
|
|
extern PETHREAD MiExpansionLockOwner;
|
|
|
|
#define MM_SET_EXPANSION_OWNER() ASSERT (MiExpansionLockOwner == NULL); \
|
|
MiExpansionLockOwner = PsGetCurrentThread();
|
|
|
|
#define MM_CLEAR_EXPANSION_OWNER() ASSERT (MiExpansionLockOwner == PsGetCurrentThread()); \
|
|
MiExpansionLockOwner = NULL;
|
|
|
|
#else
|
|
#define MM_PFN_LOCK_ASSERT()
|
|
#define MM_SET_EXPANSION_OWNER()
|
|
#define MM_CLEAR_EXPANSION_OWNER()
|
|
#endif //DBG
|
|
|
|
|
|
#define LOCK_EXPANSION(OLDIRQL) ASSERT (KeGetCurrentIrql() <= APC_LEVEL); \
|
|
ExAcquireSpinLock (&MmExpansionLock, &OLDIRQL);\
|
|
MM_SET_EXPANSION_OWNER ();
|
|
|
|
#define UNLOCK_EXPANSION(OLDIRQL) MM_CLEAR_EXPANSION_OWNER (); \
|
|
ExReleaseSpinLock (&MmExpansionLock, OLDIRQL); \
|
|
ASSERT (KeGetCurrentIrql() <= APC_LEVEL);
|
|
|
|
#define UNLOCK_EXPANSION_AND_THEN_WAIT(OLDIRQL) \
|
|
{ \
|
|
KIRQL XXX; \
|
|
ASSERT (KeGetCurrentIrql() == 2); \
|
|
ASSERT (OLDIRQL <= APC_LEVEL); \
|
|
KiLockDispatcherDatabase (&XXX); \
|
|
MM_CLEAR_EXPANSION_OWNER (); \
|
|
KiReleaseSpinLock (&MmExpansionLock); \
|
|
(KeGetCurrentThread())->WaitIrql = OLDIRQL; \
|
|
(KeGetCurrentThread())->WaitNext = TRUE; \
|
|
}
|
|
|
|
extern PETHREAD MmSystemLockOwner;
|
|
|
|
#define LOCK_SYSTEM_WS(OLDIRQL,_Thread) \
|
|
ASSERT (KeGetCurrentIrql() <= APC_LEVEL); \
|
|
KeRaiseIrql(APC_LEVEL,&OLDIRQL); \
|
|
ExAcquireResourceExclusiveLite(&MmSystemWsLock,TRUE); \
|
|
ASSERT (MmSystemLockOwner == NULL); \
|
|
MmSystemLockOwner = _Thread;
|
|
|
|
#define LOCK_SYSTEM_WS_UNSAFE(_Thread) \
|
|
ASSERT (KeGetCurrentIrql() == APC_LEVEL); \
|
|
ExAcquireResourceExclusiveLite(&MmSystemWsLock,TRUE); \
|
|
ASSERT (MmSystemLockOwner == NULL); \
|
|
MmSystemLockOwner = _Thread;
|
|
|
|
#define UNLOCK_SYSTEM_WS(OLDIRQL) \
|
|
ASSERT (MmSystemLockOwner == PsGetCurrentThread()); \
|
|
MmSystemLockOwner = NULL; \
|
|
ExReleaseResourceLite (&MmSystemWsLock); \
|
|
KeLowerIrql (OLDIRQL); \
|
|
ASSERT (KeGetCurrentIrql() <= APC_LEVEL);
|
|
|
|
#define UNLOCK_SYSTEM_WS_NO_IRQL() \
|
|
ASSERT (MmSystemLockOwner == PsGetCurrentThread()); \
|
|
MmSystemLockOwner = NULL; \
|
|
ExReleaseResourceLite (&MmSystemWsLock);
|
|
|
|
#define MM_SYSTEM_WS_LOCK_ASSERT() \
|
|
ASSERT (PsGetCurrentThread() == MmSystemLockOwner);
|
|
|
|
#define LOCK_HYPERSPACE(_Process, OLDIRQL) \
|
|
ASSERT (_Process == PsGetCurrentProcess ()); \
|
|
ExAcquireSpinLock (&_Process->HyperSpaceLock, OLDIRQL);
|
|
|
|
#define UNLOCK_HYPERSPACE(_Process, VA, OLDIRQL) \
|
|
ASSERT (_Process == PsGetCurrentProcess ()); \
|
|
MiGetPteAddress(VA)->u.Long = 0; \
|
|
ExReleaseSpinLock (&_Process->HyperSpaceLock, OLDIRQL);
|
|
|
|
#define LOCK_HYPERSPACE_AT_DPC(_Process) \
|
|
ASSERT (KeGetCurrentIrql() == DISPATCH_LEVEL); \
|
|
ASSERT (_Process == PsGetCurrentProcess ()); \
|
|
ExAcquireSpinLockAtDpcLevel (&_Process->HyperSpaceLock);
|
|
|
|
#define UNLOCK_HYPERSPACE_FROM_DPC(_Process, VA) \
|
|
ASSERT (KeGetCurrentIrql() == DISPATCH_LEVEL); \
|
|
ASSERT (_Process == PsGetCurrentProcess ()); \
|
|
MiGetPteAddress(VA)->u.Long = 0; \
|
|
ExReleaseSpinLockFromDpcLevel (&_Process->HyperSpaceLock);
|
|
|
|
#define MiUnmapPageInHyperSpace(_Process, VA, OLDIRQL) UNLOCK_HYPERSPACE(_Process, VA, OLDIRQL)
|
|
|
|
#define MiUnmapPageInHyperSpaceFromDpc(_Process, VA) UNLOCK_HYPERSPACE_FROM_DPC(_Process, VA)
|
|
|
|
#if defined (_AXP64_)
|
|
#define MI_WS_OWNER(PROCESS) ((PROCESS)->WorkingSetLock.Owner == KeGetCurrentThread())
|
|
#define MI_NOT_WS_OWNER(PROCESS) (!MI_WS_OWNER(PROCESS))
|
|
#else
|
|
#define MI_WS_OWNER(PROCESS) 1
|
|
#define MI_NOT_WS_OWNER(PROCESS) 1
|
|
#endif
|
|
|
|
#define MI_MUTEX_ACQUIRED_UNSAFE 0x88
|
|
|
|
#define MI_IS_WS_UNSAFE(PROCESS) ((PROCESS)->WorkingSetAcquiredUnsafe == MI_MUTEX_ACQUIRED_UNSAFE)
|
|
|
|
#define LOCK_WS(PROCESS) \
|
|
ASSERT (MI_NOT_WS_OWNER(PROCESS)); \
|
|
ExAcquireFastMutex( &((PROCESS)->WorkingSetLock)); \
|
|
ASSERT (!MI_IS_WS_UNSAFE(PROCESS));
|
|
|
|
#define LOCK_WS_UNSAFE(PROCESS) \
|
|
ASSERT (MI_NOT_WS_OWNER(PROCESS)); \
|
|
ASSERT (KeGetCurrentIrql() == APC_LEVEL); \
|
|
ExAcquireFastMutexUnsafe( &((PROCESS)->WorkingSetLock));\
|
|
(PROCESS)->WorkingSetAcquiredUnsafe = MI_MUTEX_ACQUIRED_UNSAFE;
|
|
|
|
#define MI_MUST_BE_UNSAFE(PROCESS) \
|
|
ASSERT (KeGetCurrentIrql() == APC_LEVEL); \
|
|
ASSERT (MI_WS_OWNER(PROCESS)); \
|
|
ASSERT (MI_IS_WS_UNSAFE(PROCESS));
|
|
|
|
#define MI_MUST_BE_SAFE(PROCESS) \
|
|
ASSERT (MI_WS_OWNER(PROCESS)); \
|
|
ASSERT (!MI_IS_WS_UNSAFE(PROCESS));
|
|
#if 0
|
|
|
|
#define MI_MUST_BE_UNSAFE(PROCESS) \
|
|
if (KeGetCurrentIrql() != APC_LEVEL) { \
|
|
KeBugCheckEx(MEMORY_MANAGEMENT, 0x32, (ULONG_PTR)PROCESS, KeGetCurrentIrql(), 0); \
|
|
} \
|
|
if (!MI_WS_OWNER(PROCESS)) { \
|
|
KeBugCheckEx(MEMORY_MANAGEMENT, 0x33, (ULONG_PTR)PROCESS, 0, 0); \
|
|
} \
|
|
if (!MI_IS_WS_UNSAFE(PROCESS)) { \
|
|
KeBugCheckEx(MEMORY_MANAGEMENT, 0x34, (ULONG_PTR)PROCESS, 0, 0); \
|
|
}
|
|
|
|
#define MI_MUST_BE_SAFE(PROCESS) \
|
|
if (!MI_WS_OWNER(PROCESS)) { \
|
|
KeBugCheckEx(MEMORY_MANAGEMENT, 0x42, (ULONG_PTR)PROCESS, 0, 0); \
|
|
} \
|
|
if (MI_IS_WS_UNSAFE(PROCESS)) { \
|
|
KeBugCheckEx(MEMORY_MANAGEMENT, 0x43, (ULONG_PTR)PROCESS, 0, 0); \
|
|
}
|
|
#endif
|
|
|
|
|
|
#define UNLOCK_WS(PROCESS) \
|
|
MI_MUST_BE_SAFE(PROCESS); \
|
|
ExReleaseFastMutex(&((PROCESS)->WorkingSetLock));
|
|
|
|
#define UNLOCK_WS_UNSAFE(PROCESS) \
|
|
MI_MUST_BE_UNSAFE(PROCESS); \
|
|
(PROCESS)->WorkingSetAcquiredUnsafe = 0; \
|
|
ExReleaseFastMutexUnsafe(&((PROCESS)->WorkingSetLock)); \
|
|
ASSERT (KeGetCurrentIrql() == APC_LEVEL);
|
|
|
|
#define LOCK_ADDRESS_SPACE(PROCESS) \
|
|
ExAcquireFastMutex( &((PROCESS)->AddressCreationLock))
|
|
|
|
#define LOCK_WS_AND_ADDRESS_SPACE(PROCESS) \
|
|
LOCK_ADDRESS_SPACE(PROCESS); \
|
|
LOCK_WS_UNSAFE(PROCESS);
|
|
|
|
#define UNLOCK_WS_AND_ADDRESS_SPACE(PROCESS) \
|
|
UNLOCK_WS_UNSAFE(PROCESS); \
|
|
UNLOCK_ADDRESS_SPACE(PROCESS);
|
|
|
|
#define UNLOCK_ADDRESS_SPACE(PROCESS) \
|
|
ExReleaseFastMutex( &((PROCESS)->AddressCreationLock))
|
|
|
|
//
|
|
// The working set lock may have been acquired safely or unsafely.
|
|
// Release and reacquire it regardless.
|
|
//
|
|
|
|
#define UNLOCK_WS_REGARDLESS(PROCESS, WSHELDSAFE) \
|
|
ASSERT (MI_WS_OWNER (PROCESS)); \
|
|
if (MI_IS_WS_UNSAFE (PROCESS)) { \
|
|
UNLOCK_WS_UNSAFE (PROCESS); \
|
|
WSHELDSAFE = FALSE; \
|
|
} \
|
|
else { \
|
|
UNLOCK_WS (PROCESS); \
|
|
WSHELDSAFE = TRUE; \
|
|
}
|
|
|
|
#define LOCK_WS_REGARDLESS(PROCESS, WSHELDSAFE) \
|
|
if (WSHELDSAFE == TRUE) { \
|
|
LOCK_WS (PROCESS); \
|
|
} \
|
|
else { \
|
|
LOCK_WS_UNSAFE (PROCESS); \
|
|
}
|
|
|
|
#define ZERO_LARGE(LargeInteger) \
|
|
(LargeInteger).LowPart = 0; \
|
|
(LargeInteger).HighPart = 0;
|
|
|
|
#define NO_BITS_FOUND ((ULONG)-1)
|
|
|
|
//++
|
|
//
|
|
// ULONG
|
|
// MI_CHECK_BIT (
|
|
// IN PULONG ARRAY
|
|
// IN ULONG BIT
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// The MI_CHECK_BIT macro checks to see if the specified bit is
|
|
// set within the specified array.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// ARRAY - First element of the array to check.
|
|
//
|
|
// BIT - bit number (first bit is 0) to check.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// Returns the value of the bit (0 or 1).
|
|
//
|
|
//--
|
|
|
|
#define MI_CHECK_BIT(ARRAY,BIT) \
|
|
(((ULONG)ARRAY[(BIT) / (sizeof(ULONG)*8)] >> ((BIT) & 0x1F)) & 1)
|
|
|
|
|
|
//++
|
|
//
|
|
// VOID
|
|
// MI_SET_BIT (
|
|
// IN PULONG ARRAY
|
|
// IN ULONG BIT
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// The MI_SET_BIT macro sets the specified bit within the
|
|
// specified array.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// ARRAY - First element of the array to set.
|
|
//
|
|
// BIT - bit number.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// None.
|
|
//
|
|
//--
|
|
|
|
#define MI_SET_BIT(ARRAY,BIT) \
|
|
ARRAY[(BIT) / (sizeof(ULONG)*8)] |= (1 << ((BIT) & 0x1F))
|
|
|
|
|
|
//++
|
|
//
|
|
// VOID
|
|
// MI_CLEAR_BIT (
|
|
// IN PULONG ARRAY
|
|
// IN ULONG BIT
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// The MI_CLEAR_BIT macro sets the specified bit within the
|
|
// specified array.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// ARRAY - First element of the array to clear.
|
|
//
|
|
// BIT - bit number.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// None.
|
|
//
|
|
//--
|
|
|
|
#define MI_CLEAR_BIT(ARRAY,BIT) \
|
|
ARRAY[(BIT) / (sizeof(ULONG)*8)] &= ~(1 << ((BIT) & 0x1F))
|
|
|
|
//
|
|
// These control the mirroring capabilities.
|
|
//
|
|
|
|
extern ULONG MmMirroring;
|
|
|
|
#define MM_MIRRORING_ENABLED 0x1 // Enable mirroring capabilities.
|
|
#define MM_MIRRORING_VERIFYING 0x2 // The HAL wants to verify the copy.
|
|
|
|
extern PRTL_BITMAP MiMirrorBitMap;
|
|
extern PRTL_BITMAP MiMirrorBitMap2;
|
|
extern LOGICAL MiMirroringActive;
|
|
|
|
#if defined (_WIN64)
|
|
#define MI_MAGIC_AWE_PTEFRAME 0xffffedcb
|
|
#else
|
|
#define MI_MAGIC_AWE_PTEFRAME 0xffedcb
|
|
#endif
|
|
|
|
#define MI_PFN_IS_AWE(Pfn1) \
|
|
((Pfn1->u2.ShareCount <= 3) && \
|
|
(Pfn1->u3.e1.PageLocation == ActiveAndValid) && \
|
|
(Pfn1->u4.VerifierAllocation == 0) && \
|
|
(Pfn1->u3.e1.LargeSessionAllocation == 0) && \
|
|
(Pfn1->u3.e1.StartOfAllocation == 1) && \
|
|
(Pfn1->u3.e1.EndOfAllocation == 1) && \
|
|
(Pfn1->u4.PteFrame == MI_MAGIC_AWE_PTEFRAME))
|
|
|
|
|
|
//
|
|
// PFN database element.
|
|
//
|
|
|
|
//
|
|
// Define pseudo fields for start and end of allocation.
|
|
//
|
|
|
|
#define StartOfAllocation ReadInProgress
|
|
|
|
#define EndOfAllocation WriteInProgress
|
|
|
|
#define LargeSessionAllocation PrototypePte
|
|
|
|
typedef struct _MMPFNENTRY {
|
|
ULONG Modified : 1;
|
|
ULONG ReadInProgress : 1;
|
|
ULONG WriteInProgress : 1;
|
|
ULONG PrototypePte: 1;
|
|
ULONG PageColor : 3;
|
|
ULONG ParityError : 1;
|
|
ULONG PageLocation : 3;
|
|
ULONG RemovalRequested : 1;
|
|
ULONG CacheAttribute : 2;
|
|
ULONG Rom : 1;
|
|
ULONG LockCharged : 1; // maintained for DBG only
|
|
ULONG DontUse : 16; // overlays USHORT for reference count field.
|
|
} MMPFNENTRY;
|
|
|
|
//
|
|
// The cache type definitions are carefully chosen to line up with the
|
|
// MEMORY_CACHING_TYPE definitions to ease conversions. Any changes here must
|
|
// be reflected throughout the code.
|
|
//
|
|
|
|
typedef enum _MI_PFN_CACHE_ATTRIBUTE {
|
|
MiNonCached,
|
|
MiCached,
|
|
MiWriteCombined,
|
|
MiNotMapped
|
|
} MI_PFN_CACHE_ATTRIBUTE, *PMI_PFN_CACHE_ATTRIBUTE;
|
|
|
|
//
|
|
// This conversion array is unfortunately needed because not all
|
|
// hardware platforms support all possible cache values. Note that
|
|
// the first range is for system RAM, the second range is for I/O space.
|
|
//
|
|
|
|
extern MI_PFN_CACHE_ATTRIBUTE MiPlatformCacheAttributes[2 * MmMaximumCacheType];
|
|
|
|
//++
|
|
//
|
|
// MI_PFN_CACHE_ATTRIBUTE
|
|
// MI_TRANSLATE_CACHETYPE (
|
|
// IN MEMORY_CACHING_TYPE InputCacheType,
|
|
// IN ULONG IoSpace
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// Returns the hardware supported cache type for the requested cachetype.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// InputCacheType - Supplies the desired cache type.
|
|
//
|
|
// IoSpace - Supplies nonzero (not necessarily 1 though) if this is
|
|
// I/O space, zero if it is main memory.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// The actual cache type.
|
|
//
|
|
//--
|
|
FORCEINLINE
|
|
MI_PFN_CACHE_ATTRIBUTE
|
|
MI_TRANSLATE_CACHETYPE(
|
|
IN MEMORY_CACHING_TYPE InputCacheType,
|
|
IN ULONG IoSpace
|
|
)
|
|
{
|
|
ASSERT (InputCacheType <= MmWriteCombined);
|
|
|
|
if (IoSpace != 0) {
|
|
IoSpace = MmMaximumCacheType;
|
|
}
|
|
return MiPlatformCacheAttributes[IoSpace + InputCacheType];
|
|
}
|
|
|
|
//++
|
|
//
|
|
// VOID
|
|
// MI_SET_CACHETYPE_TRANSLATION (
|
|
// IN MEMORY_CACHING_TYPE InputCacheType,
|
|
// IN ULONG IoSpace,
|
|
// IN MI_PFN_CACHE_ATTRIBUTE NewAttribute
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// This function sets the hardware supported cachetype for the
|
|
// specified cachetype.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// InputCacheType - Supplies the desired cache type.
|
|
//
|
|
// IoSpace - Supplies nonzero (not necessarily 1 though) if this is
|
|
// I/O space, zero if it is main memory.
|
|
//
|
|
// NewAttribute - Supplies the desired attribute.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// None.
|
|
//
|
|
//--
|
|
FORCEINLINE
|
|
VOID
|
|
MI_SET_CACHETYPE_TRANSLATION(
|
|
IN MEMORY_CACHING_TYPE InputCacheType,
|
|
IN ULONG IoSpace,
|
|
IN MI_PFN_CACHE_ATTRIBUTE NewAttribute
|
|
)
|
|
{
|
|
ASSERT (InputCacheType <= MmWriteCombined);
|
|
|
|
if (IoSpace != 0) {
|
|
IoSpace = MmMaximumCacheType;
|
|
}
|
|
|
|
MiPlatformCacheAttributes[IoSpace + InputCacheType] = NewAttribute;
|
|
}
|
|
|
|
#if defined (_X86PAE_)
|
|
#pragma pack(1)
|
|
#endif
|
|
|
|
typedef struct _MMPFN {
|
|
union {
|
|
PFN_NUMBER Flink;
|
|
WSLE_NUMBER WsIndex;
|
|
PKEVENT Event;
|
|
NTSTATUS ReadStatus;
|
|
SINGLE_LIST_ENTRY NextStackPfn;
|
|
} u1;
|
|
PMMPTE PteAddress;
|
|
union {
|
|
PFN_NUMBER Blink;
|
|
ULONG_PTR ShareCount;
|
|
} u2;
|
|
union {
|
|
MMPFNENTRY e1;
|
|
struct {
|
|
USHORT ShortFlags;
|
|
USHORT ReferenceCount;
|
|
} e2;
|
|
} u3;
|
|
#if defined (_WIN64)
|
|
ULONG UsedPageTableEntries;
|
|
#endif
|
|
MMPTE OriginalPte;
|
|
union {
|
|
ULONG_PTR EntireFrame;
|
|
struct {
|
|
#if defined (_WIN64)
|
|
ULONG_PTR PteFrame: 58;
|
|
#else
|
|
ULONG_PTR PteFrame: 26;
|
|
#endif
|
|
ULONG_PTR InPageError : 1;
|
|
ULONG_PTR VerifierAllocation : 1;
|
|
ULONG_PTR AweAllocation : 1;
|
|
ULONG_PTR LockCharged : 1; // maintained for DBG only
|
|
ULONG_PTR KernelStack : 1; // only for valid (not trans) pages
|
|
ULONG_PTR Reserved : 1;
|
|
};
|
|
} u4;
|
|
|
|
} MMPFN, *PMMPFN;
|
|
|
|
#if defined (_X86PAE_)
|
|
#pragma pack()
|
|
#endif
|
|
|
|
// #define _MI_DEBUG_DIRTY 1 // Uncomment this for dirty bit logging
|
|
|
|
#if defined (_MI_DEBUG_DIRTY)
|
|
|
|
extern ULONG MiTrackDirtys;
|
|
|
|
#define MI_DIRTY_BACKTRACE_LENGTH 17
|
|
|
|
typedef struct _MI_DIRTY_TRACES {
|
|
|
|
PETHREAD Thread;
|
|
PEPROCESS Process;
|
|
PMMPFN Pfn;
|
|
PMMPTE PointerPte;
|
|
ULONG_PTR CallerId;
|
|
ULONG_PTR ShareCount;
|
|
USHORT ShortFlags;
|
|
USHORT ReferenceCount;
|
|
PVOID StackTrace [MI_DIRTY_BACKTRACE_LENGTH];
|
|
|
|
} MI_DIRTY_TRACES, *PMI_DIRTY_TRACES;
|
|
|
|
extern LONG MiDirtyIndex;
|
|
|
|
extern PMI_DIRTY_TRACES MiDirtyTraces;
|
|
|
|
VOID
|
|
FORCEINLINE
|
|
MiSnapDirty (
|
|
IN PMMPFN Pfn,
|
|
IN ULONG NewValue,
|
|
IN ULONG CallerId
|
|
)
|
|
{
|
|
PEPROCESS Process;
|
|
PMI_DIRTY_TRACES Information;
|
|
ULONG Index;
|
|
ULONG Hash;
|
|
|
|
if (MiDirtyTraces == NULL) {
|
|
return;
|
|
}
|
|
|
|
Index = InterlockedIncrement(&MiDirtyIndex);
|
|
Index &= (MiTrackDirtys - 1);
|
|
Information = &MiDirtyTraces[Index];
|
|
|
|
Process = PsGetCurrentProcess ();
|
|
|
|
Information->Thread = PsGetCurrentThread ();
|
|
Information->Process = Process;
|
|
Information->Pfn = Pfn;
|
|
Information->PointerPte = Pfn->PteAddress;
|
|
Information->CallerId = CallerId;
|
|
Information->ShareCount = Pfn->u2.ShareCount;
|
|
Information->ShortFlags = Pfn->u3.e2.ShortFlags;
|
|
Information->ReferenceCount = Pfn->u3.e2.ReferenceCount;
|
|
|
|
if (NewValue != 0) {
|
|
Information->Process = (PEPROCESS) ((ULONG_PTR)Process | 0x1);
|
|
}
|
|
|
|
RtlZeroMemory (&Information->StackTrace[0], MI_DIRTY_BACKTRACE_LENGTH * sizeof(PVOID));
|
|
|
|
RtlCaptureStackBackTrace (0, MI_DIRTY_BACKTRACE_LENGTH, Information->StackTrace, &Hash);
|
|
}
|
|
|
|
#define MI_SNAP_DIRTY(_Pfn, _NewValue, _Callerid) MiSnapDirty(_Pfn, _NewValue, _Callerid)
|
|
|
|
#else
|
|
#define MI_SNAP_DIRTY(_Pfn, _NewValue, _Callerid)
|
|
#endif
|
|
|
|
#if 0
|
|
#define MI_STAMP_MODIFIED(Pfn,id) (Pfn)->u4.Reserved = (id);
|
|
#else
|
|
#define MI_STAMP_MODIFIED(Pfn,id)
|
|
#endif
|
|
|
|
//++
|
|
//
|
|
// VOID
|
|
// MI_SET_MODIFIED (
|
|
// IN PMMPFN Pfn,
|
|
// IN ULONG NewValue,
|
|
// IN ULONG CallerId
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// Set (or clear) the modified bit in the PFN database element.
|
|
// The PFN lock must be held.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// Pfn - Supplies the PFN to operate on.
|
|
//
|
|
// NewValue - Supplies 1 to set the modified bit, 0 to clear it.
|
|
//
|
|
// CallerId - Supplies a caller ID useful for debugging purposes.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// None.
|
|
//
|
|
//--
|
|
//
|
|
#define MI_SET_MODIFIED(_Pfn, _NewValue, _CallerId) \
|
|
MI_SNAP_DIRTY (_Pfn, _NewValue, _CallerId); \
|
|
if ((_NewValue) != 0) { \
|
|
MI_STAMP_MODIFIED (_Pfn, _CallerId); \
|
|
} \
|
|
(_Pfn)->u3.e1.Modified = (_NewValue);
|
|
|
|
//
|
|
// ccNUMA is supported in multiprocessor PAE and WIN64 systems only.
|
|
//
|
|
|
|
#if (defined(_WIN64) || defined(_X86PAE_)) && !defined(NT_UP)
|
|
#define MI_MULTINODE
|
|
|
|
VOID
|
|
MiDetermineNode (
|
|
IN PFN_NUMBER Page,
|
|
IN PMMPFN Pfn
|
|
);
|
|
|
|
#else
|
|
|
|
#define MiDetermineNode(x,y) ((y)->u3.e1.PageColor = 0)
|
|
|
|
#endif
|
|
|
|
#if defined (_WIN64)
|
|
|
|
//
|
|
// Note there are some places where these portable macros are not currently
|
|
// used because we are not in the correct address space required.
|
|
//
|
|
|
|
#define MI_CAPTURE_USED_PAGETABLE_ENTRIES(PFN) \
|
|
ASSERT ((PFN)->UsedPageTableEntries <= PTE_PER_PAGE); \
|
|
(PFN)->OriginalPte.u.Soft.UsedPageTableEntries = (PFN)->UsedPageTableEntries;
|
|
|
|
#define MI_RETRIEVE_USED_PAGETABLE_ENTRIES_FROM_PTE(RBL, PTE) \
|
|
ASSERT ((PTE)->u.Soft.UsedPageTableEntries <= PTE_PER_PAGE); \
|
|
(RBL)->UsedPageTableEntries = (ULONG)(((PMMPTE)(PTE))->u.Soft.UsedPageTableEntries);
|
|
|
|
#define MI_ZERO_USED_PAGETABLE_ENTRIES_IN_INPAGE_SUPPORT(INPAGE_SUPPORT) \
|
|
(INPAGE_SUPPORT)->UsedPageTableEntries = 0;
|
|
|
|
#define MI_ZERO_USED_PAGETABLE_ENTRIES_IN_PFN(PFN) (PFN)->UsedPageTableEntries = 0;
|
|
|
|
#define MI_INSERT_USED_PAGETABLE_ENTRIES_IN_PFN(PFN, INPAGE_SUPPORT) \
|
|
ASSERT ((INPAGE_SUPPORT)->UsedPageTableEntries <= PTE_PER_PAGE); \
|
|
(PFN)->UsedPageTableEntries = (INPAGE_SUPPORT)->UsedPageTableEntries;
|
|
|
|
#define MI_ZERO_USED_PAGETABLE_ENTRIES(PFN) \
|
|
(PFN)->UsedPageTableEntries = 0;
|
|
|
|
#define MI_CHECK_USED_PTES_HANDLE(VA) \
|
|
ASSERT (MiGetPdeAddress(VA)->u.Hard.Valid == 1);
|
|
|
|
#define MI_GET_USED_PTES_HANDLE(VA) \
|
|
((PVOID)MI_PFN_ELEMENT((PFN_NUMBER)MiGetPdeAddress(VA)->u.Hard.PageFrameNumber))
|
|
|
|
#define MI_GET_USED_PTES_FROM_HANDLE(PFN) \
|
|
((ULONG)(((PMMPFN)(PFN))->UsedPageTableEntries))
|
|
|
|
#define MI_INCREMENT_USED_PTES_BY_HANDLE(PFN) \
|
|
(((PMMPFN)(PFN))->UsedPageTableEntries += 1); \
|
|
ASSERT (((PMMPFN)(PFN))->UsedPageTableEntries <= PTE_PER_PAGE)
|
|
|
|
#define MI_DECREMENT_USED_PTES_BY_HANDLE(PFN) \
|
|
(((PMMPFN)(PFN))->UsedPageTableEntries -= 1); \
|
|
ASSERT (((PMMPFN)(PFN))->UsedPageTableEntries < PTE_PER_PAGE)
|
|
|
|
#else
|
|
|
|
#define MI_CAPTURE_USED_PAGETABLE_ENTRIES(PFN)
|
|
#define MI_RETRIEVE_USED_PAGETABLE_ENTRIES_FROM_PTE(RBL, PTE)
|
|
#define MI_ZERO_USED_PAGETABLE_ENTRIES_IN_INPAGE_SUPPORT(INPAGE_SUPPORT)
|
|
#define MI_ZERO_USED_PAGETABLE_ENTRIES_IN_PFN(PFN)
|
|
|
|
#define MI_INSERT_USED_PAGETABLE_ENTRIES_IN_PFN(PFN, INPAGE_SUPPORT)
|
|
|
|
#define MI_CHECK_USED_PTES_HANDLE(VA)
|
|
|
|
#define MI_GET_USED_PTES_HANDLE(VA) ((PVOID)&MmWorkingSetList->UsedPageTableEntries[MiGetPdeIndex(VA)])
|
|
|
|
#define MI_GET_USED_PTES_FROM_HANDLE(PDSHORT) ((ULONG)(*(PUSHORT)(PDSHORT)))
|
|
|
|
#define MI_INCREMENT_USED_PTES_BY_HANDLE(PDSHORT) \
|
|
((*(PUSHORT)(PDSHORT)) += 1); \
|
|
ASSERT (((*(PUSHORT)(PDSHORT)) <= PTE_PER_PAGE))
|
|
|
|
#define MI_DECREMENT_USED_PTES_BY_HANDLE(PDSHORT) \
|
|
((*(PUSHORT)(PDSHORT)) -= 1); \
|
|
ASSERT (((*(PUSHORT)(PDSHORT)) < PTE_PER_PAGE))
|
|
|
|
#endif
|
|
|
|
extern PFN_NUMBER MmDynamicPfn;
|
|
|
|
extern FAST_MUTEX MmDynamicMemoryMutex;
|
|
|
|
extern PFN_NUMBER MmSystemLockPagesCount;
|
|
|
|
#if DBG
|
|
|
|
#define MI_LOCK_ID_COUNTER_MAX 64
|
|
ULONG MiLockIds[MI_LOCK_ID_COUNTER_MAX];
|
|
|
|
#define MI_MARK_PFN_AS_LOCK_CHARGED(Pfn, CallerId) \
|
|
ASSERT (Pfn->u3.e1.LockCharged == 0); \
|
|
ASSERT (CallerId < MI_LOCK_ID_COUNTER_MAX); \
|
|
MiLockIds[CallerId] += 1; \
|
|
Pfn->u3.e1.LockCharged = 1;
|
|
|
|
#define MI_UNMARK_PFN_AS_LOCK_CHARGED(Pfn, CallerId) \
|
|
ASSERT (Pfn->u3.e1.LockCharged == 1); \
|
|
ASSERT (CallerId < MI_LOCK_ID_COUNTER_MAX); \
|
|
MiLockIds[CallerId] += 1; \
|
|
Pfn->u3.e1.LockCharged = 0;
|
|
|
|
#else
|
|
#define MI_MARK_PFN_AS_LOCK_CHARGED(Pfn, CallerId)
|
|
#define MI_UNMARK_PFN_AS_LOCK_CHARGED(Pfn, CallerId)
|
|
#endif
|
|
|
|
//++
|
|
//
|
|
// VOID
|
|
// MI_ADD_LOCKED_PAGE_CHARGE (
|
|
// IN PMMPFN Pfn
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// Charge the systemwide count of locked pages if this is the initial
|
|
// lock for this page (multiple concurrent locks are only charged once).
|
|
//
|
|
// Arguments:
|
|
//
|
|
// Pfn - the PFN index to operate on.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// None.
|
|
//
|
|
//--
|
|
//
|
|
#define MI_ADD_LOCKED_PAGE_CHARGE(Pfn, CallerId) \
|
|
ASSERT (Pfn->u3.e2.ReferenceCount != 0); \
|
|
if (Pfn->u3.e2.ReferenceCount == 1) { \
|
|
if (Pfn->u2.ShareCount != 0) { \
|
|
ASSERT (Pfn->u3.e1.PageLocation == ActiveAndValid); \
|
|
MI_MARK_PFN_AS_LOCK_CHARGED(Pfn, CallerId); \
|
|
MmSystemLockPagesCount += 1; \
|
|
} \
|
|
else { \
|
|
ASSERT (Pfn->u3.e1.LockCharged == 1); \
|
|
} \
|
|
}
|
|
|
|
#define MI_ADD_LOCKED_PAGE_CHARGE_FOR_MODIFIED_PAGE(Pfn, CallerId) \
|
|
ASSERT (Pfn->u3.e1.PageLocation != ActiveAndValid); \
|
|
ASSERT (Pfn->u2.ShareCount == 0); \
|
|
if (Pfn->u3.e2.ReferenceCount == 0) { \
|
|
MI_MARK_PFN_AS_LOCK_CHARGED(Pfn, CallerId); \
|
|
MmSystemLockPagesCount += 1; \
|
|
}
|
|
|
|
#define MI_ADD_LOCKED_PAGE_CHARGE_FOR_TRANSITION_PAGE(Pfn, CallerId) \
|
|
ASSERT (Pfn->u3.e1.PageLocation == ActiveAndValid); \
|
|
ASSERT (Pfn->u2.ShareCount == 0); \
|
|
ASSERT (Pfn->u3.e2.ReferenceCount != 0); \
|
|
if (Pfn->u3.e2.ReferenceCount == 1) { \
|
|
MI_MARK_PFN_AS_LOCK_CHARGED(Pfn, CallerId); \
|
|
MmSystemLockPagesCount += 1; \
|
|
}
|
|
|
|
//++
|
|
//
|
|
// VOID
|
|
// MI_REMOVE_LOCKED_PAGE_CHARGE (
|
|
// IN PMMPFN Pfn
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// Remove the charge from the systemwide count of locked pages if this
|
|
// is the last lock for this page (multiple concurrent locks are only
|
|
// charged once).
|
|
//
|
|
// The PFN reference checks are carefully ordered so the most common case
|
|
// is handled first, the next most common case next, etc. The case of
|
|
// a reference count of 2 occurs more than 1000x (yes, 3 orders of
|
|
// magnitude) more than a reference count of 1. And reference counts of >2
|
|
// occur 3 orders of magnitude more frequently than reference counts of
|
|
// exactly 1.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// Pfn - the PFN index to operate on.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// None.
|
|
//
|
|
//--
|
|
//
|
|
#define MI_REMOVE_LOCKED_PAGE_CHARGE(Pfn, CallerId) \
|
|
ASSERT (Pfn->u3.e2.ReferenceCount != 0); \
|
|
if (Pfn->u3.e2.ReferenceCount == 2) { \
|
|
if (Pfn->u2.ShareCount >= 1) { \
|
|
ASSERT (Pfn->u3.e1.PageLocation == ActiveAndValid); \
|
|
MI_UNMARK_PFN_AS_LOCK_CHARGED(Pfn, CallerId); \
|
|
MmSystemLockPagesCount -= 1; \
|
|
} \
|
|
else { \
|
|
/* \
|
|
* There are multiple referencers to this page and the \
|
|
* page is no longer valid in any process address space. \
|
|
* The systemwide lock count can only be decremented \
|
|
* by the last dereference. \
|
|
*/ \
|
|
NOTHING; \
|
|
} \
|
|
} \
|
|
else if (Pfn->u3.e2.ReferenceCount != 1) { \
|
|
/* \
|
|
* There are still multiple referencers to this page (it may \
|
|
* or may not be resident in any process address space). \
|
|
* Since the systemwide lock count can only be decremented \
|
|
* by the last dereference (and this is not it), no action \
|
|
* is taken here. \
|
|
*/ \
|
|
ASSERT (Pfn->u3.e2.ReferenceCount > 2); \
|
|
NOTHING; \
|
|
} \
|
|
else { \
|
|
/* \
|
|
* This page has already been deleted from all process address \
|
|
* spaces. It is sitting in limbo (not on any list) awaiting \
|
|
* this last dereference. \
|
|
*/ \
|
|
ASSERT (Pfn->u3.e2.ReferenceCount == 1); \
|
|
ASSERT (Pfn->u3.e1.PageLocation != ActiveAndValid); \
|
|
ASSERT (Pfn->u2.ShareCount == 0); \
|
|
MI_UNMARK_PFN_AS_LOCK_CHARGED(Pfn, CallerId); \
|
|
MmSystemLockPagesCount -= 1; \
|
|
}
|
|
|
|
//++
|
|
//
|
|
// VOID
|
|
// MI_REMOVE_LOCKED_PAGE_CHARGE_AND_DECREF (
|
|
// IN PMMPFN Pfn
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// Remove the charge from the systemwide count of locked pages if this
|
|
// is the last lock for this page (multiple concurrent locks are only
|
|
// charged once).
|
|
//
|
|
// The PFN reference checks are carefully ordered so the most common case
|
|
// is handled first, the next most common case next, etc. The case of
|
|
// a reference count of 2 occurs more than 1000x (yes, 3 orders of
|
|
// magnitude) more than a reference count of 1. And reference counts of >2
|
|
// occur 3 orders of magnitude more frequently than reference counts of
|
|
// exactly 1.
|
|
//
|
|
// The PFN reference count is then decremented.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// Pfn - the PFN index to operate on.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// None.
|
|
//
|
|
//--
|
|
//
|
|
#define MI_REMOVE_LOCKED_PAGE_CHARGE_AND_DECREF(Pfn, CallerId) \
|
|
ASSERT (Pfn->u3.e2.ReferenceCount != 0); \
|
|
if (Pfn->u3.e2.ReferenceCount == 2) { \
|
|
if (Pfn->u2.ShareCount >= 1) { \
|
|
ASSERT (Pfn->u3.e1.PageLocation == ActiveAndValid); \
|
|
MI_UNMARK_PFN_AS_LOCK_CHARGED(Pfn, CallerId); \
|
|
MmSystemLockPagesCount -= 1; \
|
|
} \
|
|
else { \
|
|
/* \
|
|
* There are multiple referencers to this page and the \
|
|
* page is no longer valid in any process address space. \
|
|
* The systemwide lock count can only be decremented \
|
|
* by the last dereference. \
|
|
*/ \
|
|
NOTHING; \
|
|
} \
|
|
Pfn->u3.e2.ReferenceCount -= 1; \
|
|
} \
|
|
else if (Pfn->u3.e2.ReferenceCount != 1) { \
|
|
/* \
|
|
* There are still multiple referencers to this page (it may \
|
|
* or may not be resident in any process address space). \
|
|
* Since the systemwide lock count can only be decremented \
|
|
* by the last dereference (and this is not it), no action \
|
|
* is taken here. \
|
|
*/ \
|
|
ASSERT (Pfn->u3.e2.ReferenceCount > 2); \
|
|
Pfn->u3.e2.ReferenceCount -= 1; \
|
|
} \
|
|
else { \
|
|
/* \
|
|
* This page has already been deleted from all process address \
|
|
* spaces. It is sitting in limbo (not on any list) awaiting \
|
|
* this last dereference. \
|
|
*/ \
|
|
PFN_NUMBER _PageFrameIndex; \
|
|
ASSERT (Pfn->u3.e2.ReferenceCount == 1); \
|
|
ASSERT (Pfn->u3.e1.PageLocation != ActiveAndValid); \
|
|
ASSERT (Pfn->u2.ShareCount == 0); \
|
|
MI_UNMARK_PFN_AS_LOCK_CHARGED(Pfn, CallerId); \
|
|
MmSystemLockPagesCount -= 1; \
|
|
_PageFrameIndex = Pfn - MmPfnDatabase; \
|
|
MiDecrementReferenceCount (_PageFrameIndex); \
|
|
}
|
|
|
|
//++
|
|
//
|
|
// VOID
|
|
// MI_ZERO_WSINDEX (
|
|
// IN PMMPFN Pfn
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// Zero the Working Set Index field of the argument PFN entry.
|
|
// There is a subtlety here on systems where the WsIndex ULONG is
|
|
// overlaid with an Event pointer and sizeof(ULONG) != sizeof(PKEVENT).
|
|
// Note this will need to be updated if we ever decide to allocate bodies of
|
|
// thread objects on 4GB boundaries.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// Pfn - the PFN index to operate on.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// None.
|
|
//
|
|
//--
|
|
//
|
|
#define MI_ZERO_WSINDEX(Pfn) \
|
|
Pfn->u1.Event = NULL;
|
|
|
|
typedef enum _MMSHARE_TYPE {
|
|
Normal,
|
|
ShareCountOnly,
|
|
AndValid
|
|
} MMSHARE_TYPE;
|
|
|
|
typedef struct _MMWSLE_HASH {
|
|
PVOID Key;
|
|
WSLE_NUMBER Index;
|
|
} MMWSLE_HASH, *PMMWSLE_HASH;
|
|
|
|
//++
|
|
//
|
|
// WSLE_NUMBER
|
|
// MI_WSLE_HASH (
|
|
// IN ULONG_PTR VirtualAddress,
|
|
// IN PMMWSL WorkingSetList
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// Hash the address
|
|
//
|
|
// Arguments:
|
|
//
|
|
// VirtualAddress - the address to hash.
|
|
//
|
|
// WorkingSetList - the working set to hash the address into.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// The hash key.
|
|
//
|
|
//--
|
|
//
|
|
#define MI_WSLE_HASH(Address, Wsl) \
|
|
((WSLE_NUMBER)(((ULONG_PTR)PAGE_ALIGN(Address) >> (PAGE_SHIFT - 2)) % \
|
|
((Wsl)->HashTableSize - 1)))
|
|
|
|
//
|
|
// Working Set List Entry.
|
|
//
|
|
|
|
typedef struct _MMWSLENTRY {
|
|
ULONG_PTR Valid : 1;
|
|
ULONG_PTR LockedInWs : 1;
|
|
ULONG_PTR LockedInMemory : 1;
|
|
ULONG_PTR Protection : 5;
|
|
ULONG_PTR SameProtectAsProto : 1;
|
|
ULONG_PTR Direct : 1;
|
|
ULONG_PTR Age : 2;
|
|
#if MM_VIRTUAL_PAGE_FILLER
|
|
ULONG_PTR Filler : MM_VIRTUAL_PAGE_FILLER;
|
|
#endif
|
|
ULONG_PTR VirtualPageNumber : MM_VIRTUAL_PAGE_SIZE;
|
|
} MMWSLENTRY;
|
|
|
|
typedef struct _MMWSLE {
|
|
union {
|
|
PVOID VirtualAddress;
|
|
ULONG_PTR Long;
|
|
MMWSLENTRY e1;
|
|
} u1;
|
|
} MMWSLE;
|
|
|
|
#define MI_GET_PROTECTION_FROM_WSLE(Wsl) ((Wsl)->u1.e1.Protection)
|
|
|
|
typedef MMWSLE *PMMWSLE;
|
|
|
|
//
|
|
// Working Set List. Must be quadword sized.
|
|
//
|
|
|
|
typedef struct _MMWSL {
|
|
SIZE_T Quota;
|
|
WSLE_NUMBER FirstFree;
|
|
WSLE_NUMBER FirstDynamic;
|
|
WSLE_NUMBER LastEntry;
|
|
WSLE_NUMBER NextSlot; // The next slot to trim
|
|
PMMWSLE Wsle;
|
|
WSLE_NUMBER LastInitializedWsle;
|
|
WSLE_NUMBER NonDirectCount;
|
|
PMMWSLE_HASH HashTable;
|
|
ULONG HashTableSize;
|
|
ULONG NumberOfCommittedPageTables;
|
|
PVOID HashTableStart;
|
|
PVOID HighestPermittedHashAddress;
|
|
ULONG NumberOfImageWaiters;
|
|
ULONG VadBitMapHint;
|
|
|
|
#if (_MI_PAGING_LEVELS >= 4)
|
|
PULONG CommittedPageTables;
|
|
|
|
ULONG NumberOfCommittedPageDirectories;
|
|
PULONG CommittedPageDirectories;
|
|
|
|
ULONG NumberOfCommittedPageDirectoryParents;
|
|
ULONG CommittedPageDirectoryParents[(MM_USER_PAGE_DIRECTORY_PARENT_PAGES + sizeof(ULONG)*8-1)/(sizeof(ULONG)*8)];
|
|
|
|
#elif (_MI_PAGING_LEVELS >= 3)
|
|
PULONG CommittedPageTables;
|
|
|
|
ULONG NumberOfCommittedPageDirectories;
|
|
ULONG CommittedPageDirectories[(MM_USER_PAGE_DIRECTORY_PAGES + sizeof(ULONG)*8-1)/(sizeof(ULONG)*8)];
|
|
|
|
#else
|
|
|
|
//
|
|
// This must be at the end.
|
|
// Not used in system cache or session working set lists.
|
|
//
|
|
|
|
USHORT UsedPageTableEntries[MM_USER_PAGE_TABLE_PAGES];
|
|
|
|
ULONG CommittedPageTables[MM_USER_PAGE_TABLE_PAGES/(sizeof(ULONG)*8)];
|
|
#endif
|
|
|
|
} MMWSL, *PMMWSL;
|
|
|
|
#if defined(_X86_)
|
|
extern PMMWSL MmWorkingSetList;
|
|
#endif
|
|
|
|
extern PKEVENT MiHighMemoryEvent;
|
|
extern PKEVENT MiLowMemoryEvent;
|
|
|
|
//
|
|
// The claim estimate of unused pages in a working set is limited
|
|
// to grow by this amount per estimation period.
|
|
//
|
|
|
|
#define MI_CLAIM_INCR 30
|
|
|
|
//
|
|
// The maximum number of different ages a page can be.
|
|
//
|
|
|
|
#define MI_USE_AGE_COUNT 4
|
|
#define MI_USE_AGE_MAX (MI_USE_AGE_COUNT - 1)
|
|
|
|
//
|
|
// If more than this "percentage" of the working set is estimated to
|
|
// be used then allow it to grow freely.
|
|
//
|
|
|
|
#define MI_REPLACEMENT_FREE_GROWTH_SHIFT 5
|
|
|
|
//
|
|
// If more than this "percentage" of the working set has been claimed
|
|
// then force replacement in low memory.
|
|
//
|
|
|
|
#define MI_REPLACEMENT_CLAIM_THRESHOLD_SHIFT 3
|
|
|
|
//
|
|
// If more than this "percentage" of the working set is estimated to
|
|
// be available then force replacement in low memory.
|
|
//
|
|
|
|
#define MI_REPLACEMENT_EAVAIL_THRESHOLD_SHIFT 3
|
|
|
|
//
|
|
// If while doing replacement a page is found of this age or older then
|
|
// replace it. Otherwise the oldest is selected.
|
|
//
|
|
|
|
#define MI_IMMEDIATE_REPLACEMENT_AGE 2
|
|
|
|
//
|
|
// When trimming, use these ages for different passes.
|
|
//
|
|
|
|
#define MI_MAX_TRIM_PASSES 4
|
|
#define MI_PASS0_TRIM_AGE 2
|
|
#define MI_PASS1_TRIM_AGE 1
|
|
#define MI_PASS2_TRIM_AGE 1
|
|
#define MI_PASS3_TRIM_AGE 1
|
|
#define MI_PASS4_TRIM_AGE 0
|
|
|
|
//
|
|
// If not a forced trim, trim pages older than this age.
|
|
//
|
|
|
|
#define MI_TRIM_AGE_THRESHOLD 2
|
|
|
|
//
|
|
// This "percentage" of a claim is up for grabs in a foreground process.
|
|
//
|
|
|
|
#define MI_FOREGROUND_CLAIM_AVAILABLE_SHIFT 3
|
|
|
|
//
|
|
// This "percentage" of a claim is up for grabs in a background process.
|
|
//
|
|
|
|
#define MI_BACKGROUND_CLAIM_AVAILABLE_SHIFT 1
|
|
|
|
//++
|
|
//
|
|
// DWORD
|
|
// MI_CALC_NEXT_VALID_ESTIMATION_SLOT (
|
|
// DWORD Previous,
|
|
// DWORD Minimum,
|
|
// DWORD Maximum,
|
|
// MI_NEXT_ESTIMATION_SLOT_CONST NextEstimationSlotConst,
|
|
// PMMWSLE Wsle
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// We iterate through the working set array in a non-sequential
|
|
// manner so that the sample is independent of any aging or trimming.
|
|
//
|
|
// This algorithm walks through the working set with a stride of
|
|
// 2^MiEstimationShift elements.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// Previous - Last slot used
|
|
//
|
|
// Minimum - Minimum acceptable slot (ie. the first dynamic one)
|
|
//
|
|
// Maximum - max slot number + 1
|
|
//
|
|
// NextEstimationSlotConst - for this algorithm it contains the stride
|
|
//
|
|
// Wsle - the working set array
|
|
//
|
|
// Return Value:
|
|
//
|
|
// Next slot.
|
|
//
|
|
// Environment:
|
|
//
|
|
// Kernel mode, APCs disabled, working set lock held and PFN lock held.
|
|
//
|
|
//--
|
|
|
|
typedef struct _MI_NEXT_ESTIMATION_SLOT_CONST {
|
|
WSLE_NUMBER Stride;
|
|
} MI_NEXT_ESTIMATION_SLOT_CONST;
|
|
|
|
|
|
#define MI_CALC_NEXT_ESTIMATION_SLOT_CONST(NextEstimationSlotConst, WorkingSetList) \
|
|
(NextEstimationSlotConst).Stride = 1 << MiEstimationShift;
|
|
|
|
#define MI_NEXT_VALID_ESTIMATION_SLOT(Previous, StartEntry, Minimum, Maximum, NextEstimationSlotConst, Wsle) \
|
|
ASSERT(((Previous) >= Minimum) && ((Previous) <= Maximum)); \
|
|
ASSERT(((StartEntry) >= Minimum) && ((StartEntry) <= Maximum)); \
|
|
do { \
|
|
(Previous) += (NextEstimationSlotConst).Stride; \
|
|
if ((Previous) > Maximum) { \
|
|
(Previous) = Minimum + ((Previous + 1) & (NextEstimationSlotConst.Stride - 1)); \
|
|
StartEntry += 1; \
|
|
(Previous) = StartEntry; \
|
|
} \
|
|
if ((Previous) > Maximum || (Previous) < Minimum) { \
|
|
StartEntry = Minimum; \
|
|
(Previous) = StartEntry; \
|
|
} \
|
|
} while (Wsle[Previous].u1.e1.Valid == 0);
|
|
|
|
//++
|
|
//
|
|
// WSLE_NUMBER
|
|
// MI_NEXT_VALID_AGING_SLOT (
|
|
// DWORD Previous,
|
|
// DWORD Minimum,
|
|
// DWORD Maximum,
|
|
// PMMWSLE Wsle
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// This finds the next slot to valid slot to age. It walks
|
|
// through the slots sequentialy.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// Previous - Last slot used
|
|
//
|
|
// Minimum - Minimum acceptable slot (ie. the first dynamic one)
|
|
//
|
|
// Maximum - Max slot number + 1
|
|
//
|
|
// Wsle - the working set array
|
|
//
|
|
// Return Value:
|
|
//
|
|
// None.
|
|
//
|
|
// Environment:
|
|
//
|
|
// Kernel mode, APCs disabled, working set lock held and PFN lock held.
|
|
//
|
|
//--
|
|
|
|
#define MI_NEXT_VALID_AGING_SLOT(Previous, Minimum, Maximum, Wsle) \
|
|
ASSERT(((Previous) >= Minimum) && ((Previous) <= Maximum)); \
|
|
do { \
|
|
(Previous) += 1; \
|
|
if ((Previous) > Maximum) { \
|
|
Previous = Minimum; \
|
|
} \
|
|
} while ((Wsle[Previous].u1.e1.Valid == 0));
|
|
|
|
//++
|
|
//
|
|
// ULONG
|
|
// MI_CALCULATE_USAGE_ESTIMATE (
|
|
// IN PULONG SampledAgeCounts.
|
|
// IN ULONG CounterShift
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// In Usage Estimation, we count the number of pages of each age in
|
|
// a sample. The function turns the SampledAgeCounts into an
|
|
// estimate of the unused pages.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// SampledAgeCounts - counts of pages of each different age in the sample
|
|
//
|
|
// CounterShift - shift necessary to apply sample to entire WS
|
|
//
|
|
// Return Value:
|
|
//
|
|
// The number of pages to walk in the working set to get a good
|
|
// estimate of the number available.
|
|
//
|
|
//--
|
|
|
|
#define MI_CALCULATE_USAGE_ESTIMATE(SampledAgeCounts, CounterShift) \
|
|
(((SampledAgeCounts)[1] + \
|
|
(SampledAgeCounts)[2] + (SampledAgeCounts)[3]) \
|
|
<< (CounterShift))
|
|
|
|
//++
|
|
//
|
|
// VOID
|
|
// MI_RESET_WSLE_AGE (
|
|
// IN PMMPTE PointerPte,
|
|
// IN PMMWSLE Wsle
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// Clear the age counter for the working set entry.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// PointerPte - pointer to the working set list entry's PTE.
|
|
//
|
|
// Wsle - pointer to the working set list entry.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// None.
|
|
//
|
|
//--
|
|
#define MI_RESET_WSLE_AGE(PointerPte, Wsle) \
|
|
(Wsle)->u1.e1.Age = 0;
|
|
|
|
//++
|
|
//
|
|
// ULONG
|
|
// MI_GET_WSLE_AGE (
|
|
// IN PMMPTE PointerPte,
|
|
// IN PMMWSLE Wsle
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// Clear the age counter for the working set entry.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// PointerPte - pointer to the working set list entry's PTE
|
|
// Wsle - pointer to the working set list entry
|
|
//
|
|
// Return Value:
|
|
//
|
|
// Age group of the working set entry
|
|
//
|
|
//--
|
|
#define MI_GET_WSLE_AGE(PointerPte, Wsle) \
|
|
((ULONG)((Wsle)->u1.e1.Age))
|
|
|
|
//++
|
|
//
|
|
// VOID
|
|
// MI_INC_WSLE_AGE (
|
|
// IN PMMPTE PointerPte,
|
|
// IN PMMWSLE Wsle,
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// Increment the age counter for the working set entry.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// PointerPte - pointer to the working set list entry's PTE.
|
|
//
|
|
// Wsle - pointer to the working set list entry.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// None
|
|
//
|
|
//--
|
|
|
|
#define MI_INC_WSLE_AGE(PointerPte, Wsle) \
|
|
if ((Wsle)->u1.e1.Age < 3) { \
|
|
(Wsle)->u1.e1.Age += 1; \
|
|
}
|
|
|
|
//++
|
|
//
|
|
// VOID
|
|
// MI_UPDATE_USE_ESTIMATE (
|
|
// IN PMMPTE PointerPte,
|
|
// IN PMMWSLE Wsle,
|
|
// IN ULONG *SampledAgeCounts
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// Update the sampled age counts.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// PointerPte - pointer to the working set list entry's PTE.
|
|
//
|
|
// Wsle - pointer to the working set list entry.
|
|
//
|
|
// SampledAgeCounts - array of age counts to be updated.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// None
|
|
//
|
|
//--
|
|
|
|
#define MI_UPDATE_USE_ESTIMATE(PointerPte, Wsle, SampledAgeCounts) \
|
|
(SampledAgeCounts)[(Wsle)->u1.e1.Age] += 1;
|
|
|
|
//++
|
|
//
|
|
// BOOLEAN
|
|
// MI_WS_GROWING_TOO_FAST (
|
|
// IN PMMSUPPORT VmSupport
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// Limit the growth rate of processes as the
|
|
// available memory approaches zero. Note the caller must ensure that
|
|
// MmAvailablePages is low enough so this calculation does not wrap.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// VmSupport - a working set.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// TRUE if the growth rate is too fast, FALSE otherwise.
|
|
//
|
|
//--
|
|
|
|
#define MI_WS_GROWING_TOO_FAST(VmSupport) \
|
|
((VmSupport)->GrowthSinceLastEstimate > \
|
|
(((MI_CLAIM_INCR * (MmAvailablePages*MmAvailablePages)) / (64*64)) + 1))
|
|
|
|
#define SECTION_BASE_ADDRESS(_NtSection) \
|
|
(*((PVOID *)&(_NtSection)->PointerToRelocations))
|
|
|
|
#define SECTION_LOCK_COUNT_POINTER(_NtSection) \
|
|
((PLONG)&(_NtSection)->NumberOfRelocations)
|
|
|
|
//
|
|
// Memory Management Object structures.
|
|
//
|
|
|
|
typedef enum _SECTION_CHECK_TYPE {
|
|
CheckDataSection,
|
|
CheckImageSection,
|
|
CheckUserDataSection,
|
|
CheckBothSection
|
|
} SECTION_CHECK_TYPE;
|
|
|
|
typedef struct _MMEXTEND_INFO {
|
|
UINT64 CommittedSize;
|
|
ULONG ReferenceCount;
|
|
} MMEXTEND_INFO, *PMMEXTEND_INFO;
|
|
|
|
typedef struct _SEGMENT {
|
|
struct _CONTROL_AREA *ControlArea;
|
|
ULONG TotalNumberOfPtes;
|
|
ULONG NonExtendedPtes;
|
|
ULONG WritableUserReferences;
|
|
|
|
UINT64 SizeOfSegment;
|
|
MMPTE SegmentPteTemplate;
|
|
|
|
SIZE_T NumberOfCommittedPages;
|
|
PMMEXTEND_INFO ExtendInfo;
|
|
|
|
PVOID SystemImageBase; // could replace with single bit in ldrmodlist
|
|
PVOID BasedAddress;
|
|
|
|
//
|
|
// The fields below are for image & pagefile-backed sections only.
|
|
// Common fields are above and new common entries must be added to
|
|
// both the SEGMENT and MAPPED_FILE_SEGMENT declarations.
|
|
//
|
|
|
|
union {
|
|
SIZE_T ImageCommitment; // for image-backed sections only
|
|
PEPROCESS CreatingProcess; // for pagefile-backed sections only
|
|
} u1;
|
|
|
|
union {
|
|
PSECTION_IMAGE_INFORMATION ImageInformation; // for images only
|
|
PVOID FirstMappedVa; // for pagefile-backed sections only
|
|
} u2;
|
|
|
|
PMMPTE PrototypePte;
|
|
MMPTE ThePtes[MM_PROTO_PTE_ALIGNMENT / PAGE_SIZE];
|
|
|
|
} SEGMENT, *PSEGMENT;
|
|
|
|
typedef struct _MAPPED_FILE_SEGMENT {
|
|
struct _CONTROL_AREA *ControlArea;
|
|
ULONG TotalNumberOfPtes;
|
|
ULONG NonExtendedPtes;
|
|
ULONG WritableUserReferences;
|
|
|
|
UINT64 SizeOfSegment;
|
|
MMPTE SegmentPteTemplate;
|
|
|
|
SIZE_T NumberOfCommittedPages;
|
|
PMMEXTEND_INFO ExtendInfo;
|
|
|
|
PVOID SystemImageBase; // could replace with single bit in ldrmodlist
|
|
PVOID BasedAddress;
|
|
struct _MSUBSECTION *LastSubsectionHint;
|
|
|
|
} MAPPED_FILE_SEGMENT, *PMAPPED_FILE_SEGMENT;
|
|
|
|
typedef struct _EVENT_COUNTER {
|
|
SINGLE_LIST_ENTRY ListEntry;
|
|
ULONG RefCount;
|
|
KEVENT Event;
|
|
} EVENT_COUNTER, *PEVENT_COUNTER;
|
|
|
|
typedef struct _MMSECTION_FLAGS {
|
|
unsigned BeingDeleted : 1;
|
|
unsigned BeingCreated : 1;
|
|
unsigned BeingPurged : 1;
|
|
unsigned NoModifiedWriting : 1;
|
|
|
|
unsigned FailAllIo : 1;
|
|
unsigned Image : 1;
|
|
unsigned Based : 1;
|
|
unsigned File : 1;
|
|
|
|
unsigned Networked : 1;
|
|
unsigned NoCache : 1;
|
|
unsigned PhysicalMemory : 1;
|
|
unsigned CopyOnWrite : 1;
|
|
|
|
unsigned Reserve : 1; // not a spare bit!
|
|
unsigned Commit : 1;
|
|
unsigned FloppyMedia : 1;
|
|
unsigned WasPurged : 1;
|
|
|
|
unsigned UserReference : 1;
|
|
unsigned GlobalMemory : 1;
|
|
unsigned DeleteOnClose : 1;
|
|
unsigned FilePointerNull : 1;
|
|
|
|
unsigned DebugSymbolsLoaded : 1;
|
|
unsigned SetMappedFileIoComplete : 1;
|
|
unsigned CollidedFlush : 1;
|
|
unsigned NoChange : 1;
|
|
|
|
unsigned HadUserReference : 1;
|
|
unsigned ImageMappedInSystemSpace : 1;
|
|
unsigned UserWritable : 1;
|
|
unsigned Accessed : 1;
|
|
|
|
unsigned GlobalOnlyPerSession : 1;
|
|
unsigned Rom : 1;
|
|
unsigned filler : 2;
|
|
} MMSECTION_FLAGS;
|
|
|
|
typedef struct _CONTROL_AREA { // must be quadword sized.
|
|
PSEGMENT Segment;
|
|
LIST_ENTRY DereferenceList;
|
|
ULONG NumberOfSectionReferences;
|
|
ULONG NumberOfPfnReferences;
|
|
ULONG NumberOfMappedViews;
|
|
USHORT NumberOfSubsections;
|
|
USHORT FlushInProgressCount;
|
|
ULONG NumberOfUserReferences;
|
|
union {
|
|
ULONG LongFlags;
|
|
MMSECTION_FLAGS Flags;
|
|
} u;
|
|
PFILE_OBJECT FilePointer;
|
|
PEVENT_COUNTER WaitingForDeletion;
|
|
USHORT ModifiedWriteCount;
|
|
USHORT NumberOfSystemCacheViews;
|
|
} CONTROL_AREA, *PCONTROL_AREA;
|
|
|
|
typedef struct _LARGE_CONTROL_AREA { // must be quadword sized.
|
|
PSEGMENT Segment;
|
|
LIST_ENTRY DereferenceList;
|
|
ULONG NumberOfSectionReferences;
|
|
ULONG NumberOfPfnReferences;
|
|
ULONG NumberOfMappedViews;
|
|
USHORT NumberOfSubsections;
|
|
USHORT FlushInProgressCount;
|
|
ULONG NumberOfUserReferences;
|
|
union {
|
|
ULONG LongFlags;
|
|
MMSECTION_FLAGS Flags;
|
|
} u;
|
|
PFILE_OBJECT FilePointer;
|
|
PEVENT_COUNTER WaitingForDeletion;
|
|
USHORT ModifiedWriteCount;
|
|
USHORT NumberOfSystemCacheViews;
|
|
PFN_NUMBER StartingFrame; // only used if Flags.Rom == 1.
|
|
LIST_ENTRY UserGlobalList;
|
|
ULONG SessionId;
|
|
} LARGE_CONTROL_AREA, *PLARGE_CONTROL_AREA;
|
|
|
|
typedef struct _MMSUBSECTION_FLAGS {
|
|
unsigned ReadOnly : 1;
|
|
unsigned ReadWrite : 1;
|
|
unsigned SubsectionStatic : 1;
|
|
unsigned GlobalMemory: 1;
|
|
unsigned Protection : 5;
|
|
unsigned LargePages : 1;
|
|
unsigned StartingSector4132 : 10; // 2 ** (42+12) == 4MB*4GB == 16K TB
|
|
unsigned SectorEndOffset : 12;
|
|
} MMSUBSECTION_FLAGS;
|
|
|
|
typedef struct _SUBSECTION { // Must start on quadword boundary and be quad sized
|
|
PCONTROL_AREA ControlArea;
|
|
union {
|
|
ULONG LongFlags;
|
|
MMSUBSECTION_FLAGS SubsectionFlags;
|
|
} u;
|
|
ULONG StartingSector;
|
|
ULONG NumberOfFullSectors; // (4GB-1) * 4K == 16TB-4K limit per subsection
|
|
PMMPTE SubsectionBase;
|
|
ULONG UnusedPtes;
|
|
ULONG PtesInSubsection;
|
|
struct _SUBSECTION *NextSubsection;
|
|
} SUBSECTION, *PSUBSECTION;
|
|
|
|
extern const ULONG MMSECT;
|
|
|
|
//
|
|
// Accesses to MMSUBSECTION_FLAGS2 are synchronized via the PFN lock
|
|
// (unlike MMSUBSECTION_FLAGS access which is not lock protected at all).
|
|
//
|
|
|
|
typedef struct _MMSUBSECTION_FLAGS2 {
|
|
unsigned SubsectionAccessed : 1;
|
|
unsigned SubsectionConverted : 1; // only needed for debug
|
|
unsigned Reserved : 30;
|
|
} MMSUBSECTION_FLAGS2;
|
|
|
|
//
|
|
// Mapped data file subsection structure. Not used for images
|
|
// or pagefile-backed shared memory.
|
|
//
|
|
|
|
typedef struct _MSUBSECTION { // Must start on quadword boundary and be quad sized
|
|
PCONTROL_AREA ControlArea;
|
|
union {
|
|
ULONG LongFlags;
|
|
MMSUBSECTION_FLAGS SubsectionFlags;
|
|
} u;
|
|
ULONG StartingSector;
|
|
ULONG NumberOfFullSectors; // (4GB-1) * 4K == 16TB-4K limit per subsection
|
|
PMMPTE SubsectionBase;
|
|
ULONG UnusedPtes;
|
|
ULONG PtesInSubsection;
|
|
struct _SUBSECTION *NextSubsection;
|
|
LIST_ENTRY DereferenceList;
|
|
ULONG_PTR NumberOfMappedViews;
|
|
union {
|
|
ULONG LongFlags2;
|
|
MMSUBSECTION_FLAGS2 SubsectionFlags2;
|
|
} u2;
|
|
} MSUBSECTION, *PMSUBSECTION;
|
|
|
|
#define MI_MAXIMUM_SECTION_SIZE ((UINT64)16*1024*1024*1024*1024*1024 - (1<<MM4K_SHIFT))
|
|
|
|
VOID
|
|
MiDecrementSubsections (
|
|
IN PSUBSECTION FirstSubsection,
|
|
IN PSUBSECTION LastSubsection OPTIONAL
|
|
);
|
|
|
|
NTSTATUS
|
|
MiAddViewsForSectionWithPfn (
|
|
IN PMSUBSECTION StartMappedSubsection,
|
|
IN ULONG LastPteOffset OPTIONAL
|
|
);
|
|
|
|
NTSTATUS
|
|
MiAddViewsForSection (
|
|
IN PMSUBSECTION MappedSubsection,
|
|
IN ULONG LastPteOffset OPTIONAL,
|
|
IN KIRQL OldIrql,
|
|
OUT PULONG Waited
|
|
);
|
|
|
|
LOGICAL
|
|
MiReferenceSubsection (
|
|
IN PMSUBSECTION MappedSubsection
|
|
);
|
|
|
|
VOID
|
|
MiRemoveViewsFromSection (
|
|
IN PMSUBSECTION StartMappedSubsection,
|
|
IN ULONG LastPteOffset OPTIONAL
|
|
);
|
|
|
|
VOID
|
|
MiRemoveViewsFromSectionWithPfn (
|
|
IN PMSUBSECTION StartMappedSubsection,
|
|
IN ULONG LastPteOffset OPTIONAL
|
|
);
|
|
|
|
VOID
|
|
MiSubsectionConsistent(
|
|
IN PSUBSECTION Subsection
|
|
);
|
|
|
|
#if DBG
|
|
#define MI_CHECK_SUBSECTION(_subsection) MiSubsectionConsistent((PSUBSECTION)(_subsection))
|
|
#else
|
|
#define MI_CHECK_SUBSECTION(_subsection)
|
|
#endif
|
|
|
|
//++
|
|
//ULONG
|
|
//Mi4KStartForSubsection (
|
|
// IN PLARGE_INTEGER address,
|
|
// IN OUT PSUBSECTION subsection
|
|
// );
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// This macro sets into the specified subsection the supplied information
|
|
// indicating the start address (in 4K units) of this portion of the file.
|
|
//
|
|
// Arguments
|
|
//
|
|
// address - Supplies the 64-bit address (in 4K units) of the start of this
|
|
// portion of the file.
|
|
//
|
|
// subsection - Supplies the subsection address to store the address in.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// None.
|
|
//
|
|
//--
|
|
|
|
#define Mi4KStartForSubsection(address, subsection) \
|
|
subsection->StartingSector = ((PLARGE_INTEGER)address)->LowPart; \
|
|
subsection->u.SubsectionFlags.StartingSector4132 = \
|
|
(((PLARGE_INTEGER)(address))->HighPart & 0x3ff);
|
|
|
|
//++
|
|
//ULONG
|
|
//Mi4KStartFromSubsection (
|
|
// IN OUT PLARGE_INTEGER address,
|
|
// IN PSUBSECTION subsection
|
|
// );
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// This macro gets the start 4K offset from the specified subsection.
|
|
//
|
|
// Arguments
|
|
//
|
|
// address - Supplies the 64-bit address (in 4K units) to place the
|
|
// start of this subsection into.
|
|
//
|
|
// subsection - Supplies the subsection address to get the address from.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// None.
|
|
//
|
|
//--
|
|
|
|
#define Mi4KStartFromSubsection(address, subsection) \
|
|
((PLARGE_INTEGER)address)->LowPart = subsection->StartingSector; \
|
|
((PLARGE_INTEGER)address)->HighPart = subsection->u.SubsectionFlags.StartingSector4132;
|
|
|
|
typedef struct _MMDEREFERENCE_SEGMENT_HEADER {
|
|
KSPIN_LOCK Lock;
|
|
KSEMAPHORE Semaphore;
|
|
LIST_ENTRY ListHead;
|
|
} MMDEREFERENCE_SEGMENT_HEADER;
|
|
|
|
//
|
|
// This entry is used for calling the segment dereference thread
|
|
// to perform page file expansion. It has a similar structure
|
|
// to a control area to allow either a control area or a page file
|
|
// expansion entry to be placed on the list. Note that for a control
|
|
// area the segment pointer is valid whereas for page file expansion
|
|
// it is null.
|
|
//
|
|
|
|
typedef struct _MMPAGE_FILE_EXPANSION {
|
|
PSEGMENT Segment;
|
|
LIST_ENTRY DereferenceList;
|
|
SIZE_T RequestedExpansionSize;
|
|
SIZE_T ActualExpansion;
|
|
KEVENT Event;
|
|
LONG InProgress;
|
|
ULONG PageFileNumber;
|
|
} MMPAGE_FILE_EXPANSION, *PMMPAGE_FILE_EXPANSION;
|
|
|
|
#define MI_EXTEND_ANY_PAGEFILE ((ULONG)-1)
|
|
#define MI_CONTRACT_PAGEFILES ((SIZE_T)-1)
|
|
|
|
typedef struct _MMWORKING_SET_EXPANSION_HEAD {
|
|
LIST_ENTRY ListHead;
|
|
} MMWORKING_SET_EXPANSION_HEAD;
|
|
|
|
#define SUBSECTION_READ_ONLY 1L
|
|
#define SUBSECTION_READ_WRITE 2L
|
|
#define SUBSECTION_COPY_ON_WRITE 4L
|
|
#define SUBSECTION_SHARE_ALLOW 8L
|
|
|
|
//
|
|
// The MMINPAGE_FLAGS relies on the fact that a pool allocation is always
|
|
// QUADWORD aligned so the low 3 bits are always available.
|
|
//
|
|
|
|
typedef struct _MMINPAGE_FLAGS {
|
|
ULONG_PTR Completed : 1;
|
|
ULONG_PTR Available1 : 1;
|
|
ULONG_PTR Available2 : 1;
|
|
#if defined (_WIN64)
|
|
ULONG_PTR PrefetchMdlHighBits : 61;
|
|
#else
|
|
ULONG_PTR PrefetchMdlHighBits : 29;
|
|
#endif
|
|
} MMINPAGE_FLAGS, *PMMINPAGE_FLAGS;
|
|
|
|
#define MI_EXTRACT_PREFETCH_MDL(_Support) ((PMDL)((ULONG_PTR)(_Support->u1.PrefetchMdl) & ~(sizeof(QUAD) - 1)))
|
|
|
|
typedef struct _MMINPAGE_SUPPORT {
|
|
KEVENT Event;
|
|
IO_STATUS_BLOCK IoStatus;
|
|
LARGE_INTEGER ReadOffset;
|
|
LONG WaitCount;
|
|
#if defined (_WIN64)
|
|
ULONG UsedPageTableEntries;
|
|
#endif
|
|
PETHREAD Thread;
|
|
PFILE_OBJECT FilePointer;
|
|
PMMPTE BasePte;
|
|
PMMPFN Pfn;
|
|
union {
|
|
MMINPAGE_FLAGS e1;
|
|
ULONG_PTR LongFlags;
|
|
PMDL PrefetchMdl; // Only used under _PREFETCH_
|
|
} u1;
|
|
MDL Mdl;
|
|
PFN_NUMBER Page[MM_MAXIMUM_READ_CLUSTER_SIZE + 1];
|
|
SINGLE_LIST_ENTRY ListEntry;
|
|
} MMINPAGE_SUPPORT, *PMMINPAGE_SUPPORT;
|
|
|
|
#define MI_PF_DUMMY_PAGE_PTE ((PMMPTE)0x23452345) // Only used by _PREFETCH_
|
|
|
|
//
|
|
// Address Node.
|
|
//
|
|
|
|
typedef struct _MMADDRESS_NODE {
|
|
ULONG_PTR StartingVpn;
|
|
ULONG_PTR EndingVpn;
|
|
struct _MMADDRESS_NODE *Parent;
|
|
struct _MMADDRESS_NODE *LeftChild;
|
|
struct _MMADDRESS_NODE *RightChild;
|
|
} MMADDRESS_NODE, *PMMADDRESS_NODE;
|
|
|
|
typedef struct _SECTION {
|
|
MMADDRESS_NODE Address;
|
|
PSEGMENT Segment;
|
|
LARGE_INTEGER SizeOfSection;
|
|
union {
|
|
ULONG LongFlags;
|
|
MMSECTION_FLAGS Flags;
|
|
} u;
|
|
ULONG InitialPageProtection;
|
|
} SECTION, *PSECTION;
|
|
|
|
//
|
|
// Banked memory descriptor. Pointed to by VAD which has
|
|
// the PhysicalMemory flags set and the Banked pointer field as
|
|
// non-NULL.
|
|
//
|
|
|
|
typedef struct _MMBANKED_SECTION {
|
|
PFN_NUMBER BasePhysicalPage;
|
|
PMMPTE BasedPte;
|
|
ULONG BankSize;
|
|
ULONG BankShift; //shift for PTEs to calculate bank number
|
|
PBANKED_SECTION_ROUTINE BankedRoutine;
|
|
PVOID Context;
|
|
PMMPTE CurrentMappedPte;
|
|
MMPTE BankTemplate[1];
|
|
} MMBANKED_SECTION, *PMMBANKED_SECTION;
|
|
|
|
|
|
//
|
|
// Virtual address descriptor
|
|
//
|
|
// ***** NOTE **********
|
|
// The first part of a virtual address descriptor is a MMADDRESS_NODE!!!
|
|
//
|
|
|
|
#if defined (_WIN64)
|
|
|
|
#define COMMIT_SIZE 51
|
|
|
|
#if ((COMMIT_SIZE + PAGE_SHIFT) < 63)
|
|
#error COMMIT_SIZE too small
|
|
#endif
|
|
|
|
#else
|
|
#define COMMIT_SIZE 19
|
|
|
|
#if ((COMMIT_SIZE + PAGE_SHIFT) < 31)
|
|
#error COMMIT_SIZE too small
|
|
#endif
|
|
#endif
|
|
|
|
#define MM_MAX_COMMIT (((ULONG_PTR) 1 << COMMIT_SIZE) - 1)
|
|
|
|
#define MM_VIEW_UNMAP 0
|
|
#define MM_VIEW_SHARE 1
|
|
|
|
typedef struct _MMVAD_FLAGS {
|
|
ULONG_PTR CommitCharge : COMMIT_SIZE; //limits system to 4k pages or bigger!
|
|
ULONG_PTR PhysicalMapping : 1;
|
|
ULONG_PTR ImageMap : 1;
|
|
ULONG_PTR UserPhysicalPages : 1;
|
|
ULONG_PTR NoChange : 1;
|
|
ULONG_PTR WriteWatch : 1;
|
|
ULONG_PTR Protection : 5;
|
|
ULONG_PTR LargePages : 1;
|
|
ULONG_PTR MemCommit: 1;
|
|
ULONG_PTR PrivateMemory : 1; //used to tell VAD from VAD_SHORT
|
|
} MMVAD_FLAGS;
|
|
|
|
typedef struct _MMVAD_FLAGS2 {
|
|
unsigned FileOffset : 24; // number of 64k units into file
|
|
unsigned SecNoChange : 1; // set if SEC_NOCHANGE specified
|
|
unsigned OneSecured : 1; // set if u3 field is a range
|
|
unsigned MultipleSecured : 1; // set if u3 field is a list head
|
|
unsigned ReadOnly : 1; // protected as ReadOnly
|
|
unsigned LongVad : 1; // set if VAD is a long VAD
|
|
unsigned ExtendableFile : 1;
|
|
unsigned Inherit : 1; //1 = ViewShare, 0 = ViewUnmap
|
|
unsigned CopyOnWrite : 1;
|
|
} MMVAD_FLAGS2;
|
|
|
|
typedef struct _MMADDRESS_LIST {
|
|
ULONG_PTR StartVpn;
|
|
ULONG_PTR EndVpn;
|
|
} MMADDRESS_LIST, *PMMADDRESS_LIST;
|
|
|
|
typedef struct _MMSECURE_ENTRY {
|
|
union {
|
|
ULONG_PTR LongFlags2;
|
|
MMVAD_FLAGS2 VadFlags2;
|
|
} u2;
|
|
ULONG_PTR StartVpn;
|
|
ULONG_PTR EndVpn;
|
|
LIST_ENTRY List;
|
|
} MMSECURE_ENTRY, *PMMSECURE_ENTRY;
|
|
|
|
typedef struct _ALIAS_VAD_INFO {
|
|
KAPC Apc;
|
|
ULONG NumberOfEntries;
|
|
ULONG MaximumEntries;
|
|
} ALIAS_VAD_INFO, *PALIAS_VAD_INFO;
|
|
|
|
typedef struct _ALIAS_VAD_INFO2 {
|
|
ULONG BaseAddress;
|
|
HANDLE SecureHandle;
|
|
} ALIAS_VAD_INFO2, *PALIAS_VAD_INFO2;
|
|
|
|
typedef struct _MMVAD {
|
|
ULONG_PTR StartingVpn;
|
|
ULONG_PTR EndingVpn;
|
|
struct _MMVAD *Parent;
|
|
struct _MMVAD *LeftChild;
|
|
struct _MMVAD *RightChild;
|
|
union {
|
|
ULONG_PTR LongFlags;
|
|
MMVAD_FLAGS VadFlags;
|
|
} u;
|
|
PCONTROL_AREA ControlArea;
|
|
PMMPTE FirstPrototypePte;
|
|
PMMPTE LastContiguousPte;
|
|
union {
|
|
ULONG LongFlags2;
|
|
MMVAD_FLAGS2 VadFlags2;
|
|
} u2;
|
|
} MMVAD, *PMMVAD;
|
|
|
|
typedef struct _MMVAD_LONG {
|
|
ULONG_PTR StartingVpn;
|
|
ULONG_PTR EndingVpn;
|
|
struct _MMVAD *Parent;
|
|
struct _MMVAD *LeftChild;
|
|
struct _MMVAD *RightChild;
|
|
union {
|
|
ULONG_PTR LongFlags;
|
|
MMVAD_FLAGS VadFlags;
|
|
} u;
|
|
PCONTROL_AREA ControlArea;
|
|
PMMPTE FirstPrototypePte;
|
|
PMMPTE LastContiguousPte;
|
|
union {
|
|
ULONG LongFlags2;
|
|
MMVAD_FLAGS2 VadFlags2;
|
|
} u2;
|
|
union {
|
|
LIST_ENTRY List;
|
|
MMADDRESS_LIST Secured;
|
|
} u3;
|
|
union {
|
|
PMMBANKED_SECTION Banked;
|
|
PMMEXTEND_INFO ExtendedInfo;
|
|
} u4;
|
|
#if defined(_MIALT4K_)
|
|
PALIAS_VAD_INFO AliasInformation;
|
|
#endif
|
|
} MMVAD_LONG, *PMMVAD_LONG;
|
|
|
|
typedef struct _MMVAD_SHORT {
|
|
ULONG_PTR StartingVpn;
|
|
ULONG_PTR EndingVpn;
|
|
struct _MMVAD *Parent;
|
|
struct _MMVAD *LeftChild;
|
|
struct _MMVAD *RightChild;
|
|
union {
|
|
ULONG_PTR LongFlags;
|
|
MMVAD_FLAGS VadFlags;
|
|
} u;
|
|
} MMVAD_SHORT, *PMMVAD_SHORT;
|
|
|
|
#define MI_GET_PROTECTION_FROM_VAD(_Vad) ((ULONG)(_Vad)->u.VadFlags.Protection)
|
|
|
|
#define MI_PHYSICAL_VIEW_AWE 0x1 // AWE region
|
|
#define MI_PHYSICAL_VIEW_PHYS 0x2 // Device\PhysicalMemory region
|
|
|
|
typedef struct _MI_PHYSICAL_VIEW {
|
|
LIST_ENTRY ListEntry;
|
|
PMMVAD Vad;
|
|
PCHAR StartVa;
|
|
PCHAR EndVa;
|
|
union {
|
|
ULONG_PTR LongFlags; // physical or AWE Vad identification
|
|
PRTL_BITMAP BitMap; // only if Vad->u.VadFlags.WriteWatch == 1
|
|
} u;
|
|
} MI_PHYSICAL_VIEW, *PMI_PHYSICAL_VIEW;
|
|
|
|
#define MI_PHYSICAL_VIEW_KEY 'vpmM'
|
|
#define MI_WRITEWATCH_VIEW_KEY 'wWmM'
|
|
|
|
//
|
|
// Stuff for support of Write Watch.
|
|
//
|
|
|
|
extern ULONG_PTR MiActiveWriteWatch;
|
|
|
|
VOID
|
|
MiCaptureWriteWatchDirtyBit (
|
|
IN PEPROCESS Process,
|
|
IN PVOID VirtualAddress
|
|
);
|
|
|
|
//
|
|
// Stuff for support of AWE (Address Windowing Extensions).
|
|
//
|
|
|
|
typedef struct _AWEINFO {
|
|
PRTL_BITMAP VadPhysicalPagesBitMap;
|
|
ULONG_PTR VadPhysicalPages;
|
|
|
|
//
|
|
// The PushLock is used to allow most of the NtMapUserPhysicalPages{Scatter}
|
|
// to execute in parallel as this is acquired shared for these calls.
|
|
// Exclusive acquisitions are used to protect maps against frees of the
|
|
// pages as well as to protect updates to the AweVadList. Collisions
|
|
// should be rare because the exclusive acquisitions should be rare.
|
|
//
|
|
|
|
PEX_PUSH_LOCK_CACHE_AWARE PushLock;
|
|
|
|
LIST_ENTRY AweVadList;
|
|
} AWEINFO, *PAWEINFO;
|
|
|
|
VOID
|
|
MiAweViewInserter (
|
|
IN PEPROCESS Process,
|
|
IN PMI_PHYSICAL_VIEW PhysicalView
|
|
);
|
|
|
|
VOID
|
|
MiAweViewRemover (
|
|
IN PEPROCESS Process,
|
|
IN PMMVAD Vad
|
|
);
|
|
|
|
//
|
|
// Stuff for support of POSIX Fork.
|
|
//
|
|
|
|
typedef struct _MMCLONE_BLOCK {
|
|
MMPTE ProtoPte;
|
|
LONG CloneRefCount;
|
|
} MMCLONE_BLOCK, *PMMCLONE_BLOCK;
|
|
|
|
typedef struct _MMCLONE_HEADER {
|
|
ULONG NumberOfPtes;
|
|
LONG NumberOfProcessReferences;
|
|
PMMCLONE_BLOCK ClonePtes;
|
|
} MMCLONE_HEADER, *PMMCLONE_HEADER;
|
|
|
|
typedef struct _MMCLONE_DESCRIPTOR {
|
|
ULONG_PTR StartingVpn;
|
|
ULONG_PTR EndingVpn;
|
|
struct _MMCLONE_DESCRIPTOR *Parent;
|
|
struct _MMCLONE_DESCRIPTOR *LeftChild;
|
|
struct _MMCLONE_DESCRIPTOR *RightChild;
|
|
ULONG NumberOfPtes;
|
|
PMMCLONE_HEADER CloneHeader;
|
|
LONG NumberOfReferences;
|
|
LONG FinalNumberOfReferences;
|
|
SIZE_T PagedPoolQuotaCharge;
|
|
} MMCLONE_DESCRIPTOR, *PMMCLONE_DESCRIPTOR;
|
|
|
|
//
|
|
// The following macro allocates and initializes a bitmap from the
|
|
// specified pool of the specified size.
|
|
//
|
|
// VOID
|
|
// MiCreateBitMap (
|
|
// OUT PRTL_BITMAP *BitMapHeader,
|
|
// IN SIZE_T SizeOfBitMap,
|
|
// IN POOL_TYPE PoolType
|
|
// );
|
|
//
|
|
|
|
#define MiCreateBitMap(BMH,S,P) { \
|
|
ULONG _S; \
|
|
ASSERT ((ULONG64)(S) < _4gb); \
|
|
_S = sizeof(RTL_BITMAP) + (ULONG)((((S) + 31) / 32) * 4); \
|
|
*(BMH) = (PRTL_BITMAP)ExAllocatePoolWithTag( (P), _S, ' mM'); \
|
|
if (*(BMH)) { \
|
|
RtlInitializeBitMap( *(BMH), (PULONG)((*(BMH))+1), (ULONG)(S)); \
|
|
} \
|
|
}
|
|
|
|
#define MiRemoveBitMap(BMH) { \
|
|
ExFreePool(*(BMH)); \
|
|
*(BMH) = NULL; \
|
|
}
|
|
|
|
#define MI_INITIALIZE_ZERO_MDL(MDL) { \
|
|
MDL->Next = (PMDL) NULL; \
|
|
MDL->MdlFlags = 0; \
|
|
MDL->StartVa = NULL; \
|
|
MDL->ByteOffset = 0; \
|
|
MDL->ByteCount = 0; \
|
|
}
|
|
|
|
//
|
|
// Page File structures.
|
|
//
|
|
|
|
typedef struct _MMMOD_WRITER_LISTHEAD {
|
|
LIST_ENTRY ListHead;
|
|
KEVENT Event;
|
|
} MMMOD_WRITER_LISTHEAD, *PMMMOD_WRITER_LISTHEAD;
|
|
|
|
typedef struct _MMMOD_WRITER_MDL_ENTRY {
|
|
LIST_ENTRY Links;
|
|
LARGE_INTEGER WriteOffset;
|
|
union {
|
|
IO_STATUS_BLOCK IoStatus;
|
|
LARGE_INTEGER LastByte;
|
|
} u;
|
|
PIRP Irp;
|
|
ULONG_PTR LastPageToWrite;
|
|
PMMMOD_WRITER_LISTHEAD PagingListHead;
|
|
PLIST_ENTRY CurrentList;
|
|
struct _MMPAGING_FILE *PagingFile;
|
|
PFILE_OBJECT File;
|
|
PCONTROL_AREA ControlArea;
|
|
PERESOURCE FileResource;
|
|
MDL Mdl;
|
|
PFN_NUMBER Page[1];
|
|
} MMMOD_WRITER_MDL_ENTRY, *PMMMOD_WRITER_MDL_ENTRY;
|
|
|
|
|
|
#define MM_PAGING_FILE_MDLS 2
|
|
|
|
typedef struct _MMPAGING_FILE {
|
|
PFN_NUMBER Size;
|
|
PFN_NUMBER MaximumSize;
|
|
PFN_NUMBER MinimumSize;
|
|
PFN_NUMBER FreeSpace;
|
|
PFN_NUMBER CurrentUsage;
|
|
PFN_NUMBER PeakUsage;
|
|
PFN_NUMBER Hint;
|
|
PFN_NUMBER HighestPage;
|
|
PMMMOD_WRITER_MDL_ENTRY Entry[MM_PAGING_FILE_MDLS];
|
|
PRTL_BITMAP Bitmap;
|
|
PFILE_OBJECT File;
|
|
UNICODE_STRING PageFileName;
|
|
ULONG PageFileNumber;
|
|
BOOLEAN Extended;
|
|
BOOLEAN HintSetToZero;
|
|
BOOLEAN BootPartition;
|
|
HANDLE FileHandle;
|
|
} MMPAGING_FILE, *PMMPAGING_FILE;
|
|
|
|
//
|
|
// System PTE structures.
|
|
//
|
|
|
|
typedef enum _MMSYSTEM_PTE_POOL_TYPE {
|
|
SystemPteSpace,
|
|
NonPagedPoolExpansion,
|
|
MaximumPtePoolTypes
|
|
} MMSYSTEM_PTE_POOL_TYPE;
|
|
|
|
typedef struct _MMFREE_POOL_ENTRY {
|
|
LIST_ENTRY List; // maintained free&chk, 1st entry only
|
|
PFN_NUMBER Size; // maintained free&chk, 1st entry only
|
|
ULONG Signature; // maintained chk only, all entries
|
|
struct _MMFREE_POOL_ENTRY *Owner; // maintained free&chk, all entries
|
|
} MMFREE_POOL_ENTRY, *PMMFREE_POOL_ENTRY;
|
|
|
|
|
|
typedef struct _MMLOCK_CONFLICT {
|
|
LIST_ENTRY List;
|
|
PETHREAD Thread;
|
|
} MMLOCK_CONFLICT, *PMMLOCK_CONFLICT;
|
|
|
|
//
|
|
// System view structures
|
|
//
|
|
|
|
typedef struct _MMVIEW {
|
|
ULONG_PTR Entry;
|
|
PCONTROL_AREA ControlArea;
|
|
} MMVIEW, *PMMVIEW;
|
|
|
|
//
|
|
// The MMSESSION structure represents kernel memory that is only valid on a
|
|
// per-session basis, thus the calling thread must be in the proper session
|
|
// to access this structure.
|
|
//
|
|
|
|
typedef struct _MMSESSION {
|
|
|
|
//
|
|
// Never refer to the SystemSpaceViewLock directly - always use the pointer
|
|
// following it or you will break support for multiple concurrent sessions.
|
|
//
|
|
|
|
FAST_MUTEX SystemSpaceViewLock;
|
|
|
|
//
|
|
// This points to the mutex above and is needed because the MMSESSION
|
|
// is mapped in session space on Hydra and the mutex needs to be globally
|
|
// visible for proper KeWaitForSingleObject & KeSetEvent operation.
|
|
//
|
|
|
|
PFAST_MUTEX SystemSpaceViewLockPointer;
|
|
PCHAR SystemSpaceViewStart;
|
|
PMMVIEW SystemSpaceViewTable;
|
|
ULONG SystemSpaceHashSize;
|
|
ULONG SystemSpaceHashEntries;
|
|
ULONG SystemSpaceHashKey;
|
|
PRTL_BITMAP SystemSpaceBitMap;
|
|
|
|
} MMSESSION, *PMMSESSION;
|
|
|
|
extern MMSESSION MmSession;
|
|
|
|
#define LOCK_SYSTEM_VIEW_SPACE(_Session) \
|
|
ExAcquireFastMutex (_Session->SystemSpaceViewLockPointer)
|
|
|
|
#define UNLOCK_SYSTEM_VIEW_SPACE(_Session) \
|
|
ExReleaseFastMutex (_Session->SystemSpaceViewLockPointer)
|
|
|
|
//
|
|
// List for flushing TBs singularly.
|
|
//
|
|
|
|
typedef struct _MMPTE_FLUSH_LIST {
|
|
ULONG Count;
|
|
PMMPTE FlushPte[MM_MAXIMUM_FLUSH_COUNT];
|
|
PVOID FlushVa[MM_MAXIMUM_FLUSH_COUNT];
|
|
} MMPTE_FLUSH_LIST, *PMMPTE_FLUSH_LIST;
|
|
|
|
typedef struct _LOCK_TRACKER {
|
|
LIST_ENTRY ListEntry;
|
|
PMDL Mdl;
|
|
PVOID StartVa;
|
|
PFN_NUMBER Count;
|
|
ULONG Offset;
|
|
ULONG Length;
|
|
PFN_NUMBER Page;
|
|
PVOID CallingAddress;
|
|
PVOID CallersCaller;
|
|
LIST_ENTRY GlobalListEntry;
|
|
ULONG Who;
|
|
PEPROCESS Process;
|
|
} LOCK_TRACKER, *PLOCK_TRACKER;
|
|
|
|
extern LOGICAL MmTrackLockedPages;
|
|
extern BOOLEAN MiTrackingAborted;
|
|
extern KSPIN_LOCK MiTrackLockedPagesLock;
|
|
|
|
typedef struct _LOCK_HEADER {
|
|
LIST_ENTRY ListHead;
|
|
PFN_NUMBER Count;
|
|
} LOCK_HEADER, *PLOCK_HEADER;
|
|
|
|
extern LOGICAL MmSnapUnloads;
|
|
|
|
#define MI_UNLOADED_DRIVERS 50
|
|
|
|
extern ULONG MmLastUnloadedDriver;
|
|
extern PUNLOADED_DRIVERS MmUnloadedDrivers;
|
|
|
|
|
|
VOID
|
|
MiInitMachineDependent (
|
|
IN PLOADER_PARAMETER_BLOCK LoaderBlock
|
|
);
|
|
|
|
VOID
|
|
MiReportPhysicalMemory (
|
|
VOID
|
|
);
|
|
|
|
extern PFN_NUMBER MiNumberOfCompressionPages;
|
|
|
|
NTSTATUS
|
|
MiArmCompressionInterrupt (
|
|
VOID
|
|
);
|
|
|
|
VOID
|
|
MiBuildPagedPool (
|
|
VOID
|
|
);
|
|
|
|
VOID
|
|
MiInitializeNonPagedPool (
|
|
VOID
|
|
);
|
|
|
|
LOGICAL
|
|
MiInitializeSystemSpaceMap (
|
|
PVOID Session OPTIONAL
|
|
);
|
|
|
|
VOID
|
|
MiFindInitializationCode (
|
|
OUT PVOID *StartVa,
|
|
OUT PVOID *EndVa
|
|
);
|
|
|
|
VOID
|
|
MiFreeInitializationCode (
|
|
IN PVOID StartVa,
|
|
IN PVOID EndVa
|
|
);
|
|
|
|
extern ULONG MiNonCachedCollisions;
|
|
|
|
//
|
|
// If /NOLOWMEM is used, this is set to the boundary PFN (pages below this
|
|
// value are not used whenever possible).
|
|
//
|
|
|
|
extern PFN_NUMBER MiNoLowMemory;
|
|
|
|
PVOID
|
|
MiAllocateLowMemory (
|
|
IN SIZE_T NumberOfBytes,
|
|
IN PFN_NUMBER LowestAcceptablePfn,
|
|
IN PFN_NUMBER HighestAcceptablePfn,
|
|
IN PFN_NUMBER BoundaryPfn,
|
|
IN PVOID CallingAddress,
|
|
IN MEMORY_CACHING_TYPE CacheType,
|
|
IN ULONG Tag
|
|
);
|
|
|
|
LOGICAL
|
|
MiFreeLowMemory (
|
|
IN PVOID BaseAddress,
|
|
IN ULONG Tag
|
|
);
|
|
|
|
//
|
|
// Move drivers out of the low 16mb that ntldr placed them at - this makes more
|
|
// memory below 16mb available for ISA-type drivers that cannot run without it.
|
|
//
|
|
|
|
extern LOGICAL MmMakeLowMemory;
|
|
|
|
VOID
|
|
MiRemoveLowPages (
|
|
IN ULONG RemovePhase
|
|
);
|
|
|
|
ULONG
|
|
MiSectionInitialization (
|
|
VOID
|
|
);
|
|
|
|
#define MI_MAX_DEREFERENCE_CHUNK (64 * 1024 / PAGE_SIZE)
|
|
|
|
typedef struct _MI_PFN_DEREFERENCE_CHUNK {
|
|
SINGLE_LIST_ENTRY ListEntry;
|
|
CSHORT Flags;
|
|
USHORT NumberOfPages;
|
|
PFN_NUMBER Pfns[MI_MAX_DEREFERENCE_CHUNK];
|
|
} MI_PFN_DEREFERENCE_CHUNK, *PMI_PFN_DEREFERENCE_CHUNK;
|
|
|
|
extern SLIST_HEADER MmPfnDereferenceSListHead;
|
|
extern PSINGLE_LIST_ENTRY MmPfnDeferredList;
|
|
|
|
#define MI_DEFER_PFN_HELD 0x1
|
|
#define MI_DEFER_DRAIN_LOCAL_ONLY 0x2
|
|
|
|
VOID
|
|
MiDeferredUnlockPages (
|
|
ULONG Flags
|
|
);
|
|
|
|
LOGICAL
|
|
MiFreeAllExpansionNonPagedPool (
|
|
IN LOGICAL PoolLockHeld
|
|
);
|
|
|
|
VOID
|
|
FASTCALL
|
|
MiDecrementReferenceCount (
|
|
IN PFN_NUMBER PageFrameIndex
|
|
);
|
|
|
|
//++
|
|
//VOID
|
|
//MiDecrementReferenceCountInline (
|
|
// IN PMMPFN PFN
|
|
// IN PFN_NUMBER FRAME
|
|
// );
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// MiDecrementReferenceCountInline decrements the reference count inline,
|
|
// only calling MiDecrementReferenceCount if the count would go to zero
|
|
// which would cause the page to be released.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// PFN - Supplies the PFN to decrement.
|
|
//
|
|
// FRAME - Supplies the frame matching the above PFN.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// None.
|
|
//
|
|
// Environment:
|
|
//
|
|
// PFN lock held.
|
|
//
|
|
//--
|
|
|
|
#define MiDecrementReferenceCountInline(PFN, FRAME) \
|
|
MM_PFN_LOCK_ASSERT(); \
|
|
ASSERT (MI_PFN_ELEMENT(FRAME) == (PFN)); \
|
|
ASSERT ((FRAME) <= MmHighestPhysicalPage); \
|
|
ASSERT ((PFN)->u3.e2.ReferenceCount != 0); \
|
|
if ((PFN)->u3.e2.ReferenceCount != 1) { \
|
|
(PFN)->u3.e2.ReferenceCount -= 1; \
|
|
} \
|
|
else { \
|
|
MiDecrementReferenceCount (FRAME); \
|
|
}
|
|
|
|
VOID
|
|
FASTCALL
|
|
MiDecrementShareCount (
|
|
IN PFN_NUMBER PageFrameIndex
|
|
);
|
|
|
|
#define MiDecrementShareCountOnly(P) MiDecrementShareCount(P)
|
|
|
|
#define MiDecrementShareAndValidCount(P) MiDecrementShareCount(P)
|
|
|
|
//++
|
|
//VOID
|
|
//MiDecrementShareCountInline (
|
|
// IN PMMPFN PFN,
|
|
// IN PFN_NUMBER FRAME
|
|
// );
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// MiDecrementShareCountInline decrements the share count inline,
|
|
// only calling MiDecrementShareCount if the count would go to zero
|
|
// which would cause the page to be released.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// PFN - Supplies the PFN to decrement.
|
|
//
|
|
// FRAME - Supplies the frame matching the above PFN.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// None.
|
|
//
|
|
// Environment:
|
|
//
|
|
// PFN lock held.
|
|
//
|
|
//--
|
|
|
|
#define MiDecrementShareCountInline(PFN, FRAME) \
|
|
MM_PFN_LOCK_ASSERT(); \
|
|
ASSERT (((FRAME) <= MmHighestPhysicalPage) && ((FRAME) > 0)); \
|
|
ASSERT (MI_PFN_ELEMENT(FRAME) == (PFN)); \
|
|
ASSERT ((PFN)->u2.ShareCount != 0); \
|
|
if ((PFN)->u3.e1.PageLocation != ActiveAndValid && (PFN)->u3.e1.PageLocation != StandbyPageList) { \
|
|
KeBugCheckEx (PFN_LIST_CORRUPT, 0x99, FRAME, (PFN)->u3.e1.PageLocation, 0); \
|
|
} \
|
|
if ((PFN)->u2.ShareCount != 1) { \
|
|
(PFN)->u2.ShareCount -= 1; \
|
|
PERFINFO_DECREFCNT((PFN), PERF_SOFT_TRIM, PERFINFO_LOG_TYPE_DECSHARCNT); \
|
|
ASSERT ((PFN)->u2.ShareCount < 0xF000000); \
|
|
} \
|
|
else { \
|
|
MiDecrementShareCount (FRAME); \
|
|
}
|
|
|
|
//
|
|
// Routines which operate on the Page Frame Database Lists
|
|
//
|
|
|
|
VOID
|
|
FASTCALL
|
|
MiInsertPageInList (
|
|
IN PMMPFNLIST ListHead,
|
|
IN PFN_NUMBER PageFrameIndex
|
|
);
|
|
|
|
VOID
|
|
FASTCALL
|
|
MiInsertPageInFreeList (
|
|
IN PFN_NUMBER PageFrameIndex
|
|
);
|
|
|
|
VOID
|
|
FASTCALL
|
|
MiInsertStandbyListAtFront (
|
|
IN PFN_NUMBER PageFrameIndex
|
|
);
|
|
|
|
PFN_NUMBER //PageFrameIndex
|
|
FASTCALL
|
|
MiRemovePageFromList (
|
|
IN PMMPFNLIST ListHead
|
|
);
|
|
|
|
VOID
|
|
FASTCALL
|
|
MiUnlinkPageFromList (
|
|
IN PMMPFN Pfn
|
|
);
|
|
|
|
VOID
|
|
MiUnlinkFreeOrZeroedPage (
|
|
IN PFN_NUMBER Page
|
|
);
|
|
|
|
VOID
|
|
FASTCALL
|
|
MiInsertFrontModifiedNoWrite (
|
|
IN PFN_NUMBER PageFrameIndex
|
|
);
|
|
|
|
#define MM_MEDIUM_LIMIT 32
|
|
|
|
#define MM_HIGH_LIMIT 128
|
|
|
|
ULONG
|
|
FASTCALL
|
|
MiEnsureAvailablePageOrWait (
|
|
IN PEPROCESS Process,
|
|
IN PVOID VirtualAddress
|
|
);
|
|
|
|
PFN_NUMBER
|
|
MiAllocatePfn (
|
|
IN PMMPTE PointerPte,
|
|
IN ULONG Protection
|
|
);
|
|
|
|
PFN_NUMBER
|
|
FASTCALL
|
|
MiRemoveAnyPage (
|
|
IN ULONG PageColor
|
|
);
|
|
|
|
PFN_NUMBER
|
|
FASTCALL
|
|
MiRemoveZeroPage (
|
|
IN ULONG PageColor
|
|
);
|
|
|
|
VOID
|
|
MiPurgeTransitionList (
|
|
VOID
|
|
);
|
|
|
|
PVOID
|
|
MiFindContiguousMemory (
|
|
IN PFN_NUMBER LowestPfn,
|
|
IN PFN_NUMBER HighestPfn,
|
|
IN PFN_NUMBER BoundaryPfn,
|
|
IN PFN_NUMBER SizeInPages,
|
|
IN MEMORY_CACHING_TYPE CacheType,
|
|
IN PVOID CallingAddress
|
|
);
|
|
|
|
PVOID
|
|
MiCheckForContiguousMemory (
|
|
IN PVOID BaseAddress,
|
|
IN PFN_NUMBER BaseAddressPages,
|
|
IN PFN_NUMBER SizeInPages,
|
|
IN PFN_NUMBER LowestPfn,
|
|
IN PFN_NUMBER HighestPfn,
|
|
IN PFN_NUMBER BoundaryPfn,
|
|
IN MI_PFN_CACHE_ATTRIBUTE CacheAttribute
|
|
);
|
|
|
|
//
|
|
// Routines which operate on the page frame database entry.
|
|
//
|
|
|
|
VOID
|
|
MiInitializePfn (
|
|
IN PFN_NUMBER PageFrameIndex,
|
|
IN PMMPTE PointerPte,
|
|
IN ULONG ModifiedState
|
|
);
|
|
|
|
VOID
|
|
MiInitializePfnForOtherProcess (
|
|
IN PFN_NUMBER PageFrameIndex,
|
|
IN PMMPTE PointerPte,
|
|
IN PFN_NUMBER ContainingPageFrame
|
|
);
|
|
|
|
VOID
|
|
MiInitializeCopyOnWritePfn (
|
|
IN PFN_NUMBER PageFrameIndex,
|
|
IN PMMPTE PointerPte,
|
|
IN WSLE_NUMBER WorkingSetIndex,
|
|
IN PVOID SessionSpace
|
|
);
|
|
|
|
VOID
|
|
MiInitializeTransitionPfn (
|
|
IN PFN_NUMBER PageFrameIndex,
|
|
IN PMMPTE PointerPte
|
|
);
|
|
|
|
extern SLIST_HEADER MmInPageSupportSListHead;
|
|
|
|
VOID
|
|
MiFreeInPageSupportBlock (
|
|
IN PMMINPAGE_SUPPORT Support
|
|
);
|
|
|
|
PMMINPAGE_SUPPORT
|
|
MiGetInPageSupportBlock (
|
|
IN LOGICAL PfnHeld,
|
|
IN PEPROCESS Process
|
|
);
|
|
|
|
//
|
|
// Routines which require a physical page to be mapped into hyperspace
|
|
// within the current process.
|
|
//
|
|
|
|
VOID
|
|
FASTCALL
|
|
MiZeroPhysicalPage (
|
|
IN PFN_NUMBER PageFrameIndex,
|
|
IN ULONG Color
|
|
);
|
|
|
|
VOID
|
|
FASTCALL
|
|
MiRestoreTransitionPte (
|
|
IN PFN_NUMBER PageFrameIndex
|
|
);
|
|
|
|
PSUBSECTION
|
|
MiGetSubsectionAndProtoFromPte (
|
|
IN PMMPTE PointerPte,
|
|
IN PMMPTE *ProtoPte
|
|
);
|
|
|
|
PVOID
|
|
MiMapPageInHyperSpace (
|
|
IN PEPROCESS Process,
|
|
IN PFN_NUMBER PageFrameIndex,
|
|
OUT PKIRQL OldIrql
|
|
);
|
|
|
|
PVOID
|
|
MiMapPageInHyperSpaceAtDpc (
|
|
IN PEPROCESS Process,
|
|
IN PFN_NUMBER PageFrameIndex
|
|
);
|
|
|
|
#define MiUnmapPageInZeroSpace(VA) \
|
|
MiGetPteAddress(VA)->u.Long = 0;
|
|
|
|
PVOID
|
|
MiMapImageHeaderInHyperSpace (
|
|
IN PFN_NUMBER PageFrameIndex
|
|
);
|
|
|
|
VOID
|
|
MiUnmapImageHeaderInHyperSpace (
|
|
VOID
|
|
);
|
|
|
|
VOID
|
|
MiUpdateImageHeaderPage (
|
|
IN PMMPTE PointerPte,
|
|
IN PFN_NUMBER PageFrameNumber,
|
|
IN PCONTROL_AREA ControlArea
|
|
);
|
|
|
|
PFN_NUMBER
|
|
MiGetPageForHeader (
|
|
VOID
|
|
);
|
|
|
|
VOID
|
|
MiRemoveImageHeaderPage (
|
|
IN PFN_NUMBER PageFrameNumber
|
|
);
|
|
|
|
PVOID
|
|
MiMapPageToZeroInHyperSpace (
|
|
IN PFN_NUMBER PageFrameIndex
|
|
);
|
|
|
|
|
|
NTSTATUS
|
|
MiGetWritablePagesInSection(
|
|
IN PSECTION Section,
|
|
OUT PULONG WritablePages
|
|
);
|
|
|
|
|
|
//
|
|
// Routines to obtain and release system PTEs.
|
|
//
|
|
|
|
PMMPTE
|
|
MiReserveSystemPtes (
|
|
IN ULONG NumberOfPtes,
|
|
IN MMSYSTEM_PTE_POOL_TYPE SystemPteType
|
|
);
|
|
|
|
PMMPTE
|
|
MiReserveAlignedSystemPtes (
|
|
IN ULONG NumberOfPtes,
|
|
IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType,
|
|
IN ULONG Alignment
|
|
);
|
|
|
|
VOID
|
|
MiReleaseSystemPtes (
|
|
IN PMMPTE StartingPte,
|
|
IN ULONG NumberOfPtes,
|
|
IN MMSYSTEM_PTE_POOL_TYPE SystemPteType
|
|
);
|
|
|
|
VOID
|
|
MiReleaseSplitSystemPtes (
|
|
IN PMMPTE StartingPte,
|
|
IN ULONG NumberOfPtes,
|
|
IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType
|
|
);
|
|
|
|
VOID
|
|
MiIncrementSystemPtes (
|
|
IN ULONG NumberOfPtes
|
|
);
|
|
|
|
LOGICAL
|
|
MiGetSystemPteAvailability (
|
|
IN ULONG NumberOfPtes,
|
|
IN MM_PAGE_PRIORITY Priority
|
|
);
|
|
|
|
VOID
|
|
MiIssueNoPtesBugcheck (
|
|
IN ULONG NumberOfPtes,
|
|
IN MMSYSTEM_PTE_POOL_TYPE SystemPteType
|
|
);
|
|
|
|
VOID
|
|
MiInitializeSystemPtes (
|
|
IN PMMPTE StartingPte,
|
|
IN ULONG NumberOfPtes,
|
|
IN MMSYSTEM_PTE_POOL_TYPE SystemPteType
|
|
);
|
|
|
|
NTSTATUS
|
|
MiAddMappedPtes (
|
|
IN PMMPTE FirstPte,
|
|
IN ULONG NumberOfPtes,
|
|
IN PCONTROL_AREA ControlArea
|
|
);
|
|
|
|
VOID
|
|
MiInitializeIoTrackers (
|
|
VOID
|
|
);
|
|
|
|
PVOID
|
|
MiMapSinglePage (
|
|
IN PVOID VirtualAddress OPTIONAL,
|
|
IN PFN_NUMBER PageFrameIndex,
|
|
IN MEMORY_CACHING_TYPE CacheType,
|
|
IN MM_PAGE_PRIORITY Priority
|
|
);
|
|
|
|
VOID
|
|
MiUnmapSinglePage (
|
|
IN PVOID BaseAddress
|
|
);
|
|
|
|
typedef struct _MM_PTE_MAPPING {
|
|
LIST_ENTRY ListEntry;
|
|
PVOID SystemVa;
|
|
PVOID SystemEndVa;
|
|
ULONG Protection;
|
|
} MM_PTE_MAPPING, *PMM_PTE_MAPPING;
|
|
|
|
extern LIST_ENTRY MmProtectedPteList;
|
|
|
|
extern KSPIN_LOCK MmProtectedPteLock;
|
|
|
|
LOGICAL
|
|
MiCheckSystemPteProtection (
|
|
IN ULONG_PTR StoreInstruction,
|
|
IN PVOID VirtualAddress
|
|
);
|
|
|
|
//
|
|
// Access Fault routines.
|
|
//
|
|
|
|
#define STATUS_ISSUE_PAGING_IO (0xC0033333)
|
|
|
|
NTSTATUS
|
|
MiDispatchFault (
|
|
IN ULONG_PTR FaultStatus,
|
|
IN PVOID VirtualAdress,
|
|
IN PMMPTE PointerPte,
|
|
IN PMMPTE PointerProtoPte,
|
|
IN PEPROCESS Process,
|
|
OUT PLOGICAL ApcNeeded
|
|
);
|
|
|
|
NTSTATUS
|
|
MiResolveDemandZeroFault (
|
|
IN PVOID VirtualAddress,
|
|
IN PMMPTE PointerPte,
|
|
IN PEPROCESS Process,
|
|
IN ULONG PrototypePte
|
|
);
|
|
|
|
NTSTATUS
|
|
MiResolveTransitionFault (
|
|
IN PVOID FaultingAddress,
|
|
IN PMMPTE PointerPte,
|
|
IN PEPROCESS Process,
|
|
IN ULONG PfnLockHeld,
|
|
OUT PLOGICAL ApcNeeded,
|
|
OUT PMMINPAGE_SUPPORT *InPageBlock
|
|
);
|
|
|
|
NTSTATUS
|
|
MiResolvePageFileFault (
|
|
IN PVOID FaultingAddress,
|
|
IN PMMPTE PointerPte,
|
|
IN PMMINPAGE_SUPPORT *ReadBlock,
|
|
IN PEPROCESS Process
|
|
);
|
|
|
|
NTSTATUS
|
|
MiResolveProtoPteFault (
|
|
IN ULONG_PTR StoreInstruction,
|
|
IN PVOID VirtualAddress,
|
|
IN PMMPTE PointerPte,
|
|
IN PMMPTE PointerProtoPte,
|
|
IN PMMINPAGE_SUPPORT *ReadBlock,
|
|
IN PEPROCESS Process,
|
|
OUT PLOGICAL ApcNeeded
|
|
);
|
|
|
|
|
|
NTSTATUS
|
|
MiResolveMappedFileFault (
|
|
IN PVOID FaultingAddress,
|
|
IN PMMPTE PointerPte,
|
|
IN PMMINPAGE_SUPPORT *ReadBlock,
|
|
IN PEPROCESS Process
|
|
);
|
|
|
|
VOID
|
|
MiAddValidPageToWorkingSet (
|
|
IN PVOID VirtualAddress,
|
|
IN PMMPTE PointerPte,
|
|
IN PMMPFN Pfn1,
|
|
IN ULONG WsleMask
|
|
);
|
|
|
|
NTSTATUS
|
|
MiWaitForInPageComplete (
|
|
IN PMMPFN Pfn,
|
|
IN PMMPTE PointerPte,
|
|
IN PVOID FaultingAddress,
|
|
IN PMMPTE PointerPteContents,
|
|
IN PMMINPAGE_SUPPORT InPageSupport,
|
|
IN PEPROCESS CurrentProcess
|
|
);
|
|
|
|
LOGICAL
|
|
FASTCALL
|
|
MiCopyOnWrite (
|
|
IN PVOID FaultingAddress,
|
|
IN PMMPTE PointerPte
|
|
);
|
|
|
|
VOID
|
|
MiSetDirtyBit (
|
|
IN PVOID FaultingAddress,
|
|
IN PMMPTE PointerPte,
|
|
IN ULONG PfnHeld
|
|
);
|
|
|
|
VOID
|
|
MiSetModifyBit (
|
|
IN PMMPFN Pfn
|
|
);
|
|
|
|
PMMPTE
|
|
MiFindActualFaultingPte (
|
|
IN PVOID FaultingAddress
|
|
);
|
|
|
|
VOID
|
|
MiInitializeReadInProgressSinglePfn (
|
|
IN PFN_NUMBER PageFrameIndex,
|
|
IN PMMPTE BasePte,
|
|
IN PKEVENT Event,
|
|
IN WSLE_NUMBER WorkingSetIndex
|
|
);
|
|
|
|
VOID
|
|
MiInitializeReadInProgressPfn (
|
|
IN PMDL Mdl,
|
|
IN PMMPTE BasePte,
|
|
IN PKEVENT Event,
|
|
IN WSLE_NUMBER WorkingSetIndex
|
|
);
|
|
|
|
NTSTATUS
|
|
MiAccessCheck (
|
|
IN PMMPTE PointerPte,
|
|
IN ULONG_PTR WriteOperation,
|
|
IN KPROCESSOR_MODE PreviousMode,
|
|
IN ULONG Protection,
|
|
IN BOOLEAN CallerHoldsPfnLock
|
|
);
|
|
|
|
NTSTATUS
|
|
FASTCALL
|
|
MiCheckForUserStackOverflow (
|
|
IN PVOID FaultingAddress
|
|
);
|
|
|
|
PMMPTE
|
|
MiCheckVirtualAddress (
|
|
IN PVOID VirtualAddress,
|
|
OUT PULONG ProtectCode
|
|
);
|
|
|
|
NTSTATUS
|
|
FASTCALL
|
|
MiCheckPdeForPagedPool (
|
|
IN PVOID VirtualAddress
|
|
);
|
|
|
|
//
|
|
// Routines which operate on an address tree.
|
|
//
|
|
|
|
PMMADDRESS_NODE
|
|
FASTCALL
|
|
MiGetNextNode (
|
|
IN PMMADDRESS_NODE Node
|
|
);
|
|
|
|
PMMADDRESS_NODE
|
|
FASTCALL
|
|
MiGetPreviousNode (
|
|
IN PMMADDRESS_NODE Node
|
|
);
|
|
|
|
|
|
PMMADDRESS_NODE
|
|
FASTCALL
|
|
MiGetFirstNode (
|
|
IN PMMADDRESS_NODE Root
|
|
);
|
|
|
|
PMMADDRESS_NODE
|
|
MiGetLastNode (
|
|
IN PMMADDRESS_NODE Root
|
|
);
|
|
|
|
VOID
|
|
FASTCALL
|
|
MiInsertNode (
|
|
IN PMMADDRESS_NODE Node,
|
|
IN OUT PMMADDRESS_NODE *Root
|
|
);
|
|
|
|
VOID
|
|
FASTCALL
|
|
MiRemoveNode (
|
|
IN PMMADDRESS_NODE Node,
|
|
IN OUT PMMADDRESS_NODE *Root
|
|
);
|
|
|
|
PMMADDRESS_NODE
|
|
FASTCALL
|
|
MiLocateAddressInTree (
|
|
IN ULONG_PTR Vpn,
|
|
IN PMMADDRESS_NODE *Root
|
|
);
|
|
|
|
PMMADDRESS_NODE
|
|
MiCheckForConflictingNode (
|
|
IN ULONG_PTR StartVpn,
|
|
IN ULONG_PTR EndVpn,
|
|
IN PMMADDRESS_NODE Root
|
|
);
|
|
|
|
NTSTATUS
|
|
MiFindEmptyAddressRangeInTree (
|
|
IN SIZE_T SizeOfRange,
|
|
IN ULONG_PTR Alignment,
|
|
IN PMMADDRESS_NODE Root,
|
|
OUT PMMADDRESS_NODE *PreviousVad,
|
|
OUT PVOID *Base
|
|
);
|
|
|
|
NTSTATUS
|
|
MiFindEmptyAddressRangeDownTree (
|
|
IN SIZE_T SizeOfRange,
|
|
IN PVOID HighestAddressToEndAt,
|
|
IN ULONG_PTR Alignment,
|
|
IN PMMADDRESS_NODE Root,
|
|
OUT PVOID *Base
|
|
);
|
|
|
|
VOID
|
|
NodeTreeWalk (
|
|
PMMADDRESS_NODE Start
|
|
);
|
|
|
|
//
|
|
// Routines which operate on the tree of virtual address descriptors.
|
|
//
|
|
|
|
NTSTATUS
|
|
MiInsertVad (
|
|
IN PMMVAD Vad
|
|
);
|
|
|
|
VOID
|
|
MiRemoveVad (
|
|
IN PMMVAD Vad
|
|
);
|
|
|
|
PMMVAD
|
|
FASTCALL
|
|
MiLocateAddress (
|
|
IN PVOID Vad
|
|
);
|
|
|
|
NTSTATUS
|
|
MiFindEmptyAddressRange (
|
|
IN SIZE_T SizeOfRange,
|
|
IN ULONG_PTR Alignment,
|
|
IN ULONG QuickCheck,
|
|
IN PVOID *Base
|
|
);
|
|
|
|
//
|
|
// Routines which operate on the clone tree structure.
|
|
//
|
|
|
|
|
|
NTSTATUS
|
|
MiCloneProcessAddressSpace (
|
|
IN PEPROCESS ProcessToClone,
|
|
IN PEPROCESS ProcessToInitialize,
|
|
IN PFN_NUMBER PdePhysicalPage,
|
|
IN PFN_NUMBER HyperPhysicalPage
|
|
);
|
|
|
|
|
|
ULONG
|
|
MiDecrementCloneBlockReference (
|
|
IN PMMCLONE_DESCRIPTOR CloneDescriptor,
|
|
IN PMMCLONE_BLOCK CloneBlock,
|
|
IN PEPROCESS CurrentProcess
|
|
);
|
|
|
|
LOGICAL
|
|
MiWaitForForkToComplete (
|
|
IN PEPROCESS CurrentProcess,
|
|
IN LOGICAL PfnHeld
|
|
);
|
|
|
|
//
|
|
// Routines which operate on the working set list.
|
|
//
|
|
|
|
WSLE_NUMBER
|
|
MiLocateAndReserveWsle (
|
|
IN PMMSUPPORT WsInfo
|
|
);
|
|
|
|
VOID
|
|
MiReleaseWsle (
|
|
IN WSLE_NUMBER WorkingSetIndex,
|
|
IN PMMSUPPORT WsInfo
|
|
);
|
|
|
|
VOID
|
|
MiUpdateWsle (
|
|
IN PWSLE_NUMBER DesiredIndex,
|
|
IN PVOID VirtualAddress,
|
|
IN PMMWSL WorkingSetList,
|
|
IN PMMPFN Pfn
|
|
);
|
|
|
|
VOID
|
|
MiInitializeWorkingSetList (
|
|
IN PEPROCESS CurrentProcess
|
|
);
|
|
|
|
VOID
|
|
MiGrowWsleHash (
|
|
IN PMMSUPPORT WsInfo
|
|
);
|
|
|
|
WSLE_NUMBER
|
|
MiTrimWorkingSet (
|
|
IN WSLE_NUMBER Reduction,
|
|
IN PMMSUPPORT WsInfo,
|
|
IN ULONG TrimAge
|
|
);
|
|
|
|
LOGICAL
|
|
MmTrimProcessMemory (
|
|
IN LOGICAL PurgeTransition
|
|
);
|
|
|
|
LOGICAL
|
|
MmTrimSessionMemory (
|
|
IN LOGICAL PurgeTransition
|
|
);
|
|
|
|
VOID
|
|
MiRemoveWorkingSetPages (
|
|
IN PMMWSL WorkingSetList,
|
|
IN PMMSUPPORT WsInfo
|
|
);
|
|
|
|
VOID
|
|
MiAgeAndEstimateAvailableInWorkingSet (
|
|
IN PMMSUPPORT VmSupport,
|
|
IN LOGICAL DoAging,
|
|
IN PWSLE_NUMBER WslesScanned,
|
|
IN OUT PPFN_NUMBER TotalClaim,
|
|
IN OUT PPFN_NUMBER TotalEstimatedAvailable
|
|
);
|
|
|
|
VOID
|
|
FASTCALL
|
|
MiInsertWsleHash (
|
|
IN WSLE_NUMBER Entry,
|
|
IN PMMWSL WorkingSetList
|
|
);
|
|
|
|
VOID
|
|
FASTCALL
|
|
MiRemoveWsle (
|
|
IN WSLE_NUMBER Entry,
|
|
IN PMMWSL WorkingSetList
|
|
);
|
|
|
|
WSLE_NUMBER
|
|
FASTCALL
|
|
MiLocateWsle (
|
|
IN PVOID VirtualAddress,
|
|
IN PMMWSL WorkingSetList,
|
|
IN WSLE_NUMBER WsPfnIndex
|
|
);
|
|
|
|
ULONG
|
|
MiFreeWsle (
|
|
IN WSLE_NUMBER WorkingSetIndex,
|
|
IN PMMSUPPORT WsInfo,
|
|
IN PMMPTE PointerPte
|
|
);
|
|
|
|
VOID
|
|
MiSwapWslEntries (
|
|
IN WSLE_NUMBER SwapEntry,
|
|
IN WSLE_NUMBER Entry,
|
|
IN PMMSUPPORT WsInfo
|
|
);
|
|
|
|
VOID
|
|
MiRemoveWsleFromFreeList (
|
|
IN WSLE_NUMBER Entry,
|
|
IN PMMWSLE Wsle,
|
|
IN PMMWSL WorkingSetList
|
|
);
|
|
|
|
ULONG
|
|
MiRemovePageFromWorkingSet (
|
|
IN PMMPTE PointerPte,
|
|
IN PMMPFN Pfn1,
|
|
IN PMMSUPPORT WsInfo
|
|
);
|
|
|
|
PFN_NUMBER
|
|
MiDeleteSystemPagableVm (
|
|
IN PMMPTE PointerPte,
|
|
IN PFN_NUMBER NumberOfPtes,
|
|
IN MMPTE NewPteValue,
|
|
IN LOGICAL SessionAllocation,
|
|
OUT PPFN_NUMBER ResidentPages OPTIONAL
|
|
);
|
|
|
|
VOID
|
|
MiLockCode (
|
|
IN PMMPTE FirstPte,
|
|
IN PMMPTE LastPte,
|
|
IN ULONG LockType
|
|
);
|
|
|
|
PKLDR_DATA_TABLE_ENTRY
|
|
MiLookupDataTableEntry (
|
|
IN PVOID AddressWithinSection,
|
|
IN ULONG ResourceHeld
|
|
);
|
|
|
|
//
|
|
// Routines which perform working set management.
|
|
//
|
|
|
|
VOID
|
|
MiObtainFreePages (
|
|
VOID
|
|
);
|
|
|
|
VOID
|
|
MiModifiedPageWriter (
|
|
IN PVOID StartContext
|
|
);
|
|
|
|
VOID
|
|
MiMappedPageWriter (
|
|
IN PVOID StartContext
|
|
);
|
|
|
|
LOGICAL
|
|
MiIssuePageExtendRequest (
|
|
IN PMMPAGE_FILE_EXPANSION PageExtend
|
|
);
|
|
|
|
VOID
|
|
MiIssuePageExtendRequestNoWait (
|
|
IN PFN_NUMBER SizeInPages
|
|
);
|
|
|
|
SIZE_T
|
|
MiExtendPagingFiles (
|
|
IN PMMPAGE_FILE_EXPANSION PageExpand
|
|
);
|
|
|
|
VOID
|
|
MiContractPagingFiles (
|
|
VOID
|
|
);
|
|
|
|
VOID
|
|
MiAttemptPageFileReduction (
|
|
VOID
|
|
);
|
|
|
|
LOGICAL
|
|
MiCancelWriteOfMappedPfn (
|
|
IN PFN_NUMBER PageToStop
|
|
);
|
|
|
|
//
|
|
// Routines to delete address space.
|
|
//
|
|
|
|
VOID
|
|
MiDeletePteRange (
|
|
IN PEPROCESS Process,
|
|
IN PMMPTE PointerPte,
|
|
IN PMMPTE LastPte,
|
|
IN LOGICAL AddressSpaceDeletion
|
|
);
|
|
|
|
VOID
|
|
MiDeleteVirtualAddresses (
|
|
IN PUCHAR StartingAddress,
|
|
IN PUCHAR EndingAddress,
|
|
IN ULONG AddressSpaceDeletion,
|
|
IN PMMVAD Vad
|
|
);
|
|
|
|
ULONG
|
|
MiDeletePte (
|
|
IN PMMPTE PointerPte,
|
|
IN PVOID VirtualAddress,
|
|
IN ULONG AddressSpaceDeletion,
|
|
IN PEPROCESS CurrentProcess,
|
|
IN PMMPTE PrototypePte,
|
|
IN PMMPTE_FLUSH_LIST PteFlushList OPTIONAL
|
|
);
|
|
|
|
VOID
|
|
MiDeletePageTablesForPhysicalRange (
|
|
IN PVOID StartingAddress,
|
|
IN PVOID EndingAddress
|
|
);
|
|
|
|
VOID
|
|
MiFlushPteList (
|
|
IN PMMPTE_FLUSH_LIST PteFlushList,
|
|
IN ULONG AllProcessors,
|
|
IN MMPTE FillPte
|
|
);
|
|
|
|
ULONG
|
|
FASTCALL
|
|
MiReleasePageFileSpace (
|
|
IN MMPTE PteContents
|
|
);
|
|
|
|
VOID
|
|
FASTCALL
|
|
MiReleaseConfirmedPageFileSpace (
|
|
IN MMPTE PteContents
|
|
);
|
|
|
|
VOID
|
|
FASTCALL
|
|
MiUpdateModifiedWriterMdls (
|
|
IN ULONG PageFileNumber
|
|
);
|
|
|
|
PVOID
|
|
MiAllocateAweInfo (
|
|
VOID
|
|
);
|
|
|
|
VOID
|
|
MiRemoveUserPhysicalPagesVad (
|
|
IN PMMVAD_SHORT FoundVad
|
|
);
|
|
|
|
VOID
|
|
MiCleanPhysicalProcessPages (
|
|
IN PEPROCESS Process
|
|
);
|
|
|
|
VOID
|
|
MiPhysicalViewRemover (
|
|
IN PEPROCESS Process,
|
|
IN PMMVAD Vad
|
|
);
|
|
|
|
VOID
|
|
MiPhysicalViewAdjuster (
|
|
IN PEPROCESS Process,
|
|
IN PMMVAD OldVad,
|
|
IN PMMVAD NewVad
|
|
);
|
|
|
|
LOGICAL
|
|
MiIsPhysicalMemoryAddress (
|
|
IN PFN_NUMBER PageFrameIndex,
|
|
IN OUT PULONG Hint,
|
|
IN LOGICAL PfnLockNeeded
|
|
);
|
|
|
|
//
|
|
// MM_SYSTEM_PAGE_COLOR - MmSystemPageColor
|
|
//
|
|
// This variable is updated frequently, on MP systems we keep
|
|
// a separate system color per processor to avoid cache line
|
|
// thrashing.
|
|
//
|
|
|
|
#if defined(NT_UP)
|
|
|
|
#define MI_SYSTEM_PAGE_COLOR MmSystemPageColor
|
|
|
|
#else
|
|
|
|
#define MI_SYSTEM_PAGE_COLOR (KeGetCurrentPrcb()->PageColor)
|
|
|
|
#endif
|
|
|
|
#if defined(MI_MULTINODE)
|
|
|
|
extern PKNODE KeNodeBlock[];
|
|
|
|
#define MI_NODE_FROM_COLOR(c) \
|
|
(KeNodeBlock[(c) >> MmSecondaryColorNodeShift])
|
|
|
|
#define MI_GET_COLOR_FROM_LIST_ENTRY(index,pfn) \
|
|
((ULONG)(((pfn)->u3.e1.PageColor << MmSecondaryColorNodeShift) | \
|
|
MI_GET_SECONDARY_COLOR((index),(pfn))))
|
|
|
|
#define MI_ADJUST_COLOR_FOR_NODE(c,n) ((c) | (n)->Color)
|
|
#define MI_CURRENT_NODE_COLOR (KeGetCurrentNode()->MmShiftedColor)
|
|
|
|
#define MiRemoveZeroPageIfAny(c) \
|
|
(KeGetCurrentNode()->FreeCount[ZeroedPageList] ? MiRemoveZeroPage(c) : 0)
|
|
|
|
#define MI_GET_PAGE_COLOR_NODE(n) \
|
|
(((MI_SYSTEM_PAGE_COLOR++) & MmSecondaryColorMask) | \
|
|
KeNodeBlock[n]->MmShiftedColor)
|
|
|
|
#else
|
|
|
|
#define MI_NODE_FROM_COLOR(c)
|
|
|
|
#define MI_GET_COLOR_FROM_LIST_ENTRY(index,pfn) \
|
|
((ULONG)MI_GET_SECONDARY_COLOR((index),(pfn)))
|
|
|
|
#define MI_ADJUST_COLOR_FOR_NODE(c,n) (c)
|
|
#define MI_CURRENT_NODE_COLOR 0
|
|
|
|
#define MiRemoveZeroPageIfAny(COLOR) \
|
|
(MmFreePagesByColor[ZeroedPageList][COLOR].Flink != MM_EMPTY_LIST) ? \
|
|
MiRemoveZeroPage(COLOR) : 0
|
|
|
|
#define MI_GET_PAGE_COLOR_NODE(n) \
|
|
((MI_SYSTEM_PAGE_COLOR++) & MmSecondaryColorMask)
|
|
|
|
#endif
|
|
|
|
FORCEINLINE
|
|
PFN_NUMBER
|
|
MiRemoveZeroPageMayReleaseLocks (
|
|
IN ULONG Color,
|
|
IN KIRQL OldIrql
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine returns a zeroed page.
|
|
|
|
It may release and reacquire the PFN lock to do so, as well as mapping
|
|
the page in hyperspace to perform the actual zeroing if necessary.
|
|
|
|
Environment:
|
|
|
|
Kernel mode. PFN lock held, hyperspace lock NOT held.
|
|
|
|
--*/
|
|
|
|
{
|
|
PFN_NUMBER PageFrameIndex;
|
|
|
|
PageFrameIndex = MiRemoveZeroPageIfAny (Color);
|
|
|
|
if (PageFrameIndex == 0) {
|
|
PageFrameIndex = MiRemoveAnyPage (Color);
|
|
UNLOCK_PFN (OldIrql);
|
|
MiZeroPhysicalPage (PageFrameIndex, Color);
|
|
LOCK_PFN (OldIrql);
|
|
}
|
|
|
|
return PageFrameIndex;
|
|
}
|
|
|
|
//
|
|
// General support routines.
|
|
//
|
|
|
|
#if (_MI_PAGING_LEVELS <= 3)
|
|
|
|
//++
|
|
//PMMPTE
|
|
//MiGetPxeAddress (
|
|
// IN PVOID va
|
|
// );
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// MiGetPxeAddress returns the address of the extended page directory parent
|
|
// entry which maps the given virtual address. This is one level above the
|
|
// page parent directory.
|
|
//
|
|
// Arguments
|
|
//
|
|
// Va - Supplies the virtual address to locate the PXE for.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// The address of the PXE.
|
|
//
|
|
//--
|
|
|
|
#define MiGetPxeAddress(va) ((PMMPTE)0)
|
|
|
|
//++
|
|
//LOGICAL
|
|
//MiIsPteOnPxeBoundary (
|
|
// IN PVOID PTE
|
|
// );
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// MiIsPteOnPxeBoundary returns TRUE if the PTE is
|
|
// on an extended page directory parent entry boundary.
|
|
//
|
|
// Arguments
|
|
//
|
|
// PTE - Supplies the PTE to check.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// TRUE if on a boundary, FALSE if not.
|
|
//
|
|
//--
|
|
|
|
#define MiIsPteOnPxeBoundary(PTE) (FALSE)
|
|
|
|
#endif
|
|
|
|
#if (_MI_PAGING_LEVELS <= 2)
|
|
|
|
//++
|
|
//PMMPTE
|
|
//MiGetPpeAddress (
|
|
// IN PVOID va
|
|
// );
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// MiGetPpeAddress returns the address of the page directory parent entry
|
|
// which maps the given virtual address. This is one level above the
|
|
// page directory.
|
|
//
|
|
// Arguments
|
|
//
|
|
// Va - Supplies the virtual address to locate the PPE for.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// The address of the PPE.
|
|
//
|
|
//--
|
|
|
|
#define MiGetPpeAddress(va) ((PMMPTE)0)
|
|
|
|
//++
|
|
//LOGICAL
|
|
//MiIsPteOnPpeBoundary (
|
|
// IN PVOID VA
|
|
// );
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// MiIsPteOnPpeBoundary returns TRUE if the PTE is
|
|
// on a page directory parent entry boundary.
|
|
//
|
|
// Arguments
|
|
//
|
|
// VA - Supplies the virtual address to check.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// TRUE if on a boundary, FALSE if not.
|
|
//
|
|
//--
|
|
|
|
#define MiIsPteOnPpeBoundary(PTE) (FALSE)
|
|
|
|
#endif
|
|
|
|
ULONG
|
|
MiDoesPdeExistAndMakeValid (
|
|
IN PMMPTE PointerPde,
|
|
IN PEPROCESS TargetProcess,
|
|
IN LOGICAL PfnLockHeld,
|
|
OUT PULONG Waited
|
|
);
|
|
|
|
#if (_MI_PAGING_LEVELS >= 3)
|
|
#define MiDoesPpeExistAndMakeValid(PPE, PROCESS, PFNLOCKHELD, WAITED) \
|
|
MiDoesPdeExistAndMakeValid(PPE, PROCESS, PFNLOCKHELD, WAITED)
|
|
#else
|
|
#define MiDoesPpeExistAndMakeValid(PPE, PROCESS, PFNLOCKHELD, WAITED) 1
|
|
#endif
|
|
|
|
#if (_MI_PAGING_LEVELS >= 4)
|
|
#define MiDoesPxeExistAndMakeValid(PXE, PROCESS, PFNLOCKHELD, WAITED) \
|
|
MiDoesPdeExistAndMakeValid(PXE, PROCESS, PFNLOCKHELD, WAITED)
|
|
#else
|
|
#define MiDoesPxeExistAndMakeValid(PXE, PROCESS, PFNLOCKHELD, WAITED) 1
|
|
#endif
|
|
|
|
VOID
|
|
MiMakePdeExistAndMakeValid (
|
|
IN PMMPTE PointerPde,
|
|
IN PEPROCESS TargetProcess,
|
|
IN LOGICAL PfnLockHeld
|
|
);
|
|
|
|
#if (_MI_PAGING_LEVELS >= 4)
|
|
VOID
|
|
MiMakePxeExistAndMakeValid (
|
|
IN PMMPTE PointerPpe,
|
|
IN PEPROCESS TargetProcess,
|
|
IN LOGICAL PfnLockHeld
|
|
);
|
|
#else
|
|
#define MiMakePxeExistAndMakeValid(PDE, PROCESS, PFNLOCKHELD)
|
|
#endif
|
|
|
|
#if (_MI_PAGING_LEVELS >= 3)
|
|
VOID
|
|
MiMakePpeExistAndMakeValid (
|
|
IN PMMPTE PointerPpe,
|
|
IN PEPROCESS TargetProcess,
|
|
IN LOGICAL PfnLockHeld
|
|
);
|
|
#else
|
|
#define MiMakePpeExistAndMakeValid(PDE, PROCESS, PFNLOCKHELD)
|
|
#endif
|
|
|
|
ULONG
|
|
FASTCALL
|
|
MiMakeSystemAddressValid (
|
|
IN PVOID VirtualAddress,
|
|
IN PEPROCESS CurrentProcess
|
|
);
|
|
|
|
ULONG
|
|
FASTCALL
|
|
MiMakeSystemAddressValidPfnWs (
|
|
IN PVOID VirtualAddress,
|
|
IN PEPROCESS CurrentProcess OPTIONAL
|
|
);
|
|
|
|
ULONG
|
|
FASTCALL
|
|
MiMakeSystemAddressValidPfnSystemWs (
|
|
IN PVOID VirtualAddress
|
|
);
|
|
|
|
ULONG
|
|
FASTCALL
|
|
MiMakeSystemAddressValidPfn (
|
|
IN PVOID VirtualAddress
|
|
);
|
|
|
|
ULONG
|
|
FASTCALL
|
|
MiLockPagedAddress (
|
|
IN PVOID VirtualAddress,
|
|
IN ULONG PfnLockHeld
|
|
);
|
|
|
|
VOID
|
|
FASTCALL
|
|
MiUnlockPagedAddress (
|
|
IN PVOID VirtualAddress,
|
|
IN ULONG PfnLockHeld
|
|
);
|
|
|
|
ULONG
|
|
FASTCALL
|
|
MiIsPteDecommittedPage (
|
|
IN PMMPTE PointerPte
|
|
);
|
|
|
|
ULONG
|
|
FASTCALL
|
|
MiIsProtectionCompatible (
|
|
IN ULONG OldProtect,
|
|
IN ULONG NewProtect
|
|
);
|
|
|
|
ULONG
|
|
FASTCALL
|
|
MiIsPteProtectionCompatible (
|
|
IN ULONG OldPteProtection,
|
|
IN ULONG NewProtect
|
|
);
|
|
|
|
ULONG
|
|
FASTCALL
|
|
MiMakeProtectionMask (
|
|
IN ULONG Protect
|
|
);
|
|
|
|
ULONG
|
|
MiIsEntireRangeCommitted (
|
|
IN PVOID StartingAddress,
|
|
IN PVOID EndingAddress,
|
|
IN PMMVAD Vad,
|
|
IN PEPROCESS Process
|
|
);
|
|
|
|
ULONG
|
|
MiIsEntireRangeDecommitted (
|
|
IN PVOID StartingAddress,
|
|
IN PVOID EndingAddress,
|
|
IN PMMVAD Vad,
|
|
IN PEPROCESS Process
|
|
);
|
|
|
|
LOGICAL
|
|
MiCheckProtoPtePageState (
|
|
IN PMMPTE PrototypePte,
|
|
IN LOGICAL PfnLockHeld,
|
|
OUT PLOGICAL DroppedPfnLock
|
|
);
|
|
|
|
//++
|
|
//PMMPTE
|
|
//MiGetProtoPteAddress (
|
|
// IN PMMPTE VAD,
|
|
// IN PVOID VA
|
|
// );
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// MiGetProtoPteAddress returns a pointer to the prototype PTE which
|
|
// is mapped by the given virtual address descriptor and address within
|
|
// the virtual address descriptor.
|
|
//
|
|
// Arguments
|
|
//
|
|
// VAD - Supplies a pointer to the virtual address descriptor that contains
|
|
// the VA.
|
|
//
|
|
// VPN - Supplies the virtual page number.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// A pointer to the proto PTE which corresponds to the VA.
|
|
//
|
|
//--
|
|
|
|
|
|
#define MiGetProtoPteAddress(VAD,VPN) \
|
|
((((((VPN) - (VAD)->StartingVpn) << PTE_SHIFT) + \
|
|
(ULONG_PTR)(VAD)->FirstPrototypePte) <= (ULONG_PTR)(VAD)->LastContiguousPte) ? \
|
|
((PMMPTE)(((((VPN) - (VAD)->StartingVpn) << PTE_SHIFT) + \
|
|
(ULONG_PTR)(VAD)->FirstPrototypePte))) : \
|
|
MiGetProtoPteAddressExtended ((VAD),(VPN)))
|
|
|
|
PMMPTE
|
|
FASTCALL
|
|
MiGetProtoPteAddressExtended (
|
|
IN PMMVAD Vad,
|
|
IN ULONG_PTR Vpn
|
|
);
|
|
|
|
PSUBSECTION
|
|
FASTCALL
|
|
MiLocateSubsection (
|
|
IN PMMVAD Vad,
|
|
IN ULONG_PTR Vpn
|
|
);
|
|
|
|
VOID
|
|
MiInitializeSystemCache (
|
|
IN ULONG MinimumWorkingSet,
|
|
IN ULONG MaximumWorkingSet
|
|
);
|
|
|
|
VOID
|
|
MiAdjustWorkingSetManagerParameters(
|
|
IN LOGICAL WorkStation
|
|
);
|
|
|
|
VOID
|
|
MiNotifyMemoryEvents (
|
|
VOID
|
|
);
|
|
|
|
extern PFN_NUMBER MmLowMemoryThreshold;
|
|
extern PFN_NUMBER MmHighMemoryThreshold;
|
|
|
|
//
|
|
// Section support
|
|
//
|
|
|
|
VOID
|
|
FASTCALL
|
|
MiInsertBasedSection (
|
|
IN PSECTION Section
|
|
);
|
|
|
|
NTSTATUS
|
|
MiMapViewOfPhysicalSection (
|
|
IN PCONTROL_AREA ControlArea,
|
|
IN PEPROCESS Process,
|
|
IN PVOID *CapturedBase,
|
|
IN PLARGE_INTEGER SectionOffset,
|
|
IN PSIZE_T CapturedViewSize,
|
|
IN ULONG ProtectionMask,
|
|
IN ULONG_PTR ZeroBits,
|
|
IN ULONG AllocationType,
|
|
IN LOGICAL WriteCombined
|
|
);
|
|
|
|
NTSTATUS
|
|
MiMapViewOfDataSection (
|
|
IN PCONTROL_AREA ControlArea,
|
|
IN PEPROCESS Process,
|
|
IN PVOID *CapturedBase,
|
|
IN PLARGE_INTEGER SectionOffset,
|
|
IN PSIZE_T CapturedViewSize,
|
|
IN PSECTION Section,
|
|
IN SECTION_INHERIT InheritDisposition,
|
|
IN ULONG ProtectionMask,
|
|
IN SIZE_T CommitSize,
|
|
IN ULONG_PTR ZeroBits,
|
|
IN ULONG AllocationType
|
|
);
|
|
|
|
NTSTATUS
|
|
MiUnmapViewOfSection (
|
|
IN PEPROCESS Process,
|
|
IN PVOID BaseAddress,
|
|
IN LOGICAL AddressSpaceMutexHeld
|
|
);
|
|
|
|
VOID
|
|
MiRemoveImageSectionObject(
|
|
IN PFILE_OBJECT File,
|
|
IN PCONTROL_AREA ControlArea
|
|
);
|
|
|
|
VOID
|
|
MiAddSystemPtes(
|
|
IN PMMPTE StartingPte,
|
|
IN ULONG NumberOfPtes,
|
|
IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType
|
|
);
|
|
|
|
VOID
|
|
MiRemoveMappedView (
|
|
IN PEPROCESS CurrentProcess,
|
|
IN PMMVAD Vad
|
|
);
|
|
|
|
VOID
|
|
MiSegmentDelete (
|
|
PSEGMENT Segment
|
|
);
|
|
|
|
VOID
|
|
MiSectionDelete (
|
|
IN PVOID Object
|
|
);
|
|
|
|
VOID
|
|
MiDereferenceSegmentThread (
|
|
IN PVOID StartContext
|
|
);
|
|
|
|
NTSTATUS
|
|
MiCreateImageFileMap (
|
|
IN PFILE_OBJECT File,
|
|
OUT PSEGMENT *Segment
|
|
);
|
|
|
|
NTSTATUS
|
|
MiCreateDataFileMap (
|
|
IN PFILE_OBJECT File,
|
|
OUT PSEGMENT *Segment,
|
|
IN PUINT64 MaximumSize,
|
|
IN ULONG SectionPageProtection,
|
|
IN ULONG AllocationAttributes,
|
|
IN ULONG IgnoreFileSizing
|
|
);
|
|
|
|
NTSTATUS
|
|
MiCreatePagingFileMap (
|
|
OUT PSEGMENT *Segment,
|
|
IN PUINT64 MaximumSize,
|
|
IN ULONG ProtectionMask,
|
|
IN ULONG AllocationAttributes
|
|
);
|
|
|
|
VOID
|
|
MiPurgeSubsectionInternal (
|
|
IN PSUBSECTION Subsection,
|
|
IN ULONG PteOffset
|
|
);
|
|
|
|
VOID
|
|
MiPurgeImageSection (
|
|
IN PCONTROL_AREA ControlArea,
|
|
IN PEPROCESS Process
|
|
);
|
|
|
|
VOID
|
|
MiCleanSection (
|
|
IN PCONTROL_AREA ControlArea,
|
|
IN LOGICAL DirtyDataPagesOk
|
|
);
|
|
|
|
VOID
|
|
MiDereferenceControlArea (
|
|
IN PCONTROL_AREA ControlArea
|
|
);
|
|
|
|
VOID
|
|
MiCheckControlArea (
|
|
IN PCONTROL_AREA ControlArea,
|
|
IN PEPROCESS CurrentProcess,
|
|
IN KIRQL PreviousIrql
|
|
);
|
|
|
|
LOGICAL
|
|
MiCheckPurgeAndUpMapCount (
|
|
IN PCONTROL_AREA ControlArea
|
|
);
|
|
|
|
VOID
|
|
MiCheckForControlAreaDeletion (
|
|
IN PCONTROL_AREA ControlArea
|
|
);
|
|
|
|
LOGICAL
|
|
MiCheckControlAreaStatus (
|
|
IN SECTION_CHECK_TYPE SectionCheckType,
|
|
IN PSECTION_OBJECT_POINTERS SectionObjectPointers,
|
|
IN ULONG DelayClose,
|
|
OUT PCONTROL_AREA *ControlArea,
|
|
OUT PKIRQL OldIrql
|
|
);
|
|
|
|
extern SLIST_HEADER MmEventCountSListHead;
|
|
|
|
PEVENT_COUNTER
|
|
MiGetEventCounter (
|
|
VOID
|
|
);
|
|
|
|
VOID
|
|
MiFreeEventCounter (
|
|
IN PEVENT_COUNTER Support
|
|
);
|
|
|
|
ULONG
|
|
MiCanFileBeTruncatedInternal (
|
|
IN PSECTION_OBJECT_POINTERS SectionPointer,
|
|
IN PLARGE_INTEGER NewFileSize OPTIONAL,
|
|
IN LOGICAL BlockNewViews,
|
|
OUT PKIRQL PreviousIrql
|
|
);
|
|
|
|
#define STATUS_MAPPED_WRITER_COLLISION (0xC0033333)
|
|
|
|
NTSTATUS
|
|
MiFlushSectionInternal (
|
|
IN PMMPTE StartingPte,
|
|
IN PMMPTE FinalPte,
|
|
IN PSUBSECTION FirstSubsection,
|
|
IN PSUBSECTION LastSubsection,
|
|
IN ULONG Synchronize,
|
|
IN LOGICAL WriteInProgressOk,
|
|
OUT PIO_STATUS_BLOCK IoStatus
|
|
);
|
|
|
|
//
|
|
// protection stuff...
|
|
//
|
|
|
|
NTSTATUS
|
|
MiProtectVirtualMemory (
|
|
IN PEPROCESS Process,
|
|
IN PVOID *CapturedBase,
|
|
IN PSIZE_T CapturedRegionSize,
|
|
IN ULONG Protect,
|
|
IN PULONG LastProtect
|
|
);
|
|
|
|
ULONG
|
|
MiGetPageProtection (
|
|
IN PMMPTE PointerPte,
|
|
IN PEPROCESS Process,
|
|
IN LOGICAL PteCapturedToLocalStack
|
|
);
|
|
|
|
NTSTATUS
|
|
MiSetProtectionOnSection (
|
|
IN PEPROCESS Process,
|
|
IN PMMVAD Vad,
|
|
IN PVOID StartingAddress,
|
|
IN PVOID EndingAddress,
|
|
IN ULONG NewProtect,
|
|
OUT PULONG CapturedOldProtect,
|
|
IN ULONG DontCharge,
|
|
OUT PULONG Locked
|
|
);
|
|
|
|
NTSTATUS
|
|
MiCheckSecuredVad (
|
|
IN PMMVAD Vad,
|
|
IN PVOID Base,
|
|
IN ULONG_PTR Size,
|
|
IN ULONG ProtectionMask
|
|
);
|
|
|
|
HANDLE
|
|
MiSecureVirtualMemory (
|
|
IN PVOID Address,
|
|
IN SIZE_T Size,
|
|
IN ULONG ProbeMode,
|
|
IN LOGICAL AddressSpaceMutexHeld
|
|
);
|
|
|
|
VOID
|
|
MiUnsecureVirtualMemory (
|
|
IN HANDLE SecureHandle,
|
|
IN LOGICAL AddressSpaceMutexHeld
|
|
);
|
|
|
|
ULONG
|
|
MiChangeNoAccessForkPte (
|
|
IN PMMPTE PointerPte,
|
|
IN ULONG ProtectionMask
|
|
);
|
|
|
|
VOID
|
|
MiSetImageProtect (
|
|
IN PSEGMENT Segment,
|
|
IN ULONG Protection
|
|
);
|
|
|
|
//
|
|
// Routines for charging quota and commitment.
|
|
//
|
|
|
|
VOID
|
|
MiTrimSegmentCache (
|
|
VOID
|
|
);
|
|
|
|
VOID
|
|
MiInitializeCommitment (
|
|
VOID
|
|
);
|
|
|
|
LOGICAL
|
|
FASTCALL
|
|
MiChargeCommitment (
|
|
IN SIZE_T QuotaCharge,
|
|
IN PEPROCESS Process OPTIONAL
|
|
);
|
|
|
|
LOGICAL
|
|
FASTCALL
|
|
MiChargeCommitmentCantExpand (
|
|
IN SIZE_T QuotaCharge,
|
|
IN ULONG MustSucceed
|
|
);
|
|
|
|
LOGICAL
|
|
FASTCALL
|
|
MiChargeTemporaryCommitmentForReduction (
|
|
IN SIZE_T QuotaCharge
|
|
);
|
|
|
|
#if defined (_MI_DEBUG_COMMIT_LEAKS)
|
|
|
|
VOID
|
|
FASTCALL
|
|
MiReturnCommitment (
|
|
IN SIZE_T QuotaCharge
|
|
);
|
|
|
|
#else
|
|
|
|
#define MiReturnCommitment(_QuotaCharge) \
|
|
ASSERT ((SSIZE_T)(_QuotaCharge) >= 0); \
|
|
ASSERT (MmTotalCommittedPages >= (_QuotaCharge)); \
|
|
InterlockedExchangeAddSizeT (&MmTotalCommittedPages, 0-((SIZE_T)(_QuotaCharge))); \
|
|
MM_TRACK_COMMIT (MM_DBG_COMMIT_RETURN_NORMAL, (_QuotaCharge));
|
|
|
|
#endif
|
|
|
|
VOID
|
|
MiCauseOverCommitPopup (
|
|
VOID
|
|
);
|
|
|
|
extern SIZE_T MmPeakCommitment;
|
|
|
|
extern SIZE_T MmTotalCommitLimitMaximum;
|
|
|
|
SIZE_T
|
|
MiCalculatePageCommitment (
|
|
IN PVOID StartingAddress,
|
|
IN PVOID EndingAddress,
|
|
IN PMMVAD Vad,
|
|
IN PEPROCESS Process
|
|
);
|
|
|
|
VOID
|
|
MiReturnPageTablePageCommitment (
|
|
IN PVOID StartingAddress,
|
|
IN PVOID EndingAddress,
|
|
IN PEPROCESS CurrentProcess,
|
|
IN PMMVAD PreviousVad,
|
|
IN PMMVAD NextVad
|
|
);
|
|
|
|
VOID
|
|
MiFlushAllPages (
|
|
VOID
|
|
);
|
|
|
|
VOID
|
|
MiModifiedPageWriterTimerDispatch (
|
|
IN PKDPC Dpc,
|
|
IN PVOID DeferredContext,
|
|
IN PVOID SystemArgument1,
|
|
IN PVOID SystemArgument2
|
|
);
|
|
|
|
LONGLONG
|
|
MiStartingOffset(
|
|
IN PSUBSECTION Subsection,
|
|
IN PMMPTE PteAddress
|
|
);
|
|
|
|
LARGE_INTEGER
|
|
MiEndingOffset(
|
|
IN PSUBSECTION Subsection
|
|
);
|
|
|
|
VOID
|
|
MiReloadBootLoadedDrivers (
|
|
IN PLOADER_PARAMETER_BLOCK LoaderBlock
|
|
);
|
|
|
|
LOGICAL
|
|
MiInitializeLoadedModuleList (
|
|
IN PLOADER_PARAMETER_BLOCK LoaderBlock
|
|
);
|
|
|
|
extern ULONG MmSpecialPoolTag;
|
|
extern PVOID MmSpecialPoolStart;
|
|
extern PVOID MmSpecialPoolEnd;
|
|
extern PVOID MmSessionSpecialPoolStart;
|
|
extern PVOID MmSessionSpecialPoolEnd;
|
|
|
|
LOGICAL
|
|
MiInitializeSpecialPool (
|
|
IN POOL_TYPE PoolType
|
|
);
|
|
|
|
LOGICAL
|
|
MiIsSpecialPoolAddressNonPaged (
|
|
IN PVOID VirtualAddress
|
|
);
|
|
|
|
#if defined (_WIN64)
|
|
LOGICAL
|
|
MiInitializeSessionSpecialPool (
|
|
VOID
|
|
);
|
|
|
|
VOID
|
|
MiDeleteSessionSpecialPool (
|
|
VOID
|
|
);
|
|
#endif
|
|
|
|
#if defined (_X86_)
|
|
LOGICAL
|
|
MiRecoverSpecialPtes (
|
|
IN ULONG NumberOfPtes
|
|
);
|
|
#endif
|
|
|
|
VOID
|
|
MiEnableRandomSpecialPool (
|
|
IN LOGICAL Enable
|
|
);
|
|
|
|
LOGICAL
|
|
MiTriageSystem (
|
|
IN PLOADER_PARAMETER_BLOCK LoaderBlock
|
|
);
|
|
|
|
LOGICAL
|
|
MiTriageAddDrivers (
|
|
IN PLOADER_PARAMETER_BLOCK LoaderBlock
|
|
);
|
|
|
|
LOGICAL
|
|
MiTriageVerifyDriver (
|
|
IN PKLDR_DATA_TABLE_ENTRY DataTableEntry
|
|
);
|
|
|
|
extern ULONG MmTriageActionTaken;
|
|
|
|
#if defined (_WIN64)
|
|
#define MM_SPECIAL_POOL_PTES ((1024 * 1024) / sizeof (MMPTE))
|
|
#else
|
|
#define MM_SPECIAL_POOL_PTES (24 * PTE_PER_PAGE)
|
|
#endif
|
|
|
|
#define MI_SUSPECT_DRIVER_BUFFER_LENGTH 512
|
|
|
|
extern WCHAR MmVerifyDriverBuffer[];
|
|
extern ULONG MmVerifyDriverBufferLength;
|
|
extern ULONG MmVerifyDriverLevel;
|
|
|
|
extern LOGICAL MmDontVerifyRandomDrivers;
|
|
extern LOGICAL MmSnapUnloads;
|
|
extern LOGICAL MmProtectFreedNonPagedPool;
|
|
extern ULONG MmEnforceWriteProtection;
|
|
extern LOGICAL MmTrackLockedPages;
|
|
extern ULONG MmTrackPtes;
|
|
|
|
#define VI_POOL_FREELIST_END ((ULONG_PTR)-1)
|
|
|
|
typedef struct _VI_POOL_ENTRY_INUSE {
|
|
PVOID VirtualAddress;
|
|
PVOID CallingAddress;
|
|
SIZE_T NumberOfBytes;
|
|
ULONG_PTR Tag;
|
|
} VI_POOL_ENTRY_INUSE, *PVI_POOL_ENTRY_INUSE;
|
|
|
|
typedef struct _VI_POOL_ENTRY {
|
|
union {
|
|
VI_POOL_ENTRY_INUSE InUse;
|
|
ULONG_PTR FreeListNext;
|
|
};
|
|
} VI_POOL_ENTRY, *PVI_POOL_ENTRY;
|
|
|
|
#define MI_VERIFIER_ENTRY_SIGNATURE 0x98761940
|
|
|
|
typedef struct _MI_VERIFIER_DRIVER_ENTRY {
|
|
LIST_ENTRY Links;
|
|
ULONG Loads;
|
|
ULONG Unloads;
|
|
|
|
UNICODE_STRING BaseName;
|
|
PVOID StartAddress;
|
|
PVOID EndAddress;
|
|
|
|
#define VI_VERIFYING_DIRECTLY 0x1
|
|
#define VI_VERIFYING_INVERSELY 0x2
|
|
#define VI_DISABLE_VERIFICATION 0x4
|
|
|
|
ULONG Flags;
|
|
ULONG_PTR Signature;
|
|
ULONG_PTR Reserved;
|
|
KSPIN_LOCK VerifierPoolLock;
|
|
|
|
PVI_POOL_ENTRY PoolHash;
|
|
ULONG_PTR PoolHashSize;
|
|
ULONG_PTR PoolHashFree;
|
|
ULONG_PTR PoolHashReserved;
|
|
|
|
ULONG CurrentPagedPoolAllocations;
|
|
ULONG CurrentNonPagedPoolAllocations;
|
|
ULONG PeakPagedPoolAllocations;
|
|
ULONG PeakNonPagedPoolAllocations;
|
|
|
|
SIZE_T PagedBytes;
|
|
SIZE_T NonPagedBytes;
|
|
SIZE_T PeakPagedBytes;
|
|
SIZE_T PeakNonPagedBytes;
|
|
|
|
} MI_VERIFIER_DRIVER_ENTRY, *PMI_VERIFIER_DRIVER_ENTRY;
|
|
|
|
typedef struct _MI_VERIFIER_POOL_HEADER {
|
|
ULONG_PTR ListIndex;
|
|
PMI_VERIFIER_DRIVER_ENTRY Verifier;
|
|
} MI_VERIFIER_POOL_HEADER, *PMI_VERIFIER_POOL_HEADER;
|
|
|
|
typedef struct _MM_DRIVER_VERIFIER_DATA {
|
|
ULONG Level;
|
|
ULONG RaiseIrqls;
|
|
ULONG AcquireSpinLocks;
|
|
ULONG SynchronizeExecutions;
|
|
|
|
ULONG AllocationsAttempted;
|
|
ULONG AllocationsSucceeded;
|
|
ULONG AllocationsSucceededSpecialPool;
|
|
ULONG AllocationsWithNoTag;
|
|
|
|
ULONG TrimRequests;
|
|
ULONG Trims;
|
|
ULONG AllocationsFailed;
|
|
ULONG AllocationsFailedDeliberately;
|
|
|
|
ULONG Loads;
|
|
ULONG Unloads;
|
|
ULONG UnTrackedPool;
|
|
ULONG UserTrims;
|
|
|
|
ULONG CurrentPagedPoolAllocations;
|
|
ULONG CurrentNonPagedPoolAllocations;
|
|
ULONG PeakPagedPoolAllocations;
|
|
ULONG PeakNonPagedPoolAllocations;
|
|
|
|
SIZE_T PagedBytes;
|
|
SIZE_T NonPagedBytes;
|
|
SIZE_T PeakPagedBytes;
|
|
SIZE_T PeakNonPagedBytes;
|
|
|
|
ULONG BurstAllocationsFailedDeliberately;
|
|
ULONG SessionTrims;
|
|
ULONG Reserved[2];
|
|
|
|
} MM_DRIVER_VERIFIER_DATA, *PMM_DRIVER_VERIFIER_DATA;
|
|
|
|
LOGICAL
|
|
MiInitializeDriverVerifierList (
|
|
IN PLOADER_PARAMETER_BLOCK LoaderBlock
|
|
);
|
|
|
|
LOGICAL
|
|
MiInitializeVerifyingComponents (
|
|
IN PLOADER_PARAMETER_BLOCK LoaderBlock
|
|
);
|
|
|
|
LOGICAL
|
|
MiApplyDriverVerifier (
|
|
IN PKLDR_DATA_TABLE_ENTRY,
|
|
IN PMI_VERIFIER_DRIVER_ENTRY Verifier
|
|
);
|
|
|
|
VOID
|
|
MiReApplyVerifierToLoadedModules(
|
|
IN PLIST_ENTRY ModuleListHead
|
|
);
|
|
|
|
VOID
|
|
MiVerifyingDriverUnloading (
|
|
IN PKLDR_DATA_TABLE_ENTRY DataTableEntry
|
|
);
|
|
|
|
VOID
|
|
MiVerifierCheckThunks (
|
|
IN PKLDR_DATA_TABLE_ENTRY DataTableEntry
|
|
);
|
|
|
|
extern ULONG MiActiveVerifierThunks;
|
|
extern LIST_ENTRY MiSuspectDriverList;
|
|
|
|
extern ULONG MiVerifierThunksAdded;
|
|
|
|
VOID
|
|
MiEnableKernelVerifier (
|
|
VOID
|
|
);
|
|
|
|
extern LOGICAL KernelVerifier;
|
|
|
|
extern MM_DRIVER_VERIFIER_DATA MmVerifierData;
|
|
|
|
#define MI_FREED_SPECIAL_POOL_SIGNATURE 0x98764321
|
|
|
|
#define MI_STACK_BYTES 1024
|
|
|
|
typedef struct _MI_FREED_SPECIAL_POOL {
|
|
POOL_HEADER OverlaidPoolHeader;
|
|
MI_VERIFIER_POOL_HEADER OverlaidVerifierPoolHeader;
|
|
|
|
ULONG Signature;
|
|
ULONG TickCount;
|
|
ULONG NumberOfBytesRequested;
|
|
ULONG Pagable;
|
|
|
|
PVOID VirtualAddress;
|
|
PVOID StackPointer;
|
|
ULONG StackBytes;
|
|
PETHREAD Thread;
|
|
|
|
UCHAR StackData[MI_STACK_BYTES];
|
|
} MI_FREED_SPECIAL_POOL, *PMI_FREED_SPECIAL_POOL;
|
|
|
|
#define MM_DBG_COMMIT_NONPAGED_POOL_EXPANSION 0
|
|
#define MM_DBG_COMMIT_PAGED_POOL_PAGETABLE 1
|
|
#define MM_DBG_COMMIT_PAGED_POOL_PAGES 2
|
|
#define MM_DBG_COMMIT_SESSION_POOL_PAGE_TABLES 3
|
|
#define MM_DBG_COMMIT_ALLOCVM1 4
|
|
#define MM_DBG_COMMIT_ALLOCVM_SEGMENT 5
|
|
#define MM_DBG_COMMIT_IMAGE 6
|
|
#define MM_DBG_COMMIT_PAGEFILE_BACKED_SHMEM 7
|
|
#define MM_DBG_COMMIT_INDEPENDENT_PAGES 8
|
|
#define MM_DBG_COMMIT_CONTIGUOUS_PAGES 9
|
|
#define MM_DBG_COMMIT_MDL_PAGES 0xA
|
|
#define MM_DBG_COMMIT_NONCACHED_PAGES 0xB
|
|
#define MM_DBG_COMMIT_MAPVIEW_DATA 0xC
|
|
#define MM_DBG_COMMIT_FILL_SYSTEM_DIRECTORY 0xD
|
|
#define MM_DBG_COMMIT_EXTRA_SYSTEM_PTES 0xE
|
|
#define MM_DBG_COMMIT_DRIVER_PAGING_AT_INIT 0xF
|
|
#define MM_DBG_COMMIT_PAGEFILE_FULL 0x10
|
|
#define MM_DBG_COMMIT_PROCESS_CREATE 0x11
|
|
#define MM_DBG_COMMIT_KERNEL_STACK_CREATE 0x12
|
|
#define MM_DBG_COMMIT_SET_PROTECTION 0x13
|
|
#define MM_DBG_COMMIT_SESSION_CREATE 0x14
|
|
#define MM_DBG_COMMIT_SESSION_IMAGE_PAGES 0x15
|
|
#define MM_DBG_COMMIT_SESSION_PAGETABLE_PAGES 0x16
|
|
#define MM_DBG_COMMIT_SESSION_SHARED_IMAGE 0x17
|
|
#define MM_DBG_COMMIT_DRIVER_PAGES 0x18
|
|
#define MM_DBG_COMMIT_INSERT_VAD 0x19
|
|
#define MM_DBG_COMMIT_SESSION_WS_INIT 0x1A
|
|
#define MM_DBG_COMMIT_SESSION_ADDITIONAL_WS_PAGES 0x1B
|
|
#define MM_DBG_COMMIT_SESSION_ADDITIONAL_WS_HASHPAGES 0x1C
|
|
#define MM_DBG_COMMIT_SPECIAL_POOL_PAGES 0x1D
|
|
#define MM_DBG_COMMIT_SPECIAL_POOL_MAPPING_PAGES 0x1E
|
|
#define MM_DBG_COMMIT_SMALL 0x1F
|
|
#define MM_DBG_COMMIT_EXTRA_WS_PAGES 0x20
|
|
#define MM_DBG_COMMIT_EXTRA_INITIAL_SESSION_WS_PAGES 0x21
|
|
#define MM_DBG_COMMIT_ALLOCVM_PROCESS 0x22
|
|
#define MM_DBG_COMMIT_INSERT_VAD_PT 0x23
|
|
#define MM_DBG_COMMIT_ALLOCVM_PROCESS2 0x24
|
|
#define MM_DBG_COMMIT_CHARGE_NORMAL 0x25
|
|
#define MM_DBG_COMMIT_CHARGE_CAUSE_POPUP 0x26
|
|
#define MM_DBG_COMMIT_CHARGE_CANT_EXPAND 0x27
|
|
|
|
#define MM_DBG_COMMIT_RETURN_NONPAGED_POOL_EXPANSION 0x40
|
|
#define MM_DBG_COMMIT_RETURN_PAGED_POOL_PAGES 0x41
|
|
#define MM_DBG_COMMIT_RETURN_SESSION_DATAPAGE 0x42
|
|
#define MM_DBG_COMMIT_RETURN_ALLOCVM_SEGMENT 0x43
|
|
#define MM_DBG_COMMIT_RETURN_ALLOCVM2 0x44
|
|
|
|
#define MM_DBG_COMMIT_RETURN_IMAGE_NO_LARGE_CA 0x46
|
|
#define MM_DBG_COMMIT_RETURN_PTE_RANGE 0x47
|
|
#define MM_DBG_COMMIT_RETURN_NTFREEVM1 0x48
|
|
#define MM_DBG_COMMIT_RETURN_NTFREEVM2 0x49
|
|
#define MM_DBG_COMMIT_RETURN_INDEPENDENT_PAGES 0x4A
|
|
#define MM_DBG_COMMIT_RETURN_AWE_EXCESS 0x4B
|
|
#define MM_DBG_COMMIT_RETURN_MDL_PAGES 0x4C
|
|
#define MM_DBG_COMMIT_RETURN_NONCACHED_PAGES 0x4D
|
|
#define MM_DBG_COMMIT_RETURN_SESSION_CREATE_FAILURE 0x4E
|
|
#define MM_DBG_COMMIT_RETURN_PAGETABLES 0x4F
|
|
#define MM_DBG_COMMIT_RETURN_PROTECTION 0x50
|
|
#define MM_DBG_COMMIT_RETURN_SEGMENT_DELETE1 0x51
|
|
#define MM_DBG_COMMIT_RETURN_SEGMENT_DELETE2 0x52
|
|
#define MM_DBG_COMMIT_RETURN_PAGEFILE_FULL 0x53
|
|
#define MM_DBG_COMMIT_RETURN_SESSION_DEREFERENCE 0x54
|
|
#define MM_DBG_COMMIT_RETURN_VAD 0x55
|
|
#define MM_DBG_COMMIT_RETURN_PROCESS_CREATE_FAILURE1 0x56
|
|
#define MM_DBG_COMMIT_RETURN_PROCESS_DELETE 0x57
|
|
#define MM_DBG_COMMIT_RETURN_PROCESS_CLEAN_PAGETABLES 0x58
|
|
#define MM_DBG_COMMIT_RETURN_KERNEL_STACK_DELETE 0x59
|
|
#define MM_DBG_COMMIT_RETURN_SESSION_DRIVER_LOAD_FAILURE1 0x5A
|
|
#define MM_DBG_COMMIT_RETURN_DRIVER_INIT_CODE 0x5B
|
|
#define MM_DBG_COMMIT_RETURN_DRIVER_UNLOAD 0x5C
|
|
#define MM_DBG_COMMIT_RETURN_DRIVER_UNLOAD1 0x5D
|
|
#define MM_DBG_COMMIT_RETURN_NORMAL 0x5E
|
|
#define MM_DBG_COMMIT_RETURN_PF_FULL_EXTEND 0x5F
|
|
#define MM_DBG_COMMIT_RETURN_EXTENDED 0x60
|
|
#define MM_DBG_COMMIT_RETURN_SEGMENT_DELETE3 0x61
|
|
|
|
#if 0
|
|
|
|
#define MM_COMMIT_COUNTER_MAX 0x80
|
|
|
|
#define MM_TRACK_COMMIT(_index, bump) \
|
|
if (_index >= MM_COMMIT_COUNTER_MAX) { \
|
|
DbgPrint("Mm: Invalid commit counter %d %d\n", _index, MM_COMMIT_COUNTER_MAX); \
|
|
DbgBreakPoint(); \
|
|
} \
|
|
else { \
|
|
InterlockedExchangeAddSizeT (&MmTrackCommit[_index], bump); \
|
|
}
|
|
|
|
#define MM_TRACK_COMMIT_REDUCTION(_index, bump) \
|
|
if (_index >= MM_COMMIT_COUNTER_MAX) { \
|
|
DbgPrint("Mm: Invalid commit counter %d %d\n", _index, MM_COMMIT_COUNTER_MAX); \
|
|
DbgBreakPoint(); \
|
|
} \
|
|
else { \
|
|
InterlockedExchangeAddSizeT (&MmTrackCommit[_index], 0 - (bump)); \
|
|
}
|
|
|
|
extern SIZE_T MmTrackCommit[MM_COMMIT_COUNTER_MAX];
|
|
|
|
#define MI_INCREMENT_TOTAL_PROCESS_COMMIT(_charge) InterlockedExchangeAddSizeT (&MmTotalProcessCommit, (_charge));
|
|
|
|
#else
|
|
|
|
#define MM_TRACK_COMMIT(_index, bump)
|
|
#define MM_TRACK_COMMIT_REDUCTION(_index, bump)
|
|
#define MI_INCREMENT_TOTAL_PROCESS_COMMIT(_charge)
|
|
|
|
#endif
|
|
|
|
|
|
#define MM_BUMP_COUNTER_MAX 60
|
|
|
|
extern SIZE_T MmResTrack[MM_BUMP_COUNTER_MAX];
|
|
|
|
#define MM_BUMP_COUNTER(_index, bump) \
|
|
ASSERT (_index < MM_BUMP_COUNTER_MAX); \
|
|
InterlockedExchangeAddSizeT (&MmResTrack[_index], (SIZE_T)(bump));
|
|
|
|
extern ULONG MiSpecialPagesNonPaged;
|
|
extern ULONG MiSpecialPagesNonPagedMaximum;
|
|
|
|
//++
|
|
//PFN_NUMBER
|
|
//MI_NONPAGABLE_MEMORY_AVAILABLE(
|
|
// VOID
|
|
// );
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// This routine lets callers know how many pages can be charged against
|
|
// the resident available, factoring in earlier Mm promises that
|
|
// may not have been redeemed at this point (ie: nonpaged pool expansion,
|
|
// etc, that must be honored at a later point if requested).
|
|
//
|
|
// Arguments
|
|
//
|
|
// None.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// The number of currently available pages in the resident available.
|
|
//
|
|
// N.B. This is a signed quantity and can be negative.
|
|
//
|
|
//--
|
|
#define MI_NONPAGABLE_MEMORY_AVAILABLE() \
|
|
((SPFN_NUMBER) \
|
|
(MmResidentAvailablePages - \
|
|
MmSystemLockPagesCount))
|
|
|
|
extern ULONG MmLargePageMinimum;
|
|
|
|
//
|
|
// hack stuff for testing.
|
|
//
|
|
|
|
VOID
|
|
MiDumpValidAddresses (
|
|
VOID
|
|
);
|
|
|
|
VOID
|
|
MiDumpPfn ( VOID );
|
|
|
|
VOID
|
|
MiDumpWsl ( VOID );
|
|
|
|
|
|
VOID
|
|
MiFormatPte (
|
|
IN PMMPTE PointerPte
|
|
);
|
|
|
|
VOID
|
|
MiCheckPfn ( VOID );
|
|
|
|
VOID
|
|
MiCheckPte ( VOID );
|
|
|
|
VOID
|
|
MiFormatPfn (
|
|
IN PMMPFN PointerPfn
|
|
);
|
|
|
|
|
|
|
|
|
|
extern const MMPTE ZeroPte;
|
|
|
|
extern const MMPTE ZeroKernelPte;
|
|
|
|
extern const MMPTE ValidKernelPteLocal;
|
|
|
|
extern MMPTE ValidKernelPte;
|
|
|
|
extern MMPTE ValidKernelPde;
|
|
|
|
extern const MMPTE ValidKernelPdeLocal;
|
|
|
|
extern const MMPTE ValidUserPte;
|
|
|
|
extern const MMPTE ValidPtePte;
|
|
|
|
extern const MMPTE ValidPdePde;
|
|
|
|
extern MMPTE DemandZeroPde;
|
|
|
|
extern const MMPTE DemandZeroPte;
|
|
|
|
extern MMPTE KernelPrototypePte;
|
|
|
|
extern const MMPTE TransitionPde;
|
|
|
|
extern MMPTE PrototypePte;
|
|
|
|
extern const MMPTE NoAccessPte;
|
|
|
|
extern ULONG_PTR MmSubsectionBase;
|
|
|
|
extern ULONG_PTR MmSubsectionTopPage;
|
|
|
|
extern ULONG ExpMultiUserTS;
|
|
|
|
//
|
|
// Virtual alignment for PTEs (machine specific) minimum value is
|
|
// 4k maximum value is 64k. The maximum value can be raised by
|
|
// changing the MM_PROTO_PTE_ALIGNMENT constant and adding more
|
|
// reserved mapping PTEs in hyperspace.
|
|
//
|
|
|
|
//
|
|
// Total number of physical pages on the system.
|
|
//
|
|
|
|
extern PFN_COUNT MmNumberOfPhysicalPages;
|
|
|
|
//
|
|
// Lowest physical page number on the system.
|
|
//
|
|
|
|
extern PFN_NUMBER MmLowestPhysicalPage;
|
|
|
|
//
|
|
// Highest physical page number on the system.
|
|
//
|
|
|
|
extern PFN_NUMBER MmHighestPhysicalPage;
|
|
|
|
//
|
|
// Highest possible physical page number in the system.
|
|
//
|
|
|
|
extern PFN_NUMBER MmHighestPossiblePhysicalPage;
|
|
|
|
#if defined (_WIN64)
|
|
|
|
#define MI_DTC_MAX_PAGES ((PFN_NUMBER)(((ULONG64)128 * 1024 * 1024 * 1024) >> PAGE_SHIFT))
|
|
|
|
#define MI_DTC_BOOTED_3GB_MAX_PAGES MI_DTC_MAX_PAGES
|
|
|
|
#define MI_ADS_MAX_PAGES ((PFN_NUMBER)(((ULONG64)64 * 1024 * 1024 * 1024) >> PAGE_SHIFT))
|
|
|
|
#define MI_DEFAULT_MAX_PAGES ((PFN_NUMBER)(((ULONG64)16 * 1024 * 1024 * 1024) >> PAGE_SHIFT))
|
|
|
|
#else
|
|
|
|
#define MI_DTC_MAX_PAGES ((PFN_NUMBER)(((ULONG64)64 * 1024 * 1024 * 1024) >> PAGE_SHIFT))
|
|
|
|
#define MI_DTC_BOOTED_3GB_MAX_PAGES ((PFN_NUMBER)(((ULONG64)16 * 1024 * 1024 * 1024) >> PAGE_SHIFT))
|
|
|
|
#define MI_ADS_MAX_PAGES ((PFN_NUMBER)(((ULONG64)32 * 1024 * 1024 * 1024) >> PAGE_SHIFT))
|
|
|
|
#define MI_DEFAULT_MAX_PAGES ((PFN_NUMBER)(((ULONG64)4 * 1024 * 1024 * 1024) >> PAGE_SHIFT))
|
|
|
|
#endif
|
|
|
|
#define MI_BLADE_MAX_PAGES ((PFN_NUMBER)(((ULONG64)2 * 1024 * 1024 * 1024) >> PAGE_SHIFT))
|
|
|
|
//
|
|
// Total number of available pages on the system. This
|
|
// is the sum of the pages on the zeroed, free and standby lists.
|
|
//
|
|
|
|
extern PFN_COUNT MmAvailablePages;
|
|
|
|
//
|
|
// Total number of free pages to base working set trimming on.
|
|
//
|
|
|
|
extern PFN_NUMBER MmMoreThanEnoughFreePages;
|
|
|
|
//
|
|
// Total number physical pages which would be usable if every process
|
|
// was at it's minimum working set size. This value is initialized
|
|
// at system initialization to MmAvailablePages - MM_FLUID_PHYSICAL_PAGES.
|
|
// Everytime a thread is created, the kernel stack is subtracted from
|
|
// this and every time a process is created, the minimum working set
|
|
// is subtracted from this. If the value would become negative, the
|
|
// operation (create process/kernel stack/ adjust working set) fails.
|
|
// The PFN LOCK must be owned to manipulate this value.
|
|
//
|
|
|
|
extern SPFN_NUMBER MmResidentAvailablePages;
|
|
|
|
//
|
|
// The total number of pages which would be removed from working sets
|
|
// if every working set was at its minimum.
|
|
//
|
|
|
|
extern PFN_NUMBER MmPagesAboveWsMinimum;
|
|
|
|
//
|
|
// The total number of pages which would be removed from working sets
|
|
// if every working set above its maximum was at its maximum.
|
|
//
|
|
|
|
extern PFN_NUMBER MmPagesAboveWsMaximum;
|
|
|
|
//
|
|
// If memory is becoming short and MmPagesAboveWsMinimum is
|
|
// greater than MmPagesAboveWsThreshold, trim working sets.
|
|
//
|
|
|
|
extern PFN_NUMBER MmPagesAboveWsThreshold;
|
|
|
|
//
|
|
// The number of pages to add to a working set if there are ample
|
|
// available pages and the working set is below its maximum.
|
|
//
|
|
|
|
extern PFN_NUMBER MmWorkingSetSizeIncrement;
|
|
|
|
//
|
|
// The number of pages to extend the maximum working set size by
|
|
// if the working set at its maximum and there are ample available pages.
|
|
|
|
extern PFN_NUMBER MmWorkingSetSizeExpansion;
|
|
|
|
extern ULONG MmPlentyFreePages;
|
|
|
|
//
|
|
// The number of pages required to be freed by working set reduction
|
|
// before working set reduction is attempted.
|
|
//
|
|
|
|
extern PFN_NUMBER MmWsAdjustThreshold;
|
|
|
|
//
|
|
// The number of pages available to allow the working set to be
|
|
// expanded above its maximum.
|
|
//
|
|
|
|
extern PFN_NUMBER MmWsExpandThreshold;
|
|
|
|
//
|
|
// The total number of pages to reduce by working set trimming.
|
|
//
|
|
|
|
extern PFN_NUMBER MmWsTrimReductionGoal;
|
|
|
|
extern LONG MiDelayPageFaults;
|
|
|
|
extern PMMPFN MmPfnDatabase;
|
|
|
|
extern MMPFNLIST MmZeroedPageListHead;
|
|
|
|
extern MMPFNLIST MmFreePageListHead;
|
|
|
|
extern MMPFNLIST MmStandbyPageListHead;
|
|
|
|
extern MMPFNLIST MmRomPageListHead;
|
|
|
|
extern MMPFNLIST MmModifiedPageListHead;
|
|
|
|
extern MMPFNLIST MmModifiedNoWritePageListHead;
|
|
|
|
extern MMPFNLIST MmBadPageListHead;
|
|
|
|
extern PMMPFNLIST MmPageLocationList[NUMBER_OF_PAGE_LISTS];
|
|
|
|
extern MMPFNLIST MmModifiedPageListByColor[MM_MAXIMUM_NUMBER_OF_COLORS];
|
|
|
|
//
|
|
// Mask for isolating secondary color from physical page number.
|
|
//
|
|
|
|
extern ULONG MmSecondaryColorMask;
|
|
|
|
//
|
|
// Mask for isolating node color from combined node and secondary
|
|
// color.
|
|
//
|
|
|
|
extern ULONG MmSecondaryColorNodeMask;
|
|
|
|
//
|
|
// Width of MmSecondaryColorMask in bits. In multi node systems,
|
|
// the node number is combined with the secondary color to make up
|
|
// the page color.
|
|
//
|
|
|
|
extern UCHAR MmSecondaryColorNodeShift;
|
|
|
|
//
|
|
// Event for available pages, set means pages are available.
|
|
//
|
|
|
|
extern KEVENT MmAvailablePagesEvent;
|
|
|
|
extern KEVENT MmAvailablePagesEventMedium;
|
|
|
|
extern KEVENT MmAvailablePagesEventHigh;
|
|
|
|
//
|
|
// Event for the zeroing page thread.
|
|
//
|
|
|
|
extern KEVENT MmZeroingPageEvent;
|
|
|
|
//
|
|
// Boolean to indicate if the zeroing page thread is currently
|
|
// active. This is set to true when the zeroing page event is
|
|
// set and set to false when the zeroing page thread is done
|
|
// zeroing all the pages on the free list.
|
|
//
|
|
|
|
extern BOOLEAN MmZeroingPageThreadActive;
|
|
|
|
//
|
|
// Minimum number of free pages before zeroing page thread starts.
|
|
//
|
|
|
|
extern PFN_NUMBER MmMinimumFreePagesToZero;
|
|
|
|
//
|
|
// Global event to synchronize mapped writing with cleaning segments.
|
|
//
|
|
|
|
extern KEVENT MmMappedFileIoComplete;
|
|
|
|
//
|
|
// Hyper space items.
|
|
//
|
|
|
|
extern PMMPTE MmFirstReservedMappingPte;
|
|
|
|
extern PMMPTE MmLastReservedMappingPte;
|
|
|
|
//
|
|
// System space sizes - MmNonPagedSystemStart to MM_NON_PAGED_SYSTEM_END
|
|
// defines the ranges of PDEs which must be copied into a new process's
|
|
// address space.
|
|
//
|
|
|
|
extern PVOID MmNonPagedSystemStart;
|
|
|
|
extern PCHAR MmSystemSpaceViewStart;
|
|
|
|
extern LOGICAL MmProtectFreedNonPagedPool;
|
|
|
|
//
|
|
// Pool sizes.
|
|
//
|
|
|
|
extern SIZE_T MmSizeOfNonPagedPoolInBytes;
|
|
|
|
extern SIZE_T MmMinimumNonPagedPoolSize;
|
|
|
|
extern SIZE_T MmDefaultMaximumNonPagedPool;
|
|
|
|
extern ULONG MmMaximumNonPagedPoolPercent;
|
|
|
|
extern ULONG MmMinAdditionNonPagedPoolPerMb;
|
|
|
|
extern ULONG MmMaxAdditionNonPagedPoolPerMb;
|
|
|
|
extern SIZE_T MmSizeOfPagedPoolInBytes;
|
|
|
|
extern SIZE_T MmMaximumNonPagedPoolInBytes;
|
|
|
|
extern PFN_NUMBER MmAllocatedNonPagedPool;
|
|
|
|
extern PVOID MmNonPagedPoolExpansionStart;
|
|
|
|
extern ULONG MmExpandedPoolBitPosition;
|
|
|
|
extern PFN_NUMBER MmNumberOfFreeNonPagedPool;
|
|
|
|
extern ULONG MmNumberOfSystemPtes;
|
|
|
|
extern ULONG MiRequestedSystemPtes;
|
|
|
|
extern ULONG MmTotalFreeSystemPtes[MaximumPtePoolTypes];
|
|
|
|
extern PMMPTE MmSystemPagePtes;
|
|
|
|
extern ULONG MmSystemPageDirectory[];
|
|
|
|
extern SIZE_T MmHeapSegmentReserve;
|
|
|
|
extern SIZE_T MmHeapSegmentCommit;
|
|
|
|
extern SIZE_T MmHeapDeCommitTotalFreeThreshold;
|
|
|
|
extern SIZE_T MmHeapDeCommitFreeBlockThreshold;
|
|
|
|
#define MI_MAX_FREE_LIST_HEADS 4
|
|
|
|
extern LIST_ENTRY MmNonPagedPoolFreeListHead[MI_MAX_FREE_LIST_HEADS];
|
|
|
|
//
|
|
// Counter for flushes of the entire TB.
|
|
//
|
|
|
|
extern ULONG MmFlushCounter;
|
|
|
|
//
|
|
// Pool start and end.
|
|
//
|
|
|
|
extern PVOID MmNonPagedPoolStart;
|
|
|
|
extern PVOID MmNonPagedPoolEnd;
|
|
|
|
extern PVOID MmPagedPoolStart;
|
|
|
|
extern PVOID MmPagedPoolEnd;
|
|
|
|
//
|
|
// Pool bit maps and other related structures.
|
|
//
|
|
|
|
typedef struct _MM_PAGED_POOL_INFO {
|
|
|
|
PRTL_BITMAP PagedPoolAllocationMap;
|
|
PRTL_BITMAP EndOfPagedPoolBitmap;
|
|
PRTL_BITMAP PagedPoolLargeSessionAllocationMap; // HYDRA only
|
|
PMMPTE FirstPteForPagedPool;
|
|
PMMPTE LastPteForPagedPool;
|
|
PMMPTE NextPdeForPagedPoolExpansion;
|
|
ULONG PagedPoolHint;
|
|
SIZE_T PagedPoolCommit;
|
|
SIZE_T AllocatedPagedPool;
|
|
|
|
} MM_PAGED_POOL_INFO, *PMM_PAGED_POOL_INFO;
|
|
|
|
extern MM_PAGED_POOL_INFO MmPagedPoolInfo;
|
|
|
|
extern PVOID MmPageAlignedPoolBase[2];
|
|
|
|
extern PRTL_BITMAP VerifierLargePagedPoolMap;
|
|
|
|
//
|
|
// MmFirstFreeSystemPte contains the offset from the
|
|
// Nonpaged system base to the first free system PTE.
|
|
// Note, that an offset of zero indicates an empty list.
|
|
//
|
|
|
|
extern MMPTE MmFirstFreeSystemPte[MaximumPtePoolTypes];
|
|
|
|
extern ULONG_PTR MiSystemViewStart;
|
|
|
|
//
|
|
// System cache sizes.
|
|
//
|
|
|
|
//extern MMSUPPORT MmSystemCacheWs;
|
|
|
|
extern PMMWSL MmSystemCacheWorkingSetList;
|
|
|
|
extern PMMWSLE MmSystemCacheWsle;
|
|
|
|
extern PVOID MmSystemCacheStart;
|
|
|
|
extern PVOID MmSystemCacheEnd;
|
|
|
|
extern PFN_NUMBER MmSystemCacheWsMinimum;
|
|
|
|
extern PFN_NUMBER MmSystemCacheWsMaximum;
|
|
|
|
//
|
|
// Virtual alignment for PTEs (machine specific) minimum value is
|
|
// 0 (no alignment) maximum value is 64k. The maximum value can be raised by
|
|
// changing the MM_PROTO_PTE_ALIGNMENT constant and adding more
|
|
// reserved mapping PTEs in hyperspace.
|
|
//
|
|
|
|
extern ULONG MmAliasAlignment;
|
|
|
|
//
|
|
// Mask to AND with virtual address to get an offset to go
|
|
// with the alignment. This value is page aligned.
|
|
//
|
|
|
|
extern ULONG MmAliasAlignmentOffset;
|
|
|
|
//
|
|
// Mask to and with PTEs to determine if the alias mapping is compatible.
|
|
// This value is usually (MmAliasAlignment - 1)
|
|
//
|
|
|
|
extern ULONG MmAliasAlignmentMask;
|
|
|
|
//
|
|
// Cells to track unused thread kernel stacks to avoid TB flushes
|
|
// every time a thread terminates.
|
|
//
|
|
|
|
extern ULONG MmMaximumDeadKernelStacks;
|
|
extern SLIST_HEADER MmDeadStackSListHead;
|
|
|
|
//
|
|
// MmSystemPteBase contains the address of 1 PTE before
|
|
// the first free system PTE (zero indicates an empty list).
|
|
// The value of this field does not change once set.
|
|
//
|
|
|
|
extern PMMPTE MmSystemPteBase;
|
|
|
|
//
|
|
// Root of system space virtual address descriptors. These define
|
|
// the pagable portion of the system.
|
|
//
|
|
|
|
extern PMMVAD MmVirtualAddressDescriptorRoot;
|
|
|
|
extern PMMADDRESS_NODE MmSectionBasedRoot;
|
|
|
|
extern PVOID MmHighSectionBase;
|
|
|
|
//
|
|
// Section commit mutex.
|
|
//
|
|
|
|
extern FAST_MUTEX MmSectionCommitMutex;
|
|
|
|
//
|
|
// Section base address mutex.
|
|
//
|
|
|
|
extern FAST_MUTEX MmSectionBasedMutex;
|
|
|
|
//
|
|
// Resource for section extension.
|
|
//
|
|
|
|
extern ERESOURCE MmSectionExtendResource;
|
|
extern ERESOURCE MmSectionExtendSetResource;
|
|
|
|
//
|
|
// Inpage cluster sizes for executable pages (set based on memory size).
|
|
//
|
|
|
|
extern ULONG MmDataClusterSize;
|
|
|
|
extern ULONG MmCodeClusterSize;
|
|
|
|
//
|
|
// Pagefile creation mutex.
|
|
//
|
|
|
|
extern FAST_MUTEX MmPageFileCreationLock;
|
|
|
|
//
|
|
// Event to set when first paging file is created.
|
|
//
|
|
|
|
extern PKEVENT MmPagingFileCreated;
|
|
|
|
//
|
|
// Paging file debug information.
|
|
//
|
|
|
|
extern ULONG_PTR MmPagingFileDebug[];
|
|
|
|
//
|
|
// Spinlock which guards PFN database. This spinlock is used by
|
|
// memory management for accessing the PFN database. The I/O
|
|
// system makes use of it for unlocking pages during I/O complete.
|
|
//
|
|
|
|
// extern KSPIN_LOCK MmPfnLock;
|
|
|
|
//
|
|
// Spinlock which guards the working set list for the system shared
|
|
// address space (paged pool, system cache, pagable drivers).
|
|
//
|
|
|
|
extern ERESOURCE MmSystemWsLock;
|
|
|
|
//
|
|
// Spin lock for allowing working set expansion.
|
|
//
|
|
|
|
extern KSPIN_LOCK MmExpansionLock;
|
|
|
|
//
|
|
// To prevent optimizations.
|
|
//
|
|
|
|
extern MMPTE GlobalPte;
|
|
|
|
//
|
|
// Page color for system working set.
|
|
//
|
|
|
|
extern ULONG MmSystemPageColor;
|
|
|
|
extern ULONG MmSecondaryColors;
|
|
|
|
extern ULONG MmProcessColorSeed;
|
|
|
|
//
|
|
// Set from ntos\config\CMDAT3.C Used by customers to disable paging
|
|
// of executive on machines with lots of memory. Worth a few TPS on a
|
|
// data base server.
|
|
//
|
|
|
|
#define MM_SYSTEM_CODE_LOCKED_DOWN 0x1
|
|
#define MM_PAGED_POOL_LOCKED_DOWN 0x2
|
|
|
|
extern ULONG MmDisablePagingExecutive;
|
|
|
|
|
|
//
|
|
// For debugging.
|
|
|
|
|
|
#if DBG
|
|
extern ULONG MmDebug;
|
|
#endif
|
|
|
|
//
|
|
// Unused segment management
|
|
//
|
|
|
|
extern MMDEREFERENCE_SEGMENT_HEADER MmDereferenceSegmentHeader;
|
|
|
|
extern LIST_ENTRY MmUnusedSegmentList;
|
|
|
|
extern LIST_ENTRY MmUnusedSubsectionList;
|
|
|
|
extern KEVENT MmUnusedSegmentCleanup;
|
|
|
|
extern ULONG MmConsumedPoolPercentage;
|
|
|
|
extern ULONG MmUnusedSegmentCount;
|
|
|
|
extern ULONG MmUnusedSubsectionCount;
|
|
|
|
extern ULONG MmUnusedSubsectionCountPeak;
|
|
|
|
extern SIZE_T MiUnusedSubsectionPagedPool;
|
|
|
|
extern SIZE_T MiUnusedSubsectionPagedPoolPeak;
|
|
|
|
#define MI_UNUSED_SUBSECTIONS_COUNT_INSERT(_MappedSubsection) \
|
|
MmUnusedSubsectionCount += 1; \
|
|
if (MmUnusedSubsectionCount > MmUnusedSubsectionCountPeak) { \
|
|
MmUnusedSubsectionCountPeak = MmUnusedSubsectionCount; \
|
|
} \
|
|
MiUnusedSubsectionPagedPool += EX_REAL_POOL_USAGE((_MappedSubsection->PtesInSubsection + _MappedSubsection->UnusedPtes) * sizeof (MMPTE)); \
|
|
if (MiUnusedSubsectionPagedPool > MiUnusedSubsectionPagedPoolPeak) { \
|
|
MiUnusedSubsectionPagedPoolPeak = MiUnusedSubsectionPagedPool; \
|
|
} \
|
|
|
|
#define MI_UNUSED_SUBSECTIONS_COUNT_REMOVE(_MappedSubsection) \
|
|
MmUnusedSubsectionCount -= 1; \
|
|
MiUnusedSubsectionPagedPool -= EX_REAL_POOL_USAGE((_MappedSubsection->PtesInSubsection + _MappedSubsection->UnusedPtes) * sizeof (MMPTE));
|
|
|
|
#define MI_FILESYSTEM_NONPAGED_POOL_CHARGE 150
|
|
|
|
#define MI_FILESYSTEM_PAGED_POOL_CHARGE 1024
|
|
|
|
//++
|
|
//LOGICAL
|
|
//MI_UNUSED_SEGMENTS_SURPLUS (
|
|
// IN PVOID va
|
|
// );
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// This routine determines whether a surplus of unused
|
|
// segments exist. If so, the caller can initiate a trim to free pool.
|
|
//
|
|
// Arguments
|
|
//
|
|
// None.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// TRUE if unused segment trimming should be initiated, FALSE if not.
|
|
//
|
|
//--
|
|
#define MI_UNUSED_SEGMENTS_SURPLUS() \
|
|
(((ULONG)((MmPagedPoolInfo.AllocatedPagedPool * 100) / (MmSizeOfPagedPoolInBytes >> PAGE_SHIFT)) > MmConsumedPoolPercentage) || \
|
|
((ULONG)((MmAllocatedNonPagedPool * 100) / (MmMaximumNonPagedPoolInBytes >> PAGE_SHIFT)) > MmConsumedPoolPercentage))
|
|
|
|
VOID
|
|
MiConvertStaticSubsections (
|
|
IN PCONTROL_AREA ControlArea
|
|
);
|
|
|
|
//++
|
|
//VOID
|
|
//MI_INSERT_UNUSED_SEGMENT (
|
|
// IN PCONTROL_AREA _ControlArea
|
|
// );
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// This routine inserts a control area into the unused segment list,
|
|
// also managing the associated pool charges.
|
|
//
|
|
// Arguments
|
|
//
|
|
// _ControlArea - Supplies the control area to obtain the pool charges from.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// None.
|
|
//
|
|
//--
|
|
#define MI_INSERT_UNUSED_SEGMENT(_ControlArea) \
|
|
{ \
|
|
MM_PFN_LOCK_ASSERT(); \
|
|
if ((_ControlArea->u.Flags.Image == 0) && \
|
|
(_ControlArea->FilePointer != NULL) && \
|
|
(_ControlArea->u.Flags.PhysicalMemory == 0)) { \
|
|
MiConvertStaticSubsections(_ControlArea); \
|
|
} \
|
|
InsertTailList (&MmUnusedSegmentList, &_ControlArea->DereferenceList); \
|
|
MmUnusedSegmentCount += 1; \
|
|
}
|
|
|
|
//++
|
|
//VOID
|
|
//MI_UNUSED_SEGMENTS_REMOVE_CHARGE (
|
|
// IN PCONTROL_AREA _ControlArea
|
|
// );
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// This routine manages pool charges during removals of segments from
|
|
// the unused segment list.
|
|
//
|
|
// Arguments
|
|
//
|
|
// _ControlArea - Supplies the control area to obtain the pool charges from.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// None.
|
|
//
|
|
//--
|
|
#define MI_UNUSED_SEGMENTS_REMOVE_CHARGE(_ControlArea) \
|
|
{ \
|
|
MM_PFN_LOCK_ASSERT(); \
|
|
MmUnusedSegmentCount -= 1; \
|
|
}
|
|
|
|
//
|
|
// List heads
|
|
//
|
|
|
|
extern MMWORKING_SET_EXPANSION_HEAD MmWorkingSetExpansionHead;
|
|
|
|
extern MMPAGE_FILE_EXPANSION MmAttemptForCantExtend;
|
|
|
|
//
|
|
// Paging files
|
|
//
|
|
|
|
extern MMMOD_WRITER_LISTHEAD MmPagingFileHeader;
|
|
|
|
extern MMMOD_WRITER_LISTHEAD MmMappedFileHeader;
|
|
|
|
extern PMMPAGING_FILE MmPagingFile[MAX_PAGE_FILES];
|
|
|
|
extern LIST_ENTRY MmFreePagingSpaceLow;
|
|
|
|
extern ULONG MmNumberOfActiveMdlEntries;
|
|
|
|
extern ULONG MmNumberOfPagingFiles;
|
|
|
|
extern KEVENT MmModifiedPageWriterEvent;
|
|
|
|
extern KEVENT MmCollidedFlushEvent;
|
|
|
|
extern KEVENT MmCollidedLockEvent;
|
|
|
|
//
|
|
// Total number of committed pages.
|
|
//
|
|
|
|
extern SIZE_T MmTotalCommittedPages;
|
|
|
|
extern SIZE_T MmTotalCommitLimit;
|
|
|
|
extern SIZE_T MmOverCommit;
|
|
|
|
extern SIZE_T MmSharedCommit;
|
|
|
|
// #define _MI_DEBUG_DATA 1 // Uncomment this for data logging
|
|
|
|
#if defined (_MI_DEBUG_DATA)
|
|
|
|
#define MI_DATA_BACKTRACE_LENGTH 8
|
|
|
|
typedef struct _MI_DATA_TRACES {
|
|
|
|
PETHREAD Thread;
|
|
PMMPFN Pfn;
|
|
PMMPTE PointerPte;
|
|
MMPFN PfnData;
|
|
ULONG CallerId;
|
|
ULONG DataInThePage[2];
|
|
PVOID StackTrace [MI_DATA_BACKTRACE_LENGTH];
|
|
|
|
} MI_DATA_TRACES, *PMI_DATA_TRACES;
|
|
|
|
extern LONG MiDataIndex;
|
|
|
|
extern ULONG MiTrackData;
|
|
|
|
extern PMI_DATA_TRACES MiDataTraces;
|
|
|
|
VOID
|
|
FORCEINLINE
|
|
MiSnapData (
|
|
IN PMMPFN Pfn,
|
|
IN PMMPTE PointerPte,
|
|
IN ULONG CallerId
|
|
)
|
|
{
|
|
KIRQL OldIrql;
|
|
PVOID Va;
|
|
PMI_DATA_TRACES Information;
|
|
ULONG Index;
|
|
ULONG Hash;
|
|
PEPROCESS CurrentProcess;
|
|
|
|
if (MiDataTraces == NULL) {
|
|
return;
|
|
}
|
|
|
|
Index = InterlockedIncrement(&MiDataIndex);
|
|
Index &= (MiTrackData - 1);
|
|
Information = &MiDataTraces[Index];
|
|
|
|
Information->Thread = PsGetCurrentThread ();
|
|
Information->Pfn = Pfn;
|
|
Information->PointerPte = PointerPte;
|
|
Information->PfnData = *Pfn;
|
|
Information->CallerId = CallerId;
|
|
|
|
CurrentProcess = PsGetCurrentProcess ();
|
|
Va = MiMapPageInHyperSpace (CurrentProcess, Pfn - MmPfnDatabase, &OldIrql);
|
|
|
|
RtlCopyMemory (&Information->DataInThePage[0],
|
|
Va,
|
|
sizeof (Information->DataInThePage));
|
|
|
|
MiUnmapPageInHyperSpace (CurrentProcess, Va, OldIrql);
|
|
|
|
RtlZeroMemory (&Information->StackTrace[0], MI_DATA_BACKTRACE_LENGTH * sizeof(PVOID)); \
|
|
|
|
RtlCaptureStackBackTrace (0, MI_DATA_BACKTRACE_LENGTH, Information->StackTrace, &Hash);
|
|
}
|
|
|
|
#define MI_SNAP_DATA(_Pfn, _Pte, _CallerId) MiSnapData(_Pfn, _Pte, _CallerId)
|
|
|
|
#else
|
|
#define MI_SNAP_DATA(_Pfn, _Pte, _CallerId)
|
|
#endif
|
|
|
|
|
|
//
|
|
// Modified page writer.
|
|
//
|
|
|
|
extern PFN_NUMBER MmMinimumFreePages;
|
|
|
|
extern PFN_NUMBER MmFreeGoal;
|
|
|
|
extern PFN_NUMBER MmModifiedPageMaximum;
|
|
|
|
extern PFN_NUMBER MmModifiedPageMinimum;
|
|
|
|
extern ULONG MmModifiedWriteClusterSize;
|
|
|
|
extern ULONG MmMinimumFreeDiskSpace;
|
|
|
|
extern ULONG MmPageFileExtension;
|
|
|
|
extern ULONG MmMinimumPageFileReduction;
|
|
|
|
extern LARGE_INTEGER MiModifiedPageLife;
|
|
|
|
extern BOOLEAN MiTimerPending;
|
|
|
|
extern KEVENT MiMappedPagesTooOldEvent;
|
|
|
|
extern KDPC MiModifiedPageWriterTimerDpc;
|
|
|
|
extern KTIMER MiModifiedPageWriterTimer;
|
|
|
|
//
|
|
// System process working set sizes.
|
|
//
|
|
|
|
extern PFN_NUMBER MmSystemProcessWorkingSetMin;
|
|
|
|
extern PFN_NUMBER MmSystemProcessWorkingSetMax;
|
|
|
|
extern PFN_NUMBER MmMinimumWorkingSetSize;
|
|
|
|
//
|
|
// Support for debugger's mapping physical memory.
|
|
//
|
|
|
|
extern PMMPTE MmDebugPte;
|
|
|
|
extern PMMPTE MmCrashDumpPte;
|
|
|
|
extern ULONG MiOverCommitCallCount;
|
|
|
|
//
|
|
// Event tracing routines
|
|
//
|
|
|
|
extern PPAGE_FAULT_NOTIFY_ROUTINE MmPageFaultNotifyRoutine;
|
|
|
|
extern SIZE_T MmSystemViewSize;
|
|
|
|
VOID
|
|
FASTCALL
|
|
MiIdentifyPfn (
|
|
IN PMMPFN Pfn1,
|
|
OUT PMMPFN_IDENTITY PfnIdentity
|
|
);
|
|
|
|
#if defined (_WIN64)
|
|
#define InterlockedExchangeAddSizeT(a, b) InterlockedExchangeAdd64((PLONGLONG)a, b)
|
|
#else
|
|
#define InterlockedExchangeAddSizeT(a, b) InterlockedExchangeAdd((PLONG)(a), b)
|
|
#endif
|
|
|
|
//
|
|
// This is a special value loaded into an EPROCESS pointer to indicate that
|
|
// the action underway is for a Hydra session, not really the current process.
|
|
// (Any value could be used here that is not a valid system pointer or NULL).
|
|
//
|
|
|
|
#define HYDRA_PROCESS ((PEPROCESS)1)
|
|
|
|
#define PREFETCH_PROCESS ((PEPROCESS)2)
|
|
|
|
#define MI_SESSION_SPACE_STRUCT_SIZE MM_ALLOCATION_GRANULARITY
|
|
|
|
#if defined (_WIN64)
|
|
|
|
/*++
|
|
|
|
Virtual memory layout of session space when loaded down from
|
|
0x2000.0002.0000.0000 (IA64) or FFFF.F980.0000.0000 (AMD64) :
|
|
|
|
Note that the sizes of mapped views, paged pool & images are registry tunable.
|
|
|
|
+------------------------------------+
|
|
2000.0002.0000.0000 | |
|
|
| win32k.sys & video drivers |
|
|
| (16MB) |
|
|
| |
|
|
+------------------------------------+
|
|
2000.0001.FF00.0000 | |
|
|
| MM_SESSION_SPACE & Session WSLs |
|
|
| (16MB) |
|
|
| |
|
|
2000.0001.FEFF.0000 +------------------------------------+
|
|
| |
|
|
| ... |
|
|
| |
|
|
+------------------------------------+
|
|
2000.0001.FE80.0000 | |
|
|
| Mapped Views for this session |
|
|
| (104MB) |
|
|
| |
|
|
+------------------------------------+
|
|
2000.0001.F800.0000 | |
|
|
| Paged Pool for this session |
|
|
| (64MB) |
|
|
| |
|
|
2000.0001.F400.0000 +------------------------------------+
|
|
| Special Pool for this session |
|
|
| (64MB) |
|
|
| |
|
|
2000.0000.0000.0000 +------------------------------------+
|
|
|
|
--*/
|
|
|
|
#define MI_SESSION_SPACE_WS_SIZE ((ULONG_PTR)(16*1024*1024) - MI_SESSION_SPACE_STRUCT_SIZE)
|
|
|
|
#define MI_SESSION_DEFAULT_IMAGE_SIZE ((ULONG_PTR)(16*1024*1024))
|
|
|
|
#define MI_SESSION_DEFAULT_VIEW_SIZE ((ULONG_PTR)(104*1024*1024))
|
|
|
|
#define MI_SESSION_DEFAULT_POOL_SIZE ((ULONG_PTR)(64*1024*1024))
|
|
|
|
#define MI_SESSION_SPACE_MAXIMUM_TOTAL_SIZE (MM_VA_MAPPED_BY_PPE)
|
|
|
|
#else
|
|
|
|
/*++
|
|
|
|
Virtual memory layout of session space when loaded down from 0xC0000000.
|
|
|
|
Note that the sizes of mapped views, paged pool and images are registry
|
|
tunable on 32-bit systems (if NOT booted /3GB, as 3GB has very limited
|
|
address space).
|
|
|
|
+------------------------------------+
|
|
C0000000 | |
|
|
| win32k.sys, video drivers and any |
|
|
| rebased NT4 printer drivers. |
|
|
| |
|
|
| (8MB) |
|
|
| |
|
|
+------------------------------------+
|
|
BF800000 | |
|
|
| MM_SESSION_SPACE & Session WSLs |
|
|
| (4MB) |
|
|
| |
|
|
+------------------------------------+
|
|
BF400000 | |
|
|
| Mapped views for this session |
|
|
| (20MB by default, but is |
|
|
| registry configurable) |
|
|
| |
|
|
+------------------------------------+
|
|
BE000000 | |
|
|
| Paged pool for this session |
|
|
| (16MB by default, but is |
|
|
| registry configurable) |
|
|
| |
|
|
BD000000 +------------------------------------+
|
|
|
|
--*/
|
|
|
|
#define MI_SESSION_SPACE_WS_SIZE (4*1024*1024 - MI_SESSION_SPACE_STRUCT_SIZE)
|
|
|
|
#define MI_SESSION_DEFAULT_IMAGE_SIZE (8*1024*1024)
|
|
|
|
#define MI_SESSION_DEFAULT_VIEW_SIZE (20*1024*1024)
|
|
|
|
#define MI_SESSION_DEFAULT_POOL_SIZE (16*1024*1024)
|
|
|
|
#define MI_SESSION_SPACE_MAXIMUM_TOTAL_SIZE \
|
|
(MM_SYSTEM_CACHE_END_EXTRA - MM_KSEG2_BASE)
|
|
|
|
#endif
|
|
|
|
|
|
|
|
#define MI_SESSION_SPACE_DEFAULT_TOTAL_SIZE \
|
|
(MI_SESSION_DEFAULT_IMAGE_SIZE + \
|
|
MI_SESSION_SPACE_STRUCT_SIZE + \
|
|
MI_SESSION_SPACE_WS_SIZE + \
|
|
MI_SESSION_DEFAULT_VIEW_SIZE + \
|
|
MI_SESSION_DEFAULT_POOL_SIZE)
|
|
|
|
extern ULONG_PTR MmSessionBase;
|
|
extern PMMPTE MiSessionBasePte;
|
|
extern PMMPTE MiSessionLastPte;
|
|
|
|
extern ULONG_PTR MiSessionSpaceWs;
|
|
|
|
extern ULONG_PTR MiSessionViewStart;
|
|
extern SIZE_T MmSessionViewSize;
|
|
|
|
extern ULONG_PTR MiSessionImageStart;
|
|
extern ULONG_PTR MiSessionImageEnd;
|
|
extern SIZE_T MmSessionImageSize;
|
|
|
|
extern PMMPTE MiSessionImagePteStart;
|
|
extern PMMPTE MiSessionImagePteEnd;
|
|
|
|
extern ULONG_PTR MiSessionPoolStart;
|
|
extern ULONG_PTR MiSessionPoolEnd;
|
|
extern SIZE_T MmSessionPoolSize;
|
|
|
|
extern ULONG_PTR MiSessionSpaceEnd;
|
|
|
|
extern ULONG MiSessionSpacePageTables;
|
|
|
|
//
|
|
// The number of page table pages required to map all of session space.
|
|
//
|
|
|
|
#define MI_SESSION_SPACE_MAXIMUM_PAGE_TABLES \
|
|
(MI_SESSION_SPACE_MAXIMUM_TOTAL_SIZE / MM_VA_MAPPED_BY_PDE)
|
|
|
|
extern SIZE_T MmSessionSize; // size of the entire session space.
|
|
|
|
//
|
|
// Macros to determine if a given address lies in the specified session range.
|
|
//
|
|
|
|
#define MI_IS_SESSION_IMAGE_ADDRESS(VirtualAddress) \
|
|
((PVOID)(VirtualAddress) >= (PVOID)MiSessionImageStart && (PVOID)(VirtualAddress) < (PVOID)(MiSessionImageEnd))
|
|
|
|
#define MI_IS_SESSION_POOL_ADDRESS(VirtualAddress) \
|
|
((PVOID)(VirtualAddress) >= (PVOID)MiSessionPoolStart && (PVOID)(VirtualAddress) < (PVOID)MiSessionPoolEnd)
|
|
|
|
#define MI_IS_SESSION_ADDRESS(_VirtualAddress) \
|
|
((PVOID)(_VirtualAddress) >= (PVOID)MmSessionBase && (PVOID)(_VirtualAddress) < (PVOID)(MiSessionSpaceEnd))
|
|
|
|
#define MI_IS_SESSION_PTE(_Pte) \
|
|
((PMMPTE)(_Pte) >= MiSessionBasePte && (PMMPTE)(_Pte) < MiSessionLastPte)
|
|
|
|
#define MI_IS_SESSION_IMAGE_PTE(_Pte) \
|
|
((PMMPTE)(_Pte) >= MiSessionImagePteStart && (PMMPTE)(_Pte) < MiSessionImagePteEnd)
|
|
|
|
#define SESSION_GLOBAL(_Session) (_Session->GlobalVirtualAddress)
|
|
|
|
#define MM_DBG_SESSION_INITIAL_PAGETABLE_ALLOC 0
|
|
#define MM_DBG_SESSION_INITIAL_PAGETABLE_FREE_RACE 1
|
|
#define MM_DBG_SESSION_INITIAL_PAGE_ALLOC 2
|
|
#define MM_DBG_SESSION_INITIAL_PAGE_FREE_FAIL1 3
|
|
#define MM_DBG_SESSION_INITIAL_PAGETABLE_FREE_FAIL1 4
|
|
#define MM_DBG_SESSION_WS_PAGE_FREE 5
|
|
#define MM_DBG_SESSION_PAGETABLE_ALLOC 6
|
|
#define MM_DBG_SESSION_SYSMAPPED_PAGES_ALLOC 7
|
|
#define MM_DBG_SESSION_WS_PAGETABLE_ALLOC 8
|
|
#define MM_DBG_SESSION_PAGEDPOOL_PAGETABLE_ALLOC 9
|
|
#define MM_DBG_SESSION_PAGEDPOOL_PAGETABLE_FREE_FAIL1 10
|
|
#define MM_DBG_SESSION_WS_PAGE_ALLOC 11
|
|
#define MM_DBG_SESSION_WS_PAGE_ALLOC_GROWTH 12
|
|
#define MM_DBG_SESSION_INITIAL_PAGE_FREE 13
|
|
#define MM_DBG_SESSION_PAGETABLE_FREE 14
|
|
#define MM_DBG_SESSION_PAGEDPOOL_PAGETABLE_ALLOC1 15
|
|
#define MM_DBG_SESSION_DRIVER_PAGES_LOCKED 16
|
|
#define MM_DBG_SESSION_DRIVER_PAGES_UNLOCKED 17
|
|
#define MM_DBG_SESSION_WS_HASHPAGE_ALLOC 18
|
|
#define MM_DBG_SESSION_SYSMAPPED_PAGES_COMMITTED 19
|
|
|
|
#define MM_DBG_SESSION_COMMIT_PAGEDPOOL_PAGES 30
|
|
#define MM_DBG_SESSION_COMMIT_DELETE_VM_RETURN 31
|
|
#define MM_DBG_SESSION_COMMIT_POOL_FREED 32
|
|
#define MM_DBG_SESSION_COMMIT_IMAGE_UNLOAD 33
|
|
#define MM_DBG_SESSION_COMMIT_IMAGELOAD_FAILED1 34
|
|
#define MM_DBG_SESSION_COMMIT_IMAGELOAD_FAILED2 35
|
|
#define MM_DBG_SESSION_COMMIT_IMAGELOAD_NOACCESS 36
|
|
|
|
#define MM_DBG_SESSION_NP_LOCK_CODE1 38
|
|
#define MM_DBG_SESSION_NP_LOCK_CODE2 39
|
|
#define MM_DBG_SESSION_NP_SESSION_CREATE 40
|
|
#define MM_DBG_SESSION_NP_PAGETABLE_ALLOC 41
|
|
#define MM_DBG_SESSION_NP_POOL_CREATE 42
|
|
#define MM_DBG_SESSION_NP_COMMIT_IMAGE 43
|
|
#define MM_DBG_SESSION_NP_COMMIT_IMAGE_PT 44
|
|
#define MM_DBG_SESSION_NP_INIT_WS 45
|
|
#define MM_DBG_SESSION_NP_WS_GROW 46
|
|
#define MM_DBG_SESSION_NP_HASH_GROW 47
|
|
|
|
#define MM_DBG_SESSION_NP_PAGE_DRIVER 48
|
|
#define MM_DBG_SESSION_NP_POOL_CREATE_FAILED 49
|
|
#define MM_DBG_SESSION_NP_WS_PAGE_FREE 50
|
|
#define MM_DBG_SESSION_NP_SESSION_DESTROY 51
|
|
#define MM_DBG_SESSION_NP_SESSION_PTDESTROY 52
|
|
#define MM_DBG_SESSION_NP_DELVA 53
|
|
|
|
#if DBG
|
|
#define MM_SESS_COUNTER_MAX 54
|
|
|
|
#define MM_BUMP_SESS_COUNTER(_index, bump) \
|
|
if (_index >= MM_SESS_COUNTER_MAX) { \
|
|
DbgPrint("Mm: Invalid bump counter %d %d\n", _index, MM_SESS_COUNTER_MAX); \
|
|
DbgBreakPoint(); \
|
|
} \
|
|
MmSessionSpace->Debug[_index] += (bump);
|
|
|
|
typedef struct _MM_SESSION_MEMORY_COUNTERS {
|
|
SIZE_T NonPagablePages;
|
|
SIZE_T CommittedPages;
|
|
} MM_SESSION_MEMORY_COUNTERS, *PMM_SESSION_MEMORY_COUNTERS;
|
|
|
|
#define MM_SESS_MEMORY_COUNTER_MAX 8
|
|
|
|
#define MM_SNAP_SESS_MEMORY_COUNTERS(_index) \
|
|
if (_index >= MM_SESS_MEMORY_COUNTER_MAX) { \
|
|
DbgPrint("Mm: Invalid session mem counter %d %d\n", _index, MM_SESS_MEMORY_COUNTER_MAX); \
|
|
DbgBreakPoint(); \
|
|
} \
|
|
else { \
|
|
MmSessionSpace->Debug2[_index].NonPagablePages = MmSessionSpace->NonPagablePages; \
|
|
MmSessionSpace->Debug2[_index].CommittedPages = MmSessionSpace->CommittedPages; \
|
|
}
|
|
|
|
#else
|
|
#define MM_BUMP_SESS_COUNTER(_index, bump)
|
|
#define MM_SNAP_SESS_MEMORY_COUNTERS(_index)
|
|
#endif
|
|
|
|
|
|
#define MM_SESSION_FAILURE_NO_IDS 0
|
|
#define MM_SESSION_FAILURE_NO_COMMIT 1
|
|
#define MM_SESSION_FAILURE_NO_RESIDENT 2
|
|
#define MM_SESSION_FAILURE_RACE_DETECTED 3
|
|
#define MM_SESSION_FAILURE_NO_SYSPTES 4
|
|
#define MM_SESSION_FAILURE_NO_PAGED_POOL 5
|
|
#define MM_SESSION_FAILURE_NO_NONPAGED_POOL 6
|
|
#define MM_SESSION_FAILURE_NO_IMAGE_VA_SPACE 7
|
|
#define MM_SESSION_FAILURE_NO_SESSION_PAGED_POOL 8
|
|
#define MM_SESSION_FAILURE_NO_AVAILABLE 9
|
|
|
|
#define MM_SESSION_FAILURE_CAUSES 10
|
|
|
|
ULONG MmSessionFailureCauses[MM_SESSION_FAILURE_CAUSES];
|
|
|
|
#define MM_BUMP_SESSION_FAILURES(_index) MmSessionFailureCauses[_index] += 1;
|
|
|
|
typedef struct _MM_SESSION_SPACE_FLAGS {
|
|
ULONG Initialized : 1;
|
|
ULONG Filler0 : 1;
|
|
ULONG WorkingSetInserted : 1;
|
|
ULONG SessionListInserted : 1;
|
|
ULONG HasWsLock : 1;
|
|
ULONG DeletePending : 1;
|
|
ULONG Filler : 26;
|
|
} MM_SESSION_SPACE_FLAGS;
|
|
|
|
//
|
|
// The session space data structure - allocated per session and only visible at
|
|
// MM_SESSION_SPACE_BASE when in the context of a process from the session.
|
|
// This virtual address space is rotated at context switch time when switching
|
|
// from a process in session A to a process in session B. This rotation is
|
|
// useful for things like providing paged pool per session so many sessions
|
|
// won't exhaust the VA space which backs the system global pool.
|
|
//
|
|
// A kernel PTE is also allocated to double map this page so that global
|
|
// pointers can be maintained to provide system access from any process context.
|
|
// This is needed for things like mutexes and WSL chains.
|
|
//
|
|
|
|
typedef struct _MM_SESSION_SPACE {
|
|
|
|
ULONG ReferenceCount;
|
|
union {
|
|
ULONG LongFlags;
|
|
MM_SESSION_SPACE_FLAGS Flags;
|
|
} u;
|
|
ULONG SessionId;
|
|
|
|
//
|
|
// All the page tables for session space use this as their parent.
|
|
// Note that it's not really a page directory - it's really a page
|
|
// table page itself (the one used to map this very structure).
|
|
//
|
|
// This provides a reference to something that won't go away and
|
|
// is relevant regardless of which process within the session is current.
|
|
//
|
|
|
|
PFN_NUMBER SessionPageDirectoryIndex;
|
|
|
|
//
|
|
// This is a pointer in global system address space, used to make various
|
|
// fields that can be referenced from any process visible from any process
|
|
// context. This is for things like mutexes, WSL chains, etc.
|
|
//
|
|
|
|
struct _MM_SESSION_SPACE *GlobalVirtualAddress;
|
|
|
|
//
|
|
// This is the list of the processes in this group that have
|
|
// session space entries.
|
|
//
|
|
|
|
LIST_ENTRY ProcessList;
|
|
|
|
//
|
|
// Pool allocation counts - these are always valid.
|
|
//
|
|
|
|
SIZE_T NonPagedPoolBytes;
|
|
SIZE_T PagedPoolBytes;
|
|
ULONG NonPagedPoolAllocations;
|
|
ULONG PagedPoolAllocations;
|
|
|
|
//
|
|
// This is the count of non paged allocations to support this session
|
|
// space. This includes the session structure page table and data pages,
|
|
// WSL page table and data pages, session pool page table pages and session
|
|
// image page table pages. These are all charged against
|
|
// MmResidentAvailable.
|
|
//
|
|
|
|
SIZE_T NonPagablePages;
|
|
|
|
//
|
|
// This is the count of pages in this session that have been charged against
|
|
// the systemwide commit. This includes all the NonPagablePages plus the
|
|
// data pages they typically map.
|
|
//
|
|
|
|
SIZE_T CommittedPages;
|
|
|
|
LARGE_INTEGER LastProcessSwappedOutTime;
|
|
|
|
#if (_MI_PAGING_LEVELS >= 3)
|
|
|
|
//
|
|
// The page directory that maps session space is saved here so
|
|
// trimmers can attach.
|
|
//
|
|
|
|
MMPTE PageDirectory;
|
|
|
|
#else
|
|
|
|
//
|
|
// The second level page tables that map session space are shared
|
|
// by all processes in the session.
|
|
//
|
|
|
|
PMMPTE PageTables;
|
|
|
|
#endif
|
|
|
|
//
|
|
// Session space paged pool support.
|
|
//
|
|
|
|
FAST_MUTEX PagedPoolMutex;
|
|
|
|
//
|
|
// Start of session paged pool virtual space.
|
|
//
|
|
|
|
PVOID PagedPoolStart;
|
|
|
|
//
|
|
// Current end of pool virtual space. Can be extended to the
|
|
// end of the session space.
|
|
//
|
|
|
|
PVOID PagedPoolEnd;
|
|
|
|
//
|
|
// PTE pointers for pool.
|
|
//
|
|
|
|
PMMPTE PagedPoolBasePde;
|
|
|
|
MM_PAGED_POOL_INFO PagedPoolInfo;
|
|
|
|
ULONG Color;
|
|
|
|
ULONG ProcessOutSwapCount;
|
|
|
|
//
|
|
// This is the list of system images currently valid in
|
|
// this session space. This information is in addition
|
|
// to the module global information in PsLoadedModuleList.
|
|
//
|
|
|
|
LIST_ENTRY ImageList;
|
|
|
|
//
|
|
// The system PTE self-map entry.
|
|
//
|
|
|
|
PMMPTE GlobalPteEntry;
|
|
|
|
ULONG CopyOnWriteCount;
|
|
|
|
ULONG SessionPoolAllocationFailures[4];
|
|
|
|
//
|
|
// The count of "known attachers and the associated event.
|
|
//
|
|
|
|
ULONG AttachCount;
|
|
|
|
KEVENT AttachEvent;
|
|
|
|
PEPROCESS LastProcess;
|
|
|
|
//
|
|
// Working set information.
|
|
//
|
|
|
|
MMSUPPORT Vm;
|
|
PMMWSLE Wsle;
|
|
|
|
ERESOURCE WsLock; // owned by WorkingSetLockOwner
|
|
|
|
//
|
|
// This chain is in global system addresses (not session VAs) and can
|
|
// be walked from any system context, ie: for WSL trimming.
|
|
//
|
|
|
|
LIST_ENTRY WsListEntry;
|
|
|
|
//
|
|
// Support for mapping system views into session space. Each desktop
|
|
// allocates a 3MB heap and the global system view space is only 48M
|
|
// total. This would limit us to only 20-30 users - rotating the
|
|
// system view space with each session removes this limitation.
|
|
//
|
|
|
|
MMSESSION Session;
|
|
|
|
//
|
|
// This is the driver object entry for WIN32K.SYS
|
|
//
|
|
// It is not a real driver object, but contained here
|
|
// for information such as the DriverUnload routine.
|
|
//
|
|
|
|
DRIVER_OBJECT Win32KDriverObject;
|
|
|
|
PETHREAD WorkingSetLockOwner;
|
|
|
|
//
|
|
// Pool descriptor for less than 1 page allocations.
|
|
//
|
|
|
|
POOL_DESCRIPTOR PagedPool;
|
|
|
|
//
|
|
// This is generally decremented in process delete (not clean) so that
|
|
// the session data page and mapping PTE can finally be freed when this
|
|
// reaches zero. smss is the only process that decrements it in other
|
|
// places as smss never exits.
|
|
//
|
|
|
|
LONG ProcessReferenceToSession;
|
|
|
|
LCID LocaleId;
|
|
|
|
#if defined (_WIN64)
|
|
|
|
//
|
|
// NT64 has enough virtual address space to support per-session special
|
|
// pool.
|
|
//
|
|
|
|
PMMPTE SpecialPoolFirstPte;
|
|
PMMPTE SpecialPoolLastPte;
|
|
PMMPTE NextPdeForSpecialPoolExpansion;
|
|
PMMPTE LastPdeForSpecialPoolExpansion;
|
|
PFN_NUMBER SpecialPagesInUse;
|
|
#endif
|
|
|
|
#if defined(_IA64_)
|
|
REGION_MAP_INFO SessionMapInfo;
|
|
PFN_NUMBER PageDirectoryParentPage;
|
|
#endif
|
|
|
|
#if DBG
|
|
ULONG Debug[MM_SESS_COUNTER_MAX];
|
|
|
|
MM_SESSION_MEMORY_COUNTERS Debug2[MM_SESS_MEMORY_COUNTER_MAX];
|
|
#endif
|
|
|
|
} MM_SESSION_SPACE, *PMM_SESSION_SPACE;
|
|
|
|
extern PMM_SESSION_SPACE MmSessionSpace;
|
|
|
|
extern ULONG MiSessionCount;
|
|
|
|
//
|
|
// This could be improved to just flush the non-global TB entries.
|
|
//
|
|
|
|
#define MI_FLUSH_SESSION_TB() KeFlushEntireTb (TRUE, TRUE);
|
|
|
|
//
|
|
// The default number of pages for the session working set minimum & maximum.
|
|
//
|
|
|
|
#define MI_SESSION_SPACE_WORKING_SET_MINIMUM 20
|
|
|
|
#define MI_SESSION_SPACE_WORKING_SET_MAXIMUM 384
|
|
|
|
NTSTATUS
|
|
MiSessionCommitPageTables (
|
|
PVOID StartVa,
|
|
PVOID EndVa
|
|
);
|
|
|
|
NTSTATUS
|
|
MiInitializeAndChargePfn (
|
|
OUT PPFN_NUMBER PageFrameIndex,
|
|
IN PMMPTE PointerPde,
|
|
IN PFN_NUMBER ContainingPageFrame,
|
|
IN LOGICAL SessionAllocation
|
|
);
|
|
|
|
VOID
|
|
MiSessionPageTableRelease (
|
|
IN PFN_NUMBER PageFrameIndex
|
|
);
|
|
|
|
NTSTATUS
|
|
MiInitializeSessionPool (
|
|
VOID
|
|
);
|
|
|
|
VOID
|
|
MiCheckSessionPoolAllocations (
|
|
VOID
|
|
);
|
|
|
|
VOID
|
|
MiFreeSessionPoolBitMaps (
|
|
VOID
|
|
);
|
|
|
|
VOID
|
|
MiDetachSession (
|
|
VOID
|
|
);
|
|
|
|
VOID
|
|
MiAttachSession (
|
|
IN PMM_SESSION_SPACE SessionGlobal
|
|
);
|
|
|
|
PVOID
|
|
MiAttachToSecureProcessInSession (
|
|
IN PRKAPC_STATE ApcState
|
|
);
|
|
|
|
VOID
|
|
MiDetachFromSecureProcessInSession (
|
|
IN PVOID OpaqueSession,
|
|
IN PRKAPC_STATE ApcState
|
|
);
|
|
|
|
VOID
|
|
MiReleaseProcessReferenceToSessionDataPage (
|
|
PMM_SESSION_SPACE SessionGlobal
|
|
);
|
|
|
|
#define MM_SET_SESSION_RESOURCE_OWNER(_Thread) \
|
|
ASSERT (MmSessionSpace->WorkingSetLockOwner == NULL); \
|
|
MmSessionSpace->WorkingSetLockOwner = _Thread;
|
|
|
|
#define MM_CLEAR_SESSION_RESOURCE_OWNER() \
|
|
ASSERT (MmSessionSpace->WorkingSetLockOwner == PsGetCurrentThread()); \
|
|
MmSessionSpace->WorkingSetLockOwner = NULL;
|
|
|
|
#define MM_SESSION_SPACE_WS_LOCK_ASSERT() \
|
|
ASSERT (MmSessionSpace->WorkingSetLockOwner == PsGetCurrentThread())
|
|
|
|
#define LOCK_SESSION_SPACE_WS(OLDIRQL,_Thread) \
|
|
ASSERT (KeGetCurrentIrql() <= APC_LEVEL); \
|
|
KeRaiseIrql(APC_LEVEL,&OLDIRQL); \
|
|
ExAcquireResourceExclusiveLite(&MmSessionSpace->WsLock, TRUE); \
|
|
MM_SET_SESSION_RESOURCE_OWNER (_Thread);
|
|
|
|
#define UNLOCK_SESSION_SPACE_WS(OLDIRQL) \
|
|
MM_CLEAR_SESSION_RESOURCE_OWNER (); \
|
|
ExReleaseResourceLite (&MmSessionSpace->WsLock); \
|
|
KeLowerIrql (OLDIRQL); \
|
|
ASSERT (KeGetCurrentIrql() <= APC_LEVEL);
|
|
|
|
extern PMMPTE MiHighestUserPte;
|
|
extern PMMPTE MiHighestUserPde;
|
|
#if (_MI_PAGING_LEVELS >= 4)
|
|
extern PMMPTE MiHighestUserPpe;
|
|
extern PMMPTE MiHighestUserPxe;
|
|
#endif
|
|
|
|
NTSTATUS
|
|
MiEmptyWorkingSet (
|
|
IN PMMSUPPORT WsInfo,
|
|
IN LOGICAL WaitOk
|
|
);
|
|
|
|
//++
|
|
//ULONG
|
|
//MiGetPdeSessionIndex (
|
|
// IN PVOID va
|
|
// );
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// MiGetPdeSessionIndex returns the session structure index for the PDE
|
|
// will (or does) map the given virtual address.
|
|
//
|
|
// Arguments
|
|
//
|
|
// Va - Supplies the virtual address to locate the PDE index for.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// The index of the PDE entry.
|
|
//
|
|
//--
|
|
|
|
#define MiGetPdeSessionIndex(va) ((ULONG)(((ULONG_PTR)(va) - (ULONG_PTR)MmSessionBase) >> PDI_SHIFT))
|
|
|
|
//
|
|
// Session space contains the image loader and tracker, virtual
|
|
// address allocator, paged pool allocator, system view image mappings,
|
|
// and working set for kernel mode virtual addresses that are instanced
|
|
// for groups of processes in a Session process group. This
|
|
// process group is identified by a SessionId.
|
|
//
|
|
// Each Session process group's loaded kernel modules, paged pool
|
|
// allocations, working set, and mapped system views are separate from
|
|
// other Session process groups, even though they have the same
|
|
// virtual addresses.
|
|
//
|
|
// This is to support the Hydra multi-user Windows NT system by
|
|
// replicating WIN32K.SYS, and its complement of video and printer drivers,
|
|
// desktop heaps, memory allocations, etc.
|
|
//
|
|
|
|
//
|
|
// Structure linked into a session space structure to describe
|
|
// which system images in PsLoadedModuleTable and
|
|
// SESSION_DRIVER_GLOBAL_LOAD_ADDRESS's
|
|
// have been allocated for the current session space.
|
|
//
|
|
// The reference count tracks the number of loads of this image within
|
|
// this session.
|
|
//
|
|
|
|
typedef struct _IMAGE_ENTRY_IN_SESSION {
|
|
LIST_ENTRY Link;
|
|
PVOID Address;
|
|
PVOID LastAddress;
|
|
ULONG ImageCountInThisSession;
|
|
PMMPTE PrototypePtes;
|
|
PKLDR_DATA_TABLE_ENTRY DataTableEntry;
|
|
} IMAGE_ENTRY_IN_SESSION, *PIMAGE_ENTRY_IN_SESSION;
|
|
|
|
extern LIST_ENTRY MiSessionWsList;
|
|
|
|
NTSTATUS
|
|
FASTCALL
|
|
MiCheckPdeForSessionSpace(
|
|
IN PVOID VirtualAddress
|
|
);
|
|
|
|
NTSTATUS
|
|
MiShareSessionImage (
|
|
IN PSECTION Section,
|
|
IN OUT PSIZE_T ViewSize
|
|
);
|
|
|
|
VOID
|
|
MiSessionWideInitializeAddresses (
|
|
VOID
|
|
);
|
|
|
|
NTSTATUS
|
|
MiSessionWideReserveImageAddress (
|
|
IN PUNICODE_STRING pImageName,
|
|
IN PSECTION Section,
|
|
IN ULONG_PTR Alignment,
|
|
OUT PVOID *ppAddr,
|
|
OUT PLOGICAL pAlreadyLoaded
|
|
);
|
|
|
|
VOID
|
|
MiInitializeSessionIds (
|
|
VOID
|
|
);
|
|
|
|
VOID
|
|
MiInitializeSessionWsSupport(
|
|
VOID
|
|
);
|
|
|
|
VOID
|
|
MiSessionAddProcess (
|
|
IN PEPROCESS NewProcess
|
|
);
|
|
|
|
VOID
|
|
MiSessionRemoveProcess (
|
|
VOID
|
|
);
|
|
|
|
NTSTATUS
|
|
MiRemovePsLoadedModule(
|
|
PKLDR_DATA_TABLE_ENTRY DataTableEntry
|
|
);
|
|
|
|
NTSTATUS
|
|
MiRemoveImageSessionWide(
|
|
IN PVOID BaseAddr
|
|
);
|
|
|
|
NTSTATUS
|
|
MiDeleteSessionVirtualAddresses(
|
|
IN PVOID VirtualAddress,
|
|
IN SIZE_T NumberOfBytes
|
|
);
|
|
|
|
NTSTATUS
|
|
MiUnloadSessionImageByForce (
|
|
IN SIZE_T NumberOfPtes,
|
|
IN PVOID ImageBase
|
|
);
|
|
|
|
NTSTATUS
|
|
MiSessionWideGetImageSize(
|
|
IN PVOID BaseAddress,
|
|
OUT PSIZE_T NumberOfBytes OPTIONAL,
|
|
OUT PSIZE_T CommitPages OPTIONAL
|
|
);
|
|
|
|
PIMAGE_ENTRY_IN_SESSION
|
|
MiSessionLookupImage (
|
|
IN PVOID BaseAddress
|
|
);
|
|
|
|
NTSTATUS
|
|
MiSessionCommitImagePages(
|
|
PVOID BaseAddr,
|
|
SIZE_T Size
|
|
);
|
|
|
|
VOID
|
|
MiSessionUnloadAllImages (
|
|
VOID
|
|
);
|
|
|
|
VOID
|
|
MiFreeSessionSpaceMap (
|
|
VOID
|
|
);
|
|
|
|
NTSTATUS
|
|
MiSessionInitializeWorkingSetList (
|
|
VOID
|
|
);
|
|
|
|
VOID
|
|
MiSessionUnlinkWorkingSet (
|
|
VOID
|
|
);
|
|
|
|
NTSTATUS
|
|
MiSessionCopyOnWrite (
|
|
IN PVOID FaultingAddress,
|
|
IN PMMPTE PointerPte
|
|
);
|
|
|
|
VOID
|
|
MiSessionOutSwapProcess (
|
|
IN PEPROCESS Process
|
|
);
|
|
|
|
VOID
|
|
MiSessionInSwapProcess (
|
|
IN PEPROCESS Process
|
|
);
|
|
|
|
#if !defined (_X86PAE_)
|
|
|
|
#define MI_GET_DIRECTORY_FRAME_FROM_PROCESS(_Process) \
|
|
MI_GET_PAGE_FRAME_FROM_PTE((PMMPTE)(&((_Process)->Pcb.DirectoryTableBase[0])))
|
|
|
|
#define MI_GET_HYPER_PAGE_TABLE_FRAME_FROM_PROCESS(_Process) \
|
|
MI_GET_PAGE_FRAME_FROM_PTE((PMMPTE)(&((_Process)->Pcb.DirectoryTableBase[1])))
|
|
|
|
#else
|
|
|
|
#define MI_GET_DIRECTORY_FRAME_FROM_PROCESS(_Process) \
|
|
(MI_GET_PAGE_FRAME_FROM_PTE(((PMMPTE)((_Process)->PaeTop)) + PD_PER_SYSTEM - 1))
|
|
|
|
#define MI_GET_HYPER_PAGE_TABLE_FRAME_FROM_PROCESS(_Process) \
|
|
((PFN_NUMBER)((_Process)->Pcb.DirectoryTableBase[1]))
|
|
|
|
#endif
|
|
|
|
#if defined(_MIALT4K_)
|
|
NTSTATUS
|
|
MiSetCopyPagesFor4kPage (
|
|
IN PEPROCESS Process,
|
|
IN PMMVAD Vad,
|
|
IN OUT PVOID StartingAddress,
|
|
IN OUT PVOID EndingAddress,
|
|
IN ULONG ProtectionMask,
|
|
OUT PMMVAD *CallerNewVad
|
|
);
|
|
|
|
VOID
|
|
MiRemoveAliasedVads (
|
|
IN PEPROCESS Process,
|
|
IN PMMVAD Vad
|
|
);
|
|
|
|
PVOID
|
|
MiDuplicateAliasVadList (
|
|
IN PMMVAD Vad
|
|
);
|
|
#endif
|
|
|
|
//
|
|
// The LDR_DATA_TABLE_ENTRY->LoadedImports is used as a list of imported DLLs.
|
|
//
|
|
// This field is zero if the module was loaded at boot time and the
|
|
// import information was never filled in.
|
|
//
|
|
// This field is -1 if no imports are defined by the module.
|
|
//
|
|
// This field contains a valid paged pool PLDR_DATA_TABLE_ENTRY pointer
|
|
// with a low-order (bit 0) tag of 1 if there is only 1 usable import needed
|
|
// by this driver.
|
|
//
|
|
// This field will contain a valid paged pool PLOAD_IMPORTS pointer in all
|
|
// other cases (ie: where at least 2 imports exist).
|
|
//
|
|
|
|
typedef struct _LOAD_IMPORTS {
|
|
SIZE_T Count;
|
|
PKLDR_DATA_TABLE_ENTRY Entry[1];
|
|
} LOAD_IMPORTS, *PLOAD_IMPORTS;
|
|
|
|
#define LOADED_AT_BOOT ((PLOAD_IMPORTS)0)
|
|
#define NO_IMPORTS_USED ((PLOAD_IMPORTS)-2)
|
|
|
|
#define SINGLE_ENTRY(ImportVoid) ((ULONG)((ULONG_PTR)(ImportVoid) & 0x1))
|
|
|
|
#define SINGLE_ENTRY_TO_POINTER(ImportVoid) ((PKLDR_DATA_TABLE_ENTRY)((ULONG_PTR)(ImportVoid) & ~0x1))
|
|
|
|
#define POINTER_TO_SINGLE_ENTRY(Pointer) ((PKLDR_DATA_TABLE_ENTRY)((ULONG_PTR)(Pointer) | 0x1))
|
|
|
|
//
|
|
// This tracks allocated group virtual addresses. The term SESSIONWIDE is used
|
|
// to denote data that is the same across all sessions (as opposed to
|
|
// per-session data which can vary from session to session).
|
|
//
|
|
// Since each driver loaded into a session space is linked and fixed up
|
|
// against the system image, it must remain at the same virtual address
|
|
// across the system regardless of the session.
|
|
//
|
|
// A list is maintained by the group allocator of which virtual
|
|
// addresses are in use and by which DLL.
|
|
//
|
|
// The reference count tracks the number of sessions that have loaded
|
|
// this image.
|
|
//
|
|
// Access to this structure is guarded by the MmSystemLoadLock.
|
|
//
|
|
|
|
typedef struct _SESSIONWIDE_DRIVER_ADDRESS {
|
|
LIST_ENTRY Link;
|
|
ULONG ReferenceCount;
|
|
PVOID Address;
|
|
ULONG_PTR Size;
|
|
ULONG_PTR WritablePages;
|
|
UNICODE_STRING FullDllName;
|
|
} SESSIONWIDE_DRIVER_ADDRESS, *PSESSIONWIDE_DRIVER_ADDRESS;
|
|
|
|
// #define _MI_DEBUG_RONLY 1 // Uncomment this for session readonly tracking
|
|
|
|
#if _MI_DEBUG_RONLY
|
|
|
|
VOID
|
|
MiAssertNotSessionData (
|
|
IN PMMPTE PointerPte
|
|
);
|
|
|
|
VOID
|
|
MiLogSessionDataStart (
|
|
IN PKLDR_DATA_TABLE_ENTRY DataTableEntry
|
|
);
|
|
|
|
#define MI_ASSERT_NOT_SESSION_DATA(PTE) MiAssertNotSessionData(PTE)
|
|
#define MI_LOG_SESSION_DATA_START(DataTableEntry) MiLogSessionDataStart(DataTableEntry)
|
|
#else
|
|
#define MI_ASSERT_NOT_SESSION_DATA(PTE)
|
|
#define MI_LOG_SESSION_DATA_START(DataTableEntry)
|
|
#endif
|
|
|
|
//
|
|
// This tracks driver-specified individual verifier thunks.
|
|
//
|
|
|
|
typedef struct _DRIVER_SPECIFIED_VERIFIER_THUNKS {
|
|
LIST_ENTRY ListEntry;
|
|
PKLDR_DATA_TABLE_ENTRY DataTableEntry;
|
|
ULONG NumberOfThunks;
|
|
} DRIVER_SPECIFIED_VERIFIER_THUNKS, *PDRIVER_SPECIFIED_VERIFIER_THUNKS;
|
|
|
|
// #define _MI_DEBUG_SUB 1 // Uncomment this for subsection logging
|
|
|
|
#if defined (_MI_DEBUG_SUB)
|
|
|
|
extern ULONG MiTrackSubs;
|
|
|
|
#define MI_SUB_BACKTRACE_LENGTH 8
|
|
|
|
typedef struct _MI_SUB_TRACES {
|
|
|
|
PETHREAD Thread;
|
|
PMSUBSECTION Subsection;
|
|
PCONTROL_AREA ControlArea;
|
|
ULONG_PTR CallerId;
|
|
PVOID StackTrace [MI_SUB_BACKTRACE_LENGTH];
|
|
|
|
MSUBSECTION SubsectionContents;
|
|
CONTROL_AREA ControlAreaContents;
|
|
|
|
} MI_SUB_TRACES, *PMI_SUB_TRACES;
|
|
|
|
extern LONG MiSubsectionIndex;
|
|
|
|
extern PMI_SUB_TRACES MiSubsectionTraces;
|
|
|
|
VOID
|
|
FORCEINLINE
|
|
MiSnapSubsection (
|
|
IN PMSUBSECTION Subsection,
|
|
IN ULONG CallerId
|
|
)
|
|
{
|
|
PMI_SUB_TRACES Information;
|
|
PCONTROL_AREA ControlArea;
|
|
ULONG Index;
|
|
ULONG Hash;
|
|
|
|
if (MiSubsectionTraces == NULL) {
|
|
return;
|
|
}
|
|
|
|
ControlArea = Subsection->ControlArea;
|
|
|
|
#if 0
|
|
if (ControlArea->u.Flags.Mft == 0) {
|
|
return;
|
|
}
|
|
#endif
|
|
|
|
Index = InterlockedIncrement(&MiSubsectionIndex);
|
|
Index &= (MiTrackSubs - 1);
|
|
Information = &MiSubsectionTraces[Index];
|
|
|
|
Information->Subsection = Subsection;
|
|
Information->ControlArea = ControlArea;
|
|
*(PMSUBSECTION)&Information->SubsectionContents = *Subsection;
|
|
*(PCONTROL_AREA)&Information->ControlAreaContents = *ControlArea;
|
|
Information->Thread = PsGetCurrentThread();
|
|
Information->CallerId = CallerId;
|
|
RtlCaptureStackBackTrace (0, MI_SUB_BACKTRACE_LENGTH, Information->StackTrace, &Hash);
|
|
}
|
|
|
|
#define MI_SNAP_SUB(_Sub, callerid) MiSnapSubsection(_Sub, callerid)
|
|
|
|
#else
|
|
#define MI_SNAP_SUB(_Sub, callerid)
|
|
#endif
|
|
|
|
#ifdef _MI_MESSAGE_SERVER
|
|
LOGICAL
|
|
MiQueueMessage (
|
|
IN PVOID Message
|
|
);
|
|
|
|
PVOID
|
|
MiRemoveMessage (
|
|
VOID
|
|
);
|
|
|
|
#define MI_INSTRUMENT_QUEUE(Message) MiQueueMessage (Message)
|
|
#define MI_INSTRUMENTR_QUEUE() MiRemoveMessage ()
|
|
|
|
#else
|
|
|
|
#define MI_INSTRUMENT_QUEUE(Message)
|
|
#define MI_INSTRUMENTR_QUEUE()
|
|
|
|
#endif
|
|
|
|
#endif // MI
|