Source code of Windows XP (NT5)
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

7483 lines
186 KiB

  1. /*++
  2. Copyright (c) 1989 Microsoft Corporation
  3. Module Name:
  4. mi.h
  5. Abstract:
  6. This module contains the private data structures and procedure
  7. prototypes for the memory management system.
  8. Author:
  9. Lou Perazzoli (loup) 20-Mar-1989
  10. Landy Wang (landyw) 02-Jun-1997
  11. Revision History:
  12. --*/
  13. #ifndef _MI_
  14. #define _MI_
  15. #pragma warning(disable:4214) // bit field types other than int
  16. #pragma warning(disable:4201) // nameless struct/union
  17. #pragma warning(disable:4324) // alignment sensitive to declspec
  18. #pragma warning(disable:4127) // condition expression is constant
  19. #pragma warning(disable:4115) // named type definition in parentheses
  20. #pragma warning(disable:4232) // dllimport not static
  21. #pragma warning(disable:4206) // translation unit empty
  22. #include "ntos.h"
  23. #include "ntimage.h"
  24. #include "ki.h"
  25. #include "fsrtl.h"
  26. #include "zwapi.h"
  27. #include "pool.h"
  28. #include "stdio.h"
  29. #include "string.h"
  30. #include "safeboot.h"
  31. #include "triage.h"
  32. #include "xip.h"
  33. #if defined(_X86_)
  34. #include "..\mm\i386\mi386.h"
  35. #elif defined(_AMD64_)
  36. #include "..\mm\amd64\miamd.h"
  37. #elif defined(_IA64_)
  38. #include "..\mm\ia64\miia64.h"
  39. #else
  40. #error "mm: a target architecture must be defined."
  41. #endif
  42. #if defined (_WIN64)
  43. #define ASSERT32(exp)
  44. #define ASSERT64(exp) ASSERT(exp)
  45. #else
  46. #define ASSERT32(exp) ASSERT(exp)
  47. #define ASSERT64(exp)
  48. #endif
  49. //
  50. // Special pool constants
  51. //
  52. #define MI_SPECIAL_POOL_PAGABLE 0x8000
  53. #define MI_SPECIAL_POOL_VERIFIER 0x4000
  54. #define MI_SPECIAL_POOL_IN_SESSION 0x2000
  55. #define MI_SPECIAL_POOL_PTE_PAGABLE 0x0002
  56. #define MI_SPECIAL_POOL_PTE_NONPAGABLE 0x0004
  57. #define _2gb 0x80000000 // 2 gigabytes
  58. #define _4gb 0x100000000 // 4 gigabytes
  59. #define MM_FLUSH_COUNTER_MASK (0xFFFFF)
  60. #define MM_FREE_WSLE_SHIFT 4
  61. #define WSLE_NULL_INDEX ((((WSLE_NUMBER)-1) >> MM_FREE_WSLE_SHIFT))
  62. #define MM_FREE_POOL_SIGNATURE (0x50554F4C)
  63. #define MM_MINIMUM_PAGED_POOL_NTAS ((SIZE_T)(48*1024*1024))
  64. #define MM_ALLOCATION_FILLS_VAD ((PMMPTE)(ULONG_PTR)~3)
  65. #define MM_WORKING_SET_LIST_SEARCH 17
  66. #define MM_FLUID_WORKING_SET 8
  67. #define MM_FLUID_PHYSICAL_PAGES 32 //see MmResidentPages below.
  68. #define MM_USABLE_PAGES_FREE 32
  69. #define X64K (ULONG)65536
  70. #define MM_HIGHEST_VAD_ADDRESS ((PVOID)((ULONG_PTR)MM_HIGHEST_USER_ADDRESS - (64 * 1024)))
  71. #define MM_NO_WS_EXPANSION ((PLIST_ENTRY)0)
  72. #define MM_WS_EXPANSION_IN_PROGRESS ((PLIST_ENTRY)35)
  73. #define MM_WS_SWAPPED_OUT ((PLIST_ENTRY)37)
  74. #define MM_IO_IN_PROGRESS ((PLIST_ENTRY)97) // MUST HAVE THE HIGHEST VALUE
  75. #define MM4K_SHIFT 12 //MUST BE LESS THAN OR EQUAL TO PAGE_SHIFT
  76. #define MM4K_MASK 0xfff
  77. #define MMSECTOR_SHIFT 9 //MUST BE LESS THAN OR EQUAL TO PAGE_SHIFT
  78. #define MMSECTOR_MASK 0x1ff
  79. #define MM_LOCK_BY_REFCOUNT 0
  80. #define MM_LOCK_BY_NONPAGE 1
  81. #define MM_FORCE_TRIM 6
  82. #define MM_GROW_WSLE_HASH 20
  83. #define MM_MAXIMUM_WRITE_CLUSTER (MM_MAXIMUM_DISK_IO_SIZE / PAGE_SIZE)
  84. //
  85. // Number of PTEs to flush singularly before flushing the entire TB.
  86. //
  87. #define MM_MAXIMUM_FLUSH_COUNT (FLUSH_MULTIPLE_MAXIMUM-1)
  88. //
  89. // Page protections
  90. //
  91. #define MM_ZERO_ACCESS 0 // this value is not used.
  92. #define MM_READONLY 1
  93. #define MM_EXECUTE 2
  94. #define MM_EXECUTE_READ 3
  95. #define MM_READWRITE 4 // bit 2 is set if this is writable.
  96. #define MM_WRITECOPY 5
  97. #define MM_EXECUTE_READWRITE 6
  98. #define MM_EXECUTE_WRITECOPY 7
  99. #define MM_NOCACHE 0x8
  100. #define MM_GUARD_PAGE 0x10
  101. #define MM_DECOMMIT 0x10 //NO_ACCESS, Guard page
  102. #define MM_NOACCESS 0x18 //NO_ACCESS, Guard_page, nocache.
  103. #define MM_UNKNOWN_PROTECTION 0x100 //bigger than 5 bits!
  104. #define MM_LARGE_PAGES 0x111
  105. #define MM_INVALID_PROTECTION ((ULONG)-1) //bigger than 5 bits!
  106. #define MM_KSTACK_OUTSWAPPED 0x1F // Denotes outswapped kernel stack pages.
  107. #define MM_PROTECTION_WRITE_MASK 4
  108. #define MM_PROTECTION_COPY_MASK 1
  109. #define MM_PROTECTION_OPERATION_MASK 7 // mask off guard page and nocache.
  110. #define MM_PROTECTION_EXECUTE_MASK 2
  111. #define MM_SECURE_DELETE_CHECK 0x55
  112. //
  113. // Debug flags
  114. //
  115. #define MM_DBG_WRITEFAULT 0x1
  116. #define MM_DBG_PTE_UPDATE 0x2
  117. #define MM_DBG_DUMP_WSL 0x4
  118. #define MM_DBG_PAGEFAULT 0x8
  119. #define MM_DBG_WS_EXPANSION 0x10
  120. #define MM_DBG_MOD_WRITE 0x20
  121. #define MM_DBG_CHECK_PTE 0x40
  122. #define MM_DBG_VAD_CONFLICT 0x80
  123. #define MM_DBG_SECTIONS 0x100
  124. #define MM_DBG_SYS_PTES 0x400
  125. #define MM_DBG_CLEAN_PROCESS 0x800
  126. #define MM_DBG_COLLIDED_PAGE 0x1000
  127. #define MM_DBG_DUMP_BOOT_PTES 0x2000
  128. #define MM_DBG_FORK 0x4000
  129. #define MM_DBG_DIR_BASE 0x8000
  130. #define MM_DBG_FLUSH_SECTION 0x10000
  131. #define MM_DBG_PRINTS_MODWRITES 0x20000
  132. #define MM_DBG_PAGE_IN_LIST 0x40000
  133. #define MM_DBG_CHECK_PFN_LOCK 0x80000
  134. #define MM_DBG_PRIVATE_PAGES 0x100000
  135. #define MM_DBG_WALK_VAD_TREE 0x200000
  136. #define MM_DBG_SWAP_PROCESS 0x400000
  137. #define MM_DBG_LOCK_CODE 0x800000
  138. #define MM_DBG_STOP_ON_ACCVIO 0x1000000
  139. #define MM_DBG_PAGE_REF_COUNT 0x2000000
  140. #define MM_DBG_SHOW_FAULTS 0x40000000
  141. #define MM_DBG_SESSIONS 0x80000000
  142. //
  143. // If the PTE.protection & MM_COPY_ON_WRITE_MASK == MM_COPY_ON_WRITE_MASK
  144. // then the PTE is copy on write.
  145. //
  146. #define MM_COPY_ON_WRITE_MASK 5
  147. extern ULONG MmProtectToValue[32];
  148. extern
  149. #if (defined(_WIN64) || defined(_X86PAE_))
  150. ULONGLONG
  151. #else
  152. ULONG
  153. #endif
  154. MmProtectToPteMask[32];
  155. extern ULONG MmMakeProtectNotWriteCopy[32];
  156. extern ACCESS_MASK MmMakeSectionAccess[8];
  157. extern ACCESS_MASK MmMakeFileAccess[8];
  158. //
  159. // Time constants
  160. //
  161. extern const LARGE_INTEGER MmSevenMinutes;
  162. extern LARGE_INTEGER MmWorkingSetProtectionTime;
  163. const extern LARGE_INTEGER MmOneSecond;
  164. const extern LARGE_INTEGER MmTwentySeconds;
  165. const extern LARGE_INTEGER MmShortTime;
  166. const extern LARGE_INTEGER MmHalfSecond;
  167. const extern LARGE_INTEGER Mm30Milliseconds;
  168. extern LARGE_INTEGER MmCriticalSectionTimeout;
  169. //
  170. // A month's worth
  171. //
  172. extern ULONG MmCritsectTimeoutSeconds;
  173. //
  174. // this is the csrss process !
  175. //
  176. extern PEPROCESS ExpDefaultErrorPortProcess;
  177. extern SIZE_T MmExtendedCommit;
  178. extern SIZE_T MmTotalProcessCommit;
  179. //
  180. // The total number of pages needed for the loader to successfully hibernate.
  181. //
  182. extern PFN_NUMBER MmHiberPages;
  183. //
  184. // The counters and reasons to retry IO to protect against verifier induced
  185. // failures and temporary conditions.
  186. //
  187. extern ULONG MiIoRetryMask;
  188. extern ULONG MiFaultRetryMask;
  189. extern ULONG MiUserFaultRetryMask;
  190. #define MmIsRetryIoStatus(S) (((S) == STATUS_INSUFFICIENT_RESOURCES) || \
  191. ((S) == STATUS_WORKING_SET_QUOTA) || \
  192. ((S) == STATUS_NO_MEMORY))
  193. #if defined (_MI_MORE_THAN_4GB_)
  194. extern PFN_NUMBER MiNoLowMemory;
  195. #if defined (_WIN64)
  196. #define MI_MAGIC_4GB_RECLAIM 0xffffedf0
  197. #else
  198. #define MI_MAGIC_4GB_RECLAIM 0xffedf0
  199. #endif
  200. #define MI_LOWMEM_MAGIC_BIT (0x80000000)
  201. extern PRTL_BITMAP MiLowMemoryBitMap;
  202. #endif
  203. //
  204. // This is a version of COMPUTE_PAGES_SPANNED that works for 32 and 64 ranges.
  205. //
  206. #define MI_COMPUTE_PAGES_SPANNED(Va, Size) \
  207. ((((ULONG_PTR)(Va) & (PAGE_SIZE -1)) + (Size) + (PAGE_SIZE - 1)) >> PAGE_SHIFT)
  208. //++
  209. //
  210. // ULONG
  211. // MI_CONVERT_FROM_PTE_PROTECTION (
  212. // IN ULONG PROTECTION_MASK
  213. // )
  214. //
  215. // Routine Description:
  216. //
  217. // This routine converts a PTE protection into a Protect value.
  218. //
  219. // Arguments:
  220. //
  221. //
  222. // Return Value:
  223. //
  224. // Returns the
  225. //
  226. //--
  227. #define MI_CONVERT_FROM_PTE_PROTECTION(PROTECTION_MASK) \
  228. (MmProtectToValue[PROTECTION_MASK])
  229. #define MI_MASK_TO_PTE(PROTECTION_MASK) MmProtectToPteMask[PROTECTION_MASK]
  230. #define MI_IS_PTE_PROTECTION_COPY_WRITE(PROTECTION_MASK) \
  231. (((PROTECTION_MASK) & MM_COPY_ON_WRITE_MASK) == MM_COPY_ON_WRITE_MASK)
  232. //++
  233. //
  234. // ULONG
  235. // MI_ROUND_TO_64K (
  236. // IN ULONG LENGTH
  237. // )
  238. //
  239. // Routine Description:
  240. //
  241. //
  242. // The ROUND_TO_64k macro takes a LENGTH in bytes and rounds it up to a multiple
  243. // of 64K.
  244. //
  245. // Arguments:
  246. //
  247. // LENGTH - LENGTH in bytes to round up to 64k.
  248. //
  249. // Return Value:
  250. //
  251. // Returns the LENGTH rounded up to a multiple of 64k.
  252. //
  253. //--
  254. #define MI_ROUND_TO_64K(LENGTH) (((LENGTH) + X64K - 1) & ~((ULONG_PTR)X64K - 1))
  255. extern ULONG MiLastVadBit;
  256. //++
  257. //
  258. // ULONG
  259. // MI_ROUND_TO_SIZE (
  260. // IN ULONG LENGTH,
  261. // IN ULONG ALIGNMENT
  262. // )
  263. //
  264. // Routine Description:
  265. //
  266. //
  267. // The ROUND_TO_SIZE macro takes a LENGTH in bytes and rounds it up to a
  268. // multiple of the alignment.
  269. //
  270. // Arguments:
  271. //
  272. // LENGTH - LENGTH in bytes to round up to.
  273. //
  274. // ALIGNMENT - alignment to round to, must be a power of 2, e.g, 2**n.
  275. //
  276. // Return Value:
  277. //
  278. // Returns the LENGTH rounded up to a multiple of the alignment.
  279. //
  280. //--
  281. #define MI_ROUND_TO_SIZE(LENGTH,ALIGNMENT) \
  282. (((LENGTH) + ((ALIGNMENT) - 1)) & ~((ALIGNMENT) - 1))
  283. //++
  284. //
  285. // PVOID
  286. // MI_64K_ALIGN (
  287. // IN PVOID VA
  288. // )
  289. //
  290. // Routine Description:
  291. //
  292. //
  293. // The MI_64K_ALIGN macro takes a virtual address and returns a 64k-aligned
  294. // virtual address for that page.
  295. //
  296. // Arguments:
  297. //
  298. // VA - Virtual address.
  299. //
  300. // Return Value:
  301. //
  302. // Returns the 64k aligned virtual address.
  303. //
  304. //--
  305. #define MI_64K_ALIGN(VA) ((PVOID)((ULONG_PTR)(VA) & ~((LONG)X64K - 1)))
  306. //++
  307. //
  308. // PVOID
  309. // MI_ALIGN_TO_SIZE (
  310. // IN PVOID VA
  311. // IN ULONG ALIGNMENT
  312. // )
  313. //
  314. // Routine Description:
  315. //
  316. //
  317. // The MI_ALIGN_TO_SIZE macro takes a virtual address and returns a
  318. // virtual address for that page with the specified alignment.
  319. //
  320. // Arguments:
  321. //
  322. // VA - Virtual address.
  323. //
  324. // ALIGNMENT - alignment to round to, must be a power of 2, e.g, 2**n.
  325. //
  326. // Return Value:
  327. //
  328. // Returns the aligned virtual address.
  329. //
  330. //--
  331. #define MI_ALIGN_TO_SIZE(VA,ALIGNMENT) ((PVOID)((ULONG_PTR)(VA) & ~((ULONG_PTR) ALIGNMENT - 1)))
  332. //++
  333. //
  334. // LONGLONG
  335. // MI_STARTING_OFFSET (
  336. // IN PSUBSECTION SUBSECT
  337. // IN PMMPTE PTE
  338. // )
  339. //
  340. // Routine Description:
  341. //
  342. // This macro takes a pointer to a PTE within a subsection and a pointer
  343. // to that subsection and calculates the offset for that PTE within the
  344. // file.
  345. //
  346. // Arguments:
  347. //
  348. // PTE - PTE within subsection.
  349. //
  350. // SUBSECT - Subsection
  351. //
  352. // Return Value:
  353. //
  354. // Offset for issuing I/O from.
  355. //
  356. //--
  357. #define MI_STARTING_OFFSET(SUBSECT,PTE) \
  358. (((LONGLONG)((ULONG_PTR)((PTE) - ((SUBSECT)->SubsectionBase))) << PAGE_SHIFT) + \
  359. ((LONGLONG)((SUBSECT)->StartingSector) << MMSECTOR_SHIFT));
  360. // NTSTATUS
  361. // MiFindEmptyAddressRangeDown (
  362. // IN ULONG_PTR SizeOfRange,
  363. // IN PVOID HighestAddressToEndAt,
  364. // IN ULONG_PTR Alignment,
  365. // OUT PVOID *Base
  366. // )
  367. //
  368. //
  369. // Routine Description:
  370. //
  371. // The function examines the virtual address descriptors to locate
  372. // an unused range of the specified size and returns the starting
  373. // address of the range. This routine looks from the top down.
  374. //
  375. // Arguments:
  376. //
  377. // SizeOfRange - Supplies the size in bytes of the range to locate.
  378. //
  379. // HighestAddressToEndAt - Supplies the virtual address to begin looking
  380. // at.
  381. //
  382. // Alignment - Supplies the alignment for the address. Must be
  383. // a power of 2 and greater than the page_size.
  384. //
  385. //Return Value:
  386. //
  387. // Returns the starting address of a suitable range.
  388. //
  389. #define MiFindEmptyAddressRangeDown(Root,SizeOfRange,HighestAddressToEndAt,Alignment,Base) \
  390. (MiFindEmptyAddressRangeDownTree( \
  391. (SizeOfRange), \
  392. (HighestAddressToEndAt), \
  393. (Alignment), \
  394. (PMMADDRESS_NODE)(Root), \
  395. (Base)))
  396. // PMMVAD
  397. // MiGetPreviousVad (
  398. // IN PMMVAD Vad
  399. // )
  400. //
  401. // Routine Description:
  402. //
  403. // This function locates the virtual address descriptor which contains
  404. // the address range which logically precedes the specified virtual
  405. // address descriptor.
  406. //
  407. // Arguments:
  408. //
  409. // Vad - Supplies a pointer to a virtual address descriptor.
  410. //
  411. // Return Value:
  412. //
  413. // Returns a pointer to the virtual address descriptor containing the
  414. // next address range, NULL if none.
  415. //
  416. //
  417. #define MiGetPreviousVad(VAD) ((PMMVAD)MiGetPreviousNode((PMMADDRESS_NODE)(VAD)))
  418. // PMMVAD
  419. // MiGetNextVad (
  420. // IN PMMVAD Vad
  421. // )
  422. //
  423. // Routine Description:
  424. //
  425. // This function locates the virtual address descriptor which contains
  426. // the address range which logically follows the specified address range.
  427. //
  428. // Arguments:
  429. //
  430. // VAD - Supplies a pointer to a virtual address descriptor.
  431. //
  432. // Return Value:
  433. //
  434. // Returns a pointer to the virtual address descriptor containing the
  435. // next address range, NULL if none.
  436. //
  437. #define MiGetNextVad(VAD) ((PMMVAD)MiGetNextNode((PMMADDRESS_NODE)(VAD)))
  438. // PMMVAD
  439. // MiGetFirstVad (
  440. // Process
  441. // )
  442. //
  443. // Routine Description:
  444. //
  445. // This function locates the virtual address descriptor which contains
  446. // the address range which logically is first within the address space.
  447. //
  448. // Arguments:
  449. //
  450. // Process - Specifies the process in which to locate the VAD.
  451. //
  452. // Return Value:
  453. //
  454. // Returns a pointer to the virtual address descriptor containing the
  455. // first address range, NULL if none.
  456. #define MiGetFirstVad(Process) \
  457. ((PMMVAD)MiGetFirstNode((PMMADDRESS_NODE)(Process->VadRoot)))
  458. LOGICAL
  459. MiCheckForConflictingVadExistence (
  460. IN PEPROCESS Process,
  461. IN PVOID StartingAddress,
  462. IN PVOID EndingAddress
  463. );
  464. // PMMVAD
  465. // MiCheckForConflictingVad (
  466. // IN PVOID StartingAddress,
  467. // IN PVOID EndingAddress
  468. // )
  469. //
  470. // Routine Description:
  471. //
  472. // The function determines if any addresses between a given starting and
  473. // ending address is contained within a virtual address descriptor.
  474. //
  475. // Arguments:
  476. //
  477. // StartingAddress - Supplies the virtual address to locate a containing
  478. // descriptor.
  479. //
  480. // EndingAddress - Supplies the virtual address to locate a containing
  481. // descriptor.
  482. //
  483. // Return Value:
  484. //
  485. // Returns a pointer to the first conflicting virtual address descriptor
  486. // if one is found, otherwise a NULL value is returned.
  487. //
  488. #define MiCheckForConflictingVad(CurrentProcess,StartingAddress,EndingAddress) \
  489. ((PMMVAD)MiCheckForConflictingNode( \
  490. MI_VA_TO_VPN(StartingAddress), \
  491. MI_VA_TO_VPN(EndingAddress), \
  492. (PMMADDRESS_NODE)(CurrentProcess->VadRoot)))
  493. // PMMCLONE_DESCRIPTOR
  494. // MiGetNextClone (
  495. // IN PMMCLONE_DESCRIPTOR Clone
  496. // )
  497. //
  498. // Routine Description:
  499. //
  500. // This function locates the virtual address descriptor which contains
  501. // the address range which logically follows the specified address range.
  502. //
  503. // Arguments:
  504. //
  505. // Clone - Supplies a pointer to a virtual address descriptor.
  506. //
  507. // Return Value:
  508. //
  509. // Returns a pointer to the virtual address descriptor containing the
  510. // next address range, NULL if none.
  511. //
  512. //
  513. #define MiGetNextClone(CLONE) \
  514. ((PMMCLONE_DESCRIPTOR)MiGetNextNode((PMMADDRESS_NODE)(CLONE)))
  515. // PMMCLONE_DESCRIPTOR
  516. // MiGetPreviousClone (
  517. // IN PMMCLONE_DESCRIPTOR Clone
  518. // )
  519. //
  520. // Routine Description:
  521. //
  522. // This function locates the virtual address descriptor which contains
  523. // the address range which logically precedes the specified virtual
  524. // address descriptor.
  525. //
  526. // Arguments:
  527. //
  528. // Clone - Supplies a pointer to a virtual address descriptor.
  529. //
  530. // Return Value:
  531. //
  532. // Returns a pointer to the virtual address descriptor containing the
  533. // next address range, NULL if none.
  534. #define MiGetPreviousClone(CLONE) \
  535. ((PMMCLONE_DESCRIPTOR)MiGetPreviousNode((PMMADDRESS_NODE)(CLONE)))
  536. // PMMCLONE_DESCRIPTOR
  537. // MiGetFirstClone (
  538. // )
  539. //
  540. // Routine Description:
  541. //
  542. // This function locates the virtual address descriptor which contains
  543. // the address range which logically is first within the address space.
  544. //
  545. // Arguments:
  546. //
  547. // None.
  548. //
  549. // Return Value:
  550. //
  551. // Returns a pointer to the virtual address descriptor containing the
  552. // first address range, NULL if none.
  553. //
  554. #define MiGetFirstClone(_CurrentProcess) \
  555. ((PMMCLONE_DESCRIPTOR)MiGetFirstNode((PMMADDRESS_NODE)(_CurrentProcess->CloneRoot)))
  556. // VOID
  557. // MiInsertClone (
  558. // IN PMMCLONE_DESCRIPTOR Clone
  559. // )
  560. //
  561. // Routine Description:
  562. //
  563. // This function inserts a virtual address descriptor into the tree and
  564. // reorders the splay tree as appropriate.
  565. //
  566. // Arguments:
  567. //
  568. // Clone - Supplies a pointer to a virtual address descriptor
  569. //
  570. //
  571. // Return Value:
  572. //
  573. // None.
  574. //
  575. #define MiInsertClone(_CurrentProcess, CLONE) \
  576. { \
  577. ASSERT ((CLONE)->NumberOfPtes != 0); \
  578. MiInsertNode(((PMMADDRESS_NODE)(CLONE)),(PMMADDRESS_NODE *)&(_CurrentProcess->CloneRoot)); \
  579. }
  580. // VOID
  581. // MiRemoveClone (
  582. // IN PMMCLONE_DESCRIPTOR Clone
  583. // )
  584. //
  585. // Routine Description:
  586. //
  587. // This function removes a virtual address descriptor from the tree and
  588. // reorders the splay tree as appropriate.
  589. //
  590. // Arguments:
  591. //
  592. // Clone - Supplies a pointer to a virtual address descriptor.
  593. //
  594. // Return Value:
  595. //
  596. // None.
  597. //
  598. #define MiRemoveClone(_CurrentProcess, CLONE) \
  599. MiRemoveNode((PMMADDRESS_NODE)(CLONE),(PMMADDRESS_NODE *)&(_CurrentProcess->CloneRoot));
  600. // PMMCLONE_DESCRIPTOR
  601. // MiLocateCloneAddress (
  602. // IN PVOID VirtualAddress
  603. // )
  604. //
  605. // /*++
  606. //
  607. // Routine Description:
  608. //
  609. // The function locates the virtual address descriptor which describes
  610. // a given address.
  611. //
  612. // Arguments:
  613. //
  614. // VirtualAddress - Supplies the virtual address to locate a descriptor
  615. // for.
  616. //
  617. // Return Value:
  618. //
  619. // Returns a pointer to the virtual address descriptor which contains
  620. // the supplied virtual address or NULL if none was located.
  621. //
  622. #define MiLocateCloneAddress(_CurrentProcess, VA) \
  623. (_CurrentProcess->CloneRoot ? \
  624. ((PMMCLONE_DESCRIPTOR)MiLocateAddressInTree(((ULONG_PTR)VA), \
  625. (PMMADDRESS_NODE *)&(_CurrentProcess->CloneRoot))) : \
  626. NULL)
  627. #define MI_VA_TO_PAGE(va) ((ULONG_PTR)(va) >> PAGE_SHIFT)
  628. #define MI_VA_TO_VPN(va) ((ULONG_PTR)(va) >> PAGE_SHIFT)
  629. #define MI_VPN_TO_VA(vpn) (PVOID)((vpn) << PAGE_SHIFT)
  630. #define MI_VPN_TO_VA_ENDING(vpn) (PVOID)(((vpn) << PAGE_SHIFT) | (PAGE_SIZE - 1))
  631. #define MiGetByteOffset(va) ((ULONG_PTR)(va) & (PAGE_SIZE - 1))
  632. #define MI_PFN_ELEMENT(index) (&MmPfnDatabase[index])
  633. //
  634. // Make a write-copy PTE, only writable.
  635. //
  636. #define MI_MAKE_PROTECT_NOT_WRITE_COPY(PROTECT) \
  637. (MmMakeProtectNotWriteCopy[PROTECT])
  638. //
  639. // Define macros to lock and unlock the PFN database.
  640. //
  641. #define MiLockPfnDatabase(OldIrql) \
  642. OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock)
  643. #define MiUnlockPfnDatabase(OldIrql) \
  644. KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql)
  645. #define MiLockPfnDatabaseAtDpcLevel() \
  646. KeAcquireQueuedSpinLockAtDpcLevel(&KeGetCurrentPrcb()->LockQueue[LockQueuePfnLock])
  647. #define MiUnlockPfnDatabaseFromDpcLevel() \
  648. KeReleaseQueuedSpinLockFromDpcLevel(&KeGetCurrentPrcb()->LockQueue[LockQueuePfnLock])
  649. #define MiTryToLockPfnDatabase(OldIrql) \
  650. KeTryToAcquireQueuedSpinLock(LockQueuePfnLock, &OldIrql)
  651. #define MiReleasePfnLock() \
  652. KeReleaseQueuedSpinLockFromDpcLevel(&KeGetCurrentPrcb()->LockQueue[LockQueuePfnLock])
  653. #define MiLockSystemSpace(OldIrql) \
  654. OldIrql = KeAcquireQueuedSpinLock(LockQueueSystemSpaceLock)
  655. #define MiUnlockSystemSpace(OldIrql) \
  656. KeReleaseQueuedSpinLock(LockQueueSystemSpaceLock, OldIrql)
  657. #define MiLockSystemSpaceAtDpcLevel() \
  658. KeAcquireQueuedSpinLockAtDpcLevel(&KeGetCurrentPrcb()->LockQueue[LockQueueSystemSpaceLock])
  659. #define MiUnlockSystemSpaceFromDpcLevel() \
  660. KeReleaseQueuedSpinLockFromDpcLevel(&KeGetCurrentPrcb()->LockQueue[LockQueueSystemSpaceLock])
  661. // #define _MI_INSTRUMENT_PFN 1
  662. #if defined (_MI_INSTRUMENT_PFN)
  663. #define MI_MAX_PFN_CALLERS 500
  664. typedef struct _MMPFNTIMINGS {
  665. LARGE_INTEGER HoldTime; // Low bit is set if another processor waited
  666. ULONG_PTR AcquiredAddress;
  667. ULONG_PTR ReleasedAddress;
  668. } MMPFNTIMINGS, *PMMPFNTIMINGS;
  669. extern ULONG MiPfnTimings;
  670. extern ULONG_PTR MiPfnAcquiredAddress;
  671. extern MMPFNTIMINGS MiPfnSorted[];
  672. extern LARGE_INTEGER MiPfnAcquired;
  673. extern LARGE_INTEGER MiPfnThreshold;
  674. ULONG_PTR
  675. MiGetExecutionAddress(
  676. VOID
  677. );
  678. #if defined(_X86_)
  679. #define MI_GET_EXECUTION_ADDRESS(varname) varname = MiGetExecutionAddress();
  680. #else
  681. #define MI_GET_EXECUTION_ADDRESS(varname) varname = 0;
  682. #endif
  683. #define LOCK_PFN_TIMESTAMP() \
  684. { \
  685. MiPfnAcquired = KeQueryPerformanceCounter (NULL);\
  686. MI_GET_EXECUTION_ADDRESS(MiPfnAcquiredAddress); \
  687. }
  688. #define UNLOCK_PFN_TIMESTAMP() \
  689. { \
  690. ULONG i; \
  691. ULONG_PTR ExecutionAddress; \
  692. LARGE_INTEGER PfnReleased; \
  693. LARGE_INTEGER PfnHoldTime; \
  694. PfnReleased = KeQueryPerformanceCounter (NULL); \
  695. MI_GET_EXECUTION_ADDRESS(ExecutionAddress); \
  696. PfnHoldTime.QuadPart = (PfnReleased.QuadPart - MiPfnAcquired.QuadPart) & ~0x1; \
  697. i = MI_MAX_PFN_CALLERS - 1; \
  698. do { \
  699. if (PfnHoldTime.QuadPart < MiPfnSorted[i].HoldTime.QuadPart) { \
  700. break; \
  701. } \
  702. i -= 1; \
  703. } while (i != (ULONG)-1); \
  704. if (i != MI_MAX_PFN_CALLERS - 1) { \
  705. i += 1; \
  706. if (i != MI_MAX_PFN_CALLERS - 1) { \
  707. RtlMoveMemory (&MiPfnSorted[i+1], &MiPfnSorted[i], (MI_MAX_PFN_CALLERS-(i+1)) * sizeof(MMPFNTIMINGS)); \
  708. } \
  709. MiPfnSorted[i].HoldTime = PfnHoldTime; \
  710. if (KeTestForWaitersQueuedSpinLock(LockQueuePfnLock) == TRUE) {\
  711. MiPfnSorted[i].HoldTime.LowPart |= 0x1; \
  712. } \
  713. MiPfnSorted[i].AcquiredAddress = MiPfnAcquiredAddress; \
  714. MiPfnSorted[i].ReleasedAddress = ExecutionAddress; \
  715. } \
  716. if ((MiPfnTimings & 0x2) && (PfnHoldTime.QuadPart >= MiPfnThreshold.QuadPart)) { \
  717. DbgBreakPoint (); \
  718. } \
  719. if (MiPfnTimings & 0x1) { \
  720. MiPfnTimings &= ~0x1; \
  721. RtlZeroMemory (&MiPfnSorted[0], MI_MAX_PFN_CALLERS * sizeof(MMPFNTIMINGS)); \
  722. } \
  723. }
  724. #else
  725. #define LOCK_PFN_TIMESTAMP()
  726. #define UNLOCK_PFN_TIMESTAMP()
  727. #endif
  728. #define LOCK_PFN(OLDIRQL) ASSERT (KeGetCurrentIrql() <= APC_LEVEL); \
  729. MiLockPfnDatabase(OLDIRQL); \
  730. LOCK_PFN_TIMESTAMP();
  731. #define LOCK_PFN_WITH_TRY(OLDIRQL) \
  732. ASSERT (KeGetCurrentIrql() <= APC_LEVEL); \
  733. do { \
  734. } while (MiTryToLockPfnDatabase(OLDIRQL) == FALSE); \
  735. LOCK_PFN_TIMESTAMP();
  736. #define UNLOCK_PFN(OLDIRQL) \
  737. UNLOCK_PFN_TIMESTAMP(); \
  738. MiUnlockPfnDatabase(OLDIRQL); \
  739. ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
  740. #define LOCK_PFN2(OLDIRQL) ASSERT (KeGetCurrentIrql() <= DISPATCH_LEVEL); \
  741. MiLockPfnDatabase(OLDIRQL); \
  742. LOCK_PFN_TIMESTAMP();
  743. #define UNLOCK_PFN2(OLDIRQL) \
  744. UNLOCK_PFN_TIMESTAMP(); \
  745. MiUnlockPfnDatabase(OLDIRQL); \
  746. ASSERT(KeGetCurrentIrql() <= DISPATCH_LEVEL);
  747. #define LOCK_PFN_AT_DPC() ASSERT (KeGetCurrentIrql() == DISPATCH_LEVEL); \
  748. MiLockPfnDatabaseAtDpcLevel(); \
  749. LOCK_PFN_TIMESTAMP();
  750. #define UNLOCK_PFN_FROM_DPC() \
  751. UNLOCK_PFN_TIMESTAMP(); \
  752. MiUnlockPfnDatabaseFromDpcLevel(); \
  753. ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
  754. #define UNLOCK_PFN_AND_THEN_WAIT(OLDIRQL) \
  755. { \
  756. KIRQL XXX; \
  757. ASSERT (KeGetCurrentIrql() == DISPATCH_LEVEL); \
  758. ASSERT (OLDIRQL <= APC_LEVEL); \
  759. KiLockDispatcherDatabase (&XXX); \
  760. UNLOCK_PFN_TIMESTAMP(); \
  761. MiReleasePfnLock(); \
  762. (KeGetCurrentThread())->WaitIrql = OLDIRQL; \
  763. (KeGetCurrentThread())->WaitNext = TRUE; \
  764. }
  765. extern KMUTANT MmSystemLoadLock;
  766. #if DBG
  767. #define SYSLOAD_LOCK_OWNED_BY_ME() ((PETHREAD)MmSystemLoadLock.OwnerThread == PsGetCurrentThread())
  768. #else
  769. #define SYSLOAD_LOCK_OWNED_BY_ME()
  770. #endif
  771. #if DBG
  772. #if defined (_MI_COMPRESSION)
  773. extern KIRQL MiCompressionIrql;
  774. #define MM_PFN_LOCK_ASSERT() \
  775. if (MmDebug & 0x80000) { \
  776. KIRQL _OldIrql; \
  777. _OldIrql = KeGetCurrentIrql(); \
  778. ASSERT ((_OldIrql == DISPATCH_LEVEL) || \
  779. ((MiCompressionIrql != 0) && (_OldIrql == MiCompressionIrql))); \
  780. }
  781. #else
  782. #define MM_PFN_LOCK_ASSERT() \
  783. if (MmDebug & 0x80000) { \
  784. KIRQL _OldIrql; \
  785. _OldIrql = KeGetCurrentIrql(); \
  786. ASSERT (_OldIrql == DISPATCH_LEVEL); \
  787. }
  788. #endif
  789. extern PETHREAD MiExpansionLockOwner;
  790. #define MM_SET_EXPANSION_OWNER() ASSERT (MiExpansionLockOwner == NULL); \
  791. MiExpansionLockOwner = PsGetCurrentThread();
  792. #define MM_CLEAR_EXPANSION_OWNER() ASSERT (MiExpansionLockOwner == PsGetCurrentThread()); \
  793. MiExpansionLockOwner = NULL;
  794. #else
  795. #define MM_PFN_LOCK_ASSERT()
  796. #define MM_SET_EXPANSION_OWNER()
  797. #define MM_CLEAR_EXPANSION_OWNER()
  798. #endif //DBG
  799. #define LOCK_EXPANSION(OLDIRQL) ASSERT (KeGetCurrentIrql() <= APC_LEVEL); \
  800. ExAcquireSpinLock (&MmExpansionLock, &OLDIRQL);\
  801. MM_SET_EXPANSION_OWNER ();
  802. #define UNLOCK_EXPANSION(OLDIRQL) MM_CLEAR_EXPANSION_OWNER (); \
  803. ExReleaseSpinLock (&MmExpansionLock, OLDIRQL); \
  804. ASSERT (KeGetCurrentIrql() <= APC_LEVEL);
  805. #define UNLOCK_EXPANSION_AND_THEN_WAIT(OLDIRQL) \
  806. { \
  807. KIRQL XXX; \
  808. ASSERT (KeGetCurrentIrql() == 2); \
  809. ASSERT (OLDIRQL <= APC_LEVEL); \
  810. KiLockDispatcherDatabase (&XXX); \
  811. MM_CLEAR_EXPANSION_OWNER (); \
  812. KiReleaseSpinLock (&MmExpansionLock); \
  813. (KeGetCurrentThread())->WaitIrql = OLDIRQL; \
  814. (KeGetCurrentThread())->WaitNext = TRUE; \
  815. }
  816. extern PETHREAD MmSystemLockOwner;
  817. #define LOCK_SYSTEM_WS(OLDIRQL,_Thread) \
  818. ASSERT (KeGetCurrentIrql() <= APC_LEVEL); \
  819. KeRaiseIrql(APC_LEVEL,&OLDIRQL); \
  820. ExAcquireResourceExclusiveLite(&MmSystemWsLock,TRUE); \
  821. ASSERT (MmSystemLockOwner == NULL); \
  822. MmSystemLockOwner = _Thread;
  823. #define LOCK_SYSTEM_WS_UNSAFE(_Thread) \
  824. ASSERT (KeGetCurrentIrql() == APC_LEVEL); \
  825. ExAcquireResourceExclusiveLite(&MmSystemWsLock,TRUE); \
  826. ASSERT (MmSystemLockOwner == NULL); \
  827. MmSystemLockOwner = _Thread;
  828. #define UNLOCK_SYSTEM_WS(OLDIRQL) \
  829. ASSERT (MmSystemLockOwner == PsGetCurrentThread()); \
  830. MmSystemLockOwner = NULL; \
  831. ExReleaseResourceLite (&MmSystemWsLock); \
  832. KeLowerIrql (OLDIRQL); \
  833. ASSERT (KeGetCurrentIrql() <= APC_LEVEL);
  834. #define UNLOCK_SYSTEM_WS_NO_IRQL() \
  835. ASSERT (MmSystemLockOwner == PsGetCurrentThread()); \
  836. MmSystemLockOwner = NULL; \
  837. ExReleaseResourceLite (&MmSystemWsLock);
  838. #define MM_SYSTEM_WS_LOCK_ASSERT() \
  839. ASSERT (PsGetCurrentThread() == MmSystemLockOwner);
  840. #define LOCK_HYPERSPACE(_Process, OLDIRQL) \
  841. ASSERT (_Process == PsGetCurrentProcess ()); \
  842. ExAcquireSpinLock (&_Process->HyperSpaceLock, OLDIRQL);
  843. #define UNLOCK_HYPERSPACE(_Process, VA, OLDIRQL) \
  844. ASSERT (_Process == PsGetCurrentProcess ()); \
  845. MiGetPteAddress(VA)->u.Long = 0; \
  846. ExReleaseSpinLock (&_Process->HyperSpaceLock, OLDIRQL);
  847. #define LOCK_HYPERSPACE_AT_DPC(_Process) \
  848. ASSERT (KeGetCurrentIrql() == DISPATCH_LEVEL); \
  849. ASSERT (_Process == PsGetCurrentProcess ()); \
  850. ExAcquireSpinLockAtDpcLevel (&_Process->HyperSpaceLock);
  851. #define UNLOCK_HYPERSPACE_FROM_DPC(_Process, VA) \
  852. ASSERT (KeGetCurrentIrql() == DISPATCH_LEVEL); \
  853. ASSERT (_Process == PsGetCurrentProcess ()); \
  854. MiGetPteAddress(VA)->u.Long = 0; \
  855. ExReleaseSpinLockFromDpcLevel (&_Process->HyperSpaceLock);
  856. #define MiUnmapPageInHyperSpace(_Process, VA, OLDIRQL) UNLOCK_HYPERSPACE(_Process, VA, OLDIRQL)
  857. #define MiUnmapPageInHyperSpaceFromDpc(_Process, VA) UNLOCK_HYPERSPACE_FROM_DPC(_Process, VA)
  858. #if defined (_AXP64_)
  859. #define MI_WS_OWNER(PROCESS) ((PROCESS)->WorkingSetLock.Owner == KeGetCurrentThread())
  860. #define MI_NOT_WS_OWNER(PROCESS) (!MI_WS_OWNER(PROCESS))
  861. #else
  862. #define MI_WS_OWNER(PROCESS) 1
  863. #define MI_NOT_WS_OWNER(PROCESS) 1
  864. #endif
  865. #define MI_MUTEX_ACQUIRED_UNSAFE 0x88
  866. #define MI_IS_WS_UNSAFE(PROCESS) ((PROCESS)->WorkingSetAcquiredUnsafe == MI_MUTEX_ACQUIRED_UNSAFE)
  867. #define LOCK_WS(PROCESS) \
  868. ASSERT (MI_NOT_WS_OWNER(PROCESS)); \
  869. ExAcquireFastMutex( &((PROCESS)->WorkingSetLock)); \
  870. ASSERT (!MI_IS_WS_UNSAFE(PROCESS));
  871. #define LOCK_WS_UNSAFE(PROCESS) \
  872. ASSERT (MI_NOT_WS_OWNER(PROCESS)); \
  873. ASSERT (KeGetCurrentIrql() == APC_LEVEL); \
  874. ExAcquireFastMutexUnsafe( &((PROCESS)->WorkingSetLock));\
  875. (PROCESS)->WorkingSetAcquiredUnsafe = MI_MUTEX_ACQUIRED_UNSAFE;
  876. #define MI_MUST_BE_UNSAFE(PROCESS) \
  877. ASSERT (KeGetCurrentIrql() == APC_LEVEL); \
  878. ASSERT (MI_WS_OWNER(PROCESS)); \
  879. ASSERT (MI_IS_WS_UNSAFE(PROCESS));
  880. #define MI_MUST_BE_SAFE(PROCESS) \
  881. ASSERT (MI_WS_OWNER(PROCESS)); \
  882. ASSERT (!MI_IS_WS_UNSAFE(PROCESS));
  883. #if 0
  884. #define MI_MUST_BE_UNSAFE(PROCESS) \
  885. if (KeGetCurrentIrql() != APC_LEVEL) { \
  886. KeBugCheckEx(MEMORY_MANAGEMENT, 0x32, (ULONG_PTR)PROCESS, KeGetCurrentIrql(), 0); \
  887. } \
  888. if (!MI_WS_OWNER(PROCESS)) { \
  889. KeBugCheckEx(MEMORY_MANAGEMENT, 0x33, (ULONG_PTR)PROCESS, 0, 0); \
  890. } \
  891. if (!MI_IS_WS_UNSAFE(PROCESS)) { \
  892. KeBugCheckEx(MEMORY_MANAGEMENT, 0x34, (ULONG_PTR)PROCESS, 0, 0); \
  893. }
  894. #define MI_MUST_BE_SAFE(PROCESS) \
  895. if (!MI_WS_OWNER(PROCESS)) { \
  896. KeBugCheckEx(MEMORY_MANAGEMENT, 0x42, (ULONG_PTR)PROCESS, 0, 0); \
  897. } \
  898. if (MI_IS_WS_UNSAFE(PROCESS)) { \
  899. KeBugCheckEx(MEMORY_MANAGEMENT, 0x43, (ULONG_PTR)PROCESS, 0, 0); \
  900. }
  901. #endif
  902. #define UNLOCK_WS(PROCESS) \
  903. MI_MUST_BE_SAFE(PROCESS); \
  904. ExReleaseFastMutex(&((PROCESS)->WorkingSetLock));
  905. #define UNLOCK_WS_UNSAFE(PROCESS) \
  906. MI_MUST_BE_UNSAFE(PROCESS); \
  907. (PROCESS)->WorkingSetAcquiredUnsafe = 0; \
  908. ExReleaseFastMutexUnsafe(&((PROCESS)->WorkingSetLock)); \
  909. ASSERT (KeGetCurrentIrql() == APC_LEVEL);
  910. #define LOCK_ADDRESS_SPACE(PROCESS) \
  911. ExAcquireFastMutex( &((PROCESS)->AddressCreationLock))
  912. #define LOCK_WS_AND_ADDRESS_SPACE(PROCESS) \
  913. LOCK_ADDRESS_SPACE(PROCESS); \
  914. LOCK_WS_UNSAFE(PROCESS);
  915. #define UNLOCK_WS_AND_ADDRESS_SPACE(PROCESS) \
  916. UNLOCK_WS_UNSAFE(PROCESS); \
  917. UNLOCK_ADDRESS_SPACE(PROCESS);
  918. #define UNLOCK_ADDRESS_SPACE(PROCESS) \
  919. ExReleaseFastMutex( &((PROCESS)->AddressCreationLock))
  920. //
  921. // The working set lock may have been acquired safely or unsafely.
  922. // Release and reacquire it regardless.
  923. //
  924. #define UNLOCK_WS_REGARDLESS(PROCESS, WSHELDSAFE) \
  925. ASSERT (MI_WS_OWNER (PROCESS)); \
  926. if (MI_IS_WS_UNSAFE (PROCESS)) { \
  927. UNLOCK_WS_UNSAFE (PROCESS); \
  928. WSHELDSAFE = FALSE; \
  929. } \
  930. else { \
  931. UNLOCK_WS (PROCESS); \
  932. WSHELDSAFE = TRUE; \
  933. }
  934. #define LOCK_WS_REGARDLESS(PROCESS, WSHELDSAFE) \
  935. if (WSHELDSAFE == TRUE) { \
  936. LOCK_WS (PROCESS); \
  937. } \
  938. else { \
  939. LOCK_WS_UNSAFE (PROCESS); \
  940. }
  941. #define ZERO_LARGE(LargeInteger) \
  942. (LargeInteger).LowPart = 0; \
  943. (LargeInteger).HighPart = 0;
  944. #define NO_BITS_FOUND ((ULONG)-1)
  945. //++
  946. //
  947. // ULONG
  948. // MI_CHECK_BIT (
  949. // IN PULONG ARRAY
  950. // IN ULONG BIT
  951. // )
  952. //
  953. // Routine Description:
  954. //
  955. // The MI_CHECK_BIT macro checks to see if the specified bit is
  956. // set within the specified array.
  957. //
  958. // Arguments:
  959. //
  960. // ARRAY - First element of the array to check.
  961. //
  962. // BIT - bit number (first bit is 0) to check.
  963. //
  964. // Return Value:
  965. //
  966. // Returns the value of the bit (0 or 1).
  967. //
  968. //--
  969. #define MI_CHECK_BIT(ARRAY,BIT) \
  970. (((ULONG)ARRAY[(BIT) / (sizeof(ULONG)*8)] >> ((BIT) & 0x1F)) & 1)
  971. //++
  972. //
  973. // VOID
  974. // MI_SET_BIT (
  975. // IN PULONG ARRAY
  976. // IN ULONG BIT
  977. // )
  978. //
  979. // Routine Description:
  980. //
  981. // The MI_SET_BIT macro sets the specified bit within the
  982. // specified array.
  983. //
  984. // Arguments:
  985. //
  986. // ARRAY - First element of the array to set.
  987. //
  988. // BIT - bit number.
  989. //
  990. // Return Value:
  991. //
  992. // None.
  993. //
  994. //--
  995. #define MI_SET_BIT(ARRAY,BIT) \
  996. ARRAY[(BIT) / (sizeof(ULONG)*8)] |= (1 << ((BIT) & 0x1F))
  997. //++
  998. //
  999. // VOID
  1000. // MI_CLEAR_BIT (
  1001. // IN PULONG ARRAY
  1002. // IN ULONG BIT
  1003. // )
  1004. //
  1005. // Routine Description:
  1006. //
  1007. // The MI_CLEAR_BIT macro sets the specified bit within the
  1008. // specified array.
  1009. //
  1010. // Arguments:
  1011. //
  1012. // ARRAY - First element of the array to clear.
  1013. //
  1014. // BIT - bit number.
  1015. //
  1016. // Return Value:
  1017. //
  1018. // None.
  1019. //
  1020. //--
  1021. #define MI_CLEAR_BIT(ARRAY,BIT) \
  1022. ARRAY[(BIT) / (sizeof(ULONG)*8)] &= ~(1 << ((BIT) & 0x1F))
  1023. //
  1024. // These control the mirroring capabilities.
  1025. //
  1026. extern ULONG MmMirroring;
  1027. #define MM_MIRRORING_ENABLED 0x1 // Enable mirroring capabilities.
  1028. #define MM_MIRRORING_VERIFYING 0x2 // The HAL wants to verify the copy.
  1029. extern PRTL_BITMAP MiMirrorBitMap;
  1030. extern PRTL_BITMAP MiMirrorBitMap2;
  1031. extern LOGICAL MiMirroringActive;
  1032. #if defined (_WIN64)
  1033. #define MI_MAGIC_AWE_PTEFRAME 0xffffedcb
  1034. #else
  1035. #define MI_MAGIC_AWE_PTEFRAME 0xffedcb
  1036. #endif
  1037. #define MI_PFN_IS_AWE(Pfn1) \
  1038. ((Pfn1->u2.ShareCount <= 3) && \
  1039. (Pfn1->u3.e1.PageLocation == ActiveAndValid) && \
  1040. (Pfn1->u4.VerifierAllocation == 0) && \
  1041. (Pfn1->u3.e1.LargeSessionAllocation == 0) && \
  1042. (Pfn1->u3.e1.StartOfAllocation == 1) && \
  1043. (Pfn1->u3.e1.EndOfAllocation == 1) && \
  1044. (Pfn1->u4.PteFrame == MI_MAGIC_AWE_PTEFRAME))
  1045. //
  1046. // PFN database element.
  1047. //
  1048. //
  1049. // Define pseudo fields for start and end of allocation.
  1050. //
  1051. #define StartOfAllocation ReadInProgress
  1052. #define EndOfAllocation WriteInProgress
  1053. #define LargeSessionAllocation PrototypePte
  1054. typedef struct _MMPFNENTRY {
  1055. ULONG Modified : 1;
  1056. ULONG ReadInProgress : 1;
  1057. ULONG WriteInProgress : 1;
  1058. ULONG PrototypePte: 1;
  1059. ULONG PageColor : 3;
  1060. ULONG ParityError : 1;
  1061. ULONG PageLocation : 3;
  1062. ULONG RemovalRequested : 1;
  1063. ULONG CacheAttribute : 2;
  1064. ULONG Rom : 1;
  1065. ULONG LockCharged : 1; // maintained for DBG only
  1066. ULONG DontUse : 16; // overlays USHORT for reference count field.
  1067. } MMPFNENTRY;
  1068. //
  1069. // The cache type definitions are carefully chosen to line up with the
  1070. // MEMORY_CACHING_TYPE definitions to ease conversions. Any changes here must
  1071. // be reflected throughout the code.
  1072. //
  1073. typedef enum _MI_PFN_CACHE_ATTRIBUTE {
  1074. MiNonCached,
  1075. MiCached,
  1076. MiWriteCombined,
  1077. MiNotMapped
  1078. } MI_PFN_CACHE_ATTRIBUTE, *PMI_PFN_CACHE_ATTRIBUTE;
  1079. //
  1080. // This conversion array is unfortunately needed because not all
  1081. // hardware platforms support all possible cache values. Note that
  1082. // the first range is for system RAM, the second range is for I/O space.
  1083. //
  1084. extern MI_PFN_CACHE_ATTRIBUTE MiPlatformCacheAttributes[2 * MmMaximumCacheType];
  1085. //++
  1086. //
  1087. // MI_PFN_CACHE_ATTRIBUTE
  1088. // MI_TRANSLATE_CACHETYPE (
  1089. // IN MEMORY_CACHING_TYPE InputCacheType,
  1090. // IN ULONG IoSpace
  1091. // )
  1092. //
  1093. // Routine Description:
  1094. //
  1095. // Returns the hardware supported cache type for the requested cachetype.
  1096. //
  1097. // Arguments:
  1098. //
  1099. // InputCacheType - Supplies the desired cache type.
  1100. //
  1101. // IoSpace - Supplies nonzero (not necessarily 1 though) if this is
  1102. // I/O space, zero if it is main memory.
  1103. //
  1104. // Return Value:
  1105. //
  1106. // The actual cache type.
  1107. //
  1108. //--
  1109. FORCEINLINE
  1110. MI_PFN_CACHE_ATTRIBUTE
  1111. MI_TRANSLATE_CACHETYPE(
  1112. IN MEMORY_CACHING_TYPE InputCacheType,
  1113. IN ULONG IoSpace
  1114. )
  1115. {
  1116. ASSERT (InputCacheType <= MmWriteCombined);
  1117. if (IoSpace != 0) {
  1118. IoSpace = MmMaximumCacheType;
  1119. }
  1120. return MiPlatformCacheAttributes[IoSpace + InputCacheType];
  1121. }
  1122. //++
  1123. //
  1124. // VOID
  1125. // MI_SET_CACHETYPE_TRANSLATION (
  1126. // IN MEMORY_CACHING_TYPE InputCacheType,
  1127. // IN ULONG IoSpace,
  1128. // IN MI_PFN_CACHE_ATTRIBUTE NewAttribute
  1129. // )
  1130. //
  1131. // Routine Description:
  1132. //
  1133. // This function sets the hardware supported cachetype for the
  1134. // specified cachetype.
  1135. //
  1136. // Arguments:
  1137. //
  1138. // InputCacheType - Supplies the desired cache type.
  1139. //
  1140. // IoSpace - Supplies nonzero (not necessarily 1 though) if this is
  1141. // I/O space, zero if it is main memory.
  1142. //
  1143. // NewAttribute - Supplies the desired attribute.
  1144. //
  1145. // Return Value:
  1146. //
  1147. // None.
  1148. //
  1149. //--
  1150. FORCEINLINE
  1151. VOID
  1152. MI_SET_CACHETYPE_TRANSLATION(
  1153. IN MEMORY_CACHING_TYPE InputCacheType,
  1154. IN ULONG IoSpace,
  1155. IN MI_PFN_CACHE_ATTRIBUTE NewAttribute
  1156. )
  1157. {
  1158. ASSERT (InputCacheType <= MmWriteCombined);
  1159. if (IoSpace != 0) {
  1160. IoSpace = MmMaximumCacheType;
  1161. }
  1162. MiPlatformCacheAttributes[IoSpace + InputCacheType] = NewAttribute;
  1163. }
  1164. #if defined (_X86PAE_)
  1165. #pragma pack(1)
  1166. #endif
  1167. typedef struct _MMPFN {
  1168. union {
  1169. PFN_NUMBER Flink;
  1170. WSLE_NUMBER WsIndex;
  1171. PKEVENT Event;
  1172. NTSTATUS ReadStatus;
  1173. SINGLE_LIST_ENTRY NextStackPfn;
  1174. } u1;
  1175. PMMPTE PteAddress;
  1176. union {
  1177. PFN_NUMBER Blink;
  1178. ULONG_PTR ShareCount;
  1179. } u2;
  1180. union {
  1181. MMPFNENTRY e1;
  1182. struct {
  1183. USHORT ShortFlags;
  1184. USHORT ReferenceCount;
  1185. } e2;
  1186. } u3;
  1187. #if defined (_WIN64)
  1188. ULONG UsedPageTableEntries;
  1189. #endif
  1190. MMPTE OriginalPte;
  1191. union {
  1192. ULONG_PTR EntireFrame;
  1193. struct {
  1194. #if defined (_WIN64)
  1195. ULONG_PTR PteFrame: 58;
  1196. #else
  1197. ULONG_PTR PteFrame: 26;
  1198. #endif
  1199. ULONG_PTR InPageError : 1;
  1200. ULONG_PTR VerifierAllocation : 1;
  1201. ULONG_PTR AweAllocation : 1;
  1202. ULONG_PTR LockCharged : 1; // maintained for DBG only
  1203. ULONG_PTR KernelStack : 1; // only for valid (not trans) pages
  1204. ULONG_PTR Reserved : 1;
  1205. };
  1206. } u4;
  1207. } MMPFN, *PMMPFN;
  1208. #if defined (_X86PAE_)
  1209. #pragma pack()
  1210. #endif
  1211. // #define _MI_DEBUG_DIRTY 1 // Uncomment this for dirty bit logging
  1212. #if defined (_MI_DEBUG_DIRTY)
  1213. extern ULONG MiTrackDirtys;
  1214. #define MI_DIRTY_BACKTRACE_LENGTH 17
  1215. typedef struct _MI_DIRTY_TRACES {
  1216. PETHREAD Thread;
  1217. PEPROCESS Process;
  1218. PMMPFN Pfn;
  1219. PMMPTE PointerPte;
  1220. ULONG_PTR CallerId;
  1221. ULONG_PTR ShareCount;
  1222. USHORT ShortFlags;
  1223. USHORT ReferenceCount;
  1224. PVOID StackTrace [MI_DIRTY_BACKTRACE_LENGTH];
  1225. } MI_DIRTY_TRACES, *PMI_DIRTY_TRACES;
  1226. extern LONG MiDirtyIndex;
  1227. extern PMI_DIRTY_TRACES MiDirtyTraces;
  1228. VOID
  1229. FORCEINLINE
  1230. MiSnapDirty (
  1231. IN PMMPFN Pfn,
  1232. IN ULONG NewValue,
  1233. IN ULONG CallerId
  1234. )
  1235. {
  1236. PEPROCESS Process;
  1237. PMI_DIRTY_TRACES Information;
  1238. ULONG Index;
  1239. ULONG Hash;
  1240. if (MiDirtyTraces == NULL) {
  1241. return;
  1242. }
  1243. Index = InterlockedIncrement(&MiDirtyIndex);
  1244. Index &= (MiTrackDirtys - 1);
  1245. Information = &MiDirtyTraces[Index];
  1246. Process = PsGetCurrentProcess ();
  1247. Information->Thread = PsGetCurrentThread ();
  1248. Information->Process = Process;
  1249. Information->Pfn = Pfn;
  1250. Information->PointerPte = Pfn->PteAddress;
  1251. Information->CallerId = CallerId;
  1252. Information->ShareCount = Pfn->u2.ShareCount;
  1253. Information->ShortFlags = Pfn->u3.e2.ShortFlags;
  1254. Information->ReferenceCount = Pfn->u3.e2.ReferenceCount;
  1255. if (NewValue != 0) {
  1256. Information->Process = (PEPROCESS) ((ULONG_PTR)Process | 0x1);
  1257. }
  1258. RtlZeroMemory (&Information->StackTrace[0], MI_DIRTY_BACKTRACE_LENGTH * sizeof(PVOID));
  1259. RtlCaptureStackBackTrace (0, MI_DIRTY_BACKTRACE_LENGTH, Information->StackTrace, &Hash);
  1260. }
  1261. #define MI_SNAP_DIRTY(_Pfn, _NewValue, _Callerid) MiSnapDirty(_Pfn, _NewValue, _Callerid)
  1262. #else
  1263. #define MI_SNAP_DIRTY(_Pfn, _NewValue, _Callerid)
  1264. #endif
  1265. #if 0
  1266. #define MI_STAMP_MODIFIED(Pfn,id) (Pfn)->u4.Reserved = (id);
  1267. #else
  1268. #define MI_STAMP_MODIFIED(Pfn,id)
  1269. #endif
  1270. //++
  1271. //
  1272. // VOID
  1273. // MI_SET_MODIFIED (
  1274. // IN PMMPFN Pfn,
  1275. // IN ULONG NewValue,
  1276. // IN ULONG CallerId
  1277. // )
  1278. //
  1279. // Routine Description:
  1280. //
  1281. // Set (or clear) the modified bit in the PFN database element.
  1282. // The PFN lock must be held.
  1283. //
  1284. // Arguments:
  1285. //
  1286. // Pfn - Supplies the PFN to operate on.
  1287. //
  1288. // NewValue - Supplies 1 to set the modified bit, 0 to clear it.
  1289. //
  1290. // CallerId - Supplies a caller ID useful for debugging purposes.
  1291. //
  1292. // Return Value:
  1293. //
  1294. // None.
  1295. //
  1296. //--
  1297. //
  1298. #define MI_SET_MODIFIED(_Pfn, _NewValue, _CallerId) \
  1299. MI_SNAP_DIRTY (_Pfn, _NewValue, _CallerId); \
  1300. if ((_NewValue) != 0) { \
  1301. MI_STAMP_MODIFIED (_Pfn, _CallerId); \
  1302. } \
  1303. (_Pfn)->u3.e1.Modified = (_NewValue);
  1304. //
  1305. // ccNUMA is supported in multiprocessor PAE and WIN64 systems only.
  1306. //
  1307. #if (defined(_WIN64) || defined(_X86PAE_)) && !defined(NT_UP)
  1308. #define MI_MULTINODE
  1309. VOID
  1310. MiDetermineNode (
  1311. IN PFN_NUMBER Page,
  1312. IN PMMPFN Pfn
  1313. );
  1314. #else
  1315. #define MiDetermineNode(x,y) ((y)->u3.e1.PageColor = 0)
  1316. #endif
  1317. #if defined (_WIN64)
  1318. //
  1319. // Note there are some places where these portable macros are not currently
  1320. // used because we are not in the correct address space required.
  1321. //
  1322. #define MI_CAPTURE_USED_PAGETABLE_ENTRIES(PFN) \
  1323. ASSERT ((PFN)->UsedPageTableEntries <= PTE_PER_PAGE); \
  1324. (PFN)->OriginalPte.u.Soft.UsedPageTableEntries = (PFN)->UsedPageTableEntries;
  1325. #define MI_RETRIEVE_USED_PAGETABLE_ENTRIES_FROM_PTE(RBL, PTE) \
  1326. ASSERT ((PTE)->u.Soft.UsedPageTableEntries <= PTE_PER_PAGE); \
  1327. (RBL)->UsedPageTableEntries = (ULONG)(((PMMPTE)(PTE))->u.Soft.UsedPageTableEntries);
  1328. #define MI_ZERO_USED_PAGETABLE_ENTRIES_IN_INPAGE_SUPPORT(INPAGE_SUPPORT) \
  1329. (INPAGE_SUPPORT)->UsedPageTableEntries = 0;
  1330. #define MI_ZERO_USED_PAGETABLE_ENTRIES_IN_PFN(PFN) (PFN)->UsedPageTableEntries = 0;
  1331. #define MI_INSERT_USED_PAGETABLE_ENTRIES_IN_PFN(PFN, INPAGE_SUPPORT) \
  1332. ASSERT ((INPAGE_SUPPORT)->UsedPageTableEntries <= PTE_PER_PAGE); \
  1333. (PFN)->UsedPageTableEntries = (INPAGE_SUPPORT)->UsedPageTableEntries;
  1334. #define MI_ZERO_USED_PAGETABLE_ENTRIES(PFN) \
  1335. (PFN)->UsedPageTableEntries = 0;
  1336. #define MI_CHECK_USED_PTES_HANDLE(VA) \
  1337. ASSERT (MiGetPdeAddress(VA)->u.Hard.Valid == 1);
  1338. #define MI_GET_USED_PTES_HANDLE(VA) \
  1339. ((PVOID)MI_PFN_ELEMENT((PFN_NUMBER)MiGetPdeAddress(VA)->u.Hard.PageFrameNumber))
  1340. #define MI_GET_USED_PTES_FROM_HANDLE(PFN) \
  1341. ((ULONG)(((PMMPFN)(PFN))->UsedPageTableEntries))
  1342. #define MI_INCREMENT_USED_PTES_BY_HANDLE(PFN) \
  1343. (((PMMPFN)(PFN))->UsedPageTableEntries += 1); \
  1344. ASSERT (((PMMPFN)(PFN))->UsedPageTableEntries <= PTE_PER_PAGE)
  1345. #define MI_DECREMENT_USED_PTES_BY_HANDLE(PFN) \
  1346. (((PMMPFN)(PFN))->UsedPageTableEntries -= 1); \
  1347. ASSERT (((PMMPFN)(PFN))->UsedPageTableEntries < PTE_PER_PAGE)
  1348. #else
  1349. #define MI_CAPTURE_USED_PAGETABLE_ENTRIES(PFN)
  1350. #define MI_RETRIEVE_USED_PAGETABLE_ENTRIES_FROM_PTE(RBL, PTE)
  1351. #define MI_ZERO_USED_PAGETABLE_ENTRIES_IN_INPAGE_SUPPORT(INPAGE_SUPPORT)
  1352. #define MI_ZERO_USED_PAGETABLE_ENTRIES_IN_PFN(PFN)
  1353. #define MI_INSERT_USED_PAGETABLE_ENTRIES_IN_PFN(PFN, INPAGE_SUPPORT)
  1354. #define MI_CHECK_USED_PTES_HANDLE(VA)
  1355. #define MI_GET_USED_PTES_HANDLE(VA) ((PVOID)&MmWorkingSetList->UsedPageTableEntries[MiGetPdeIndex(VA)])
  1356. #define MI_GET_USED_PTES_FROM_HANDLE(PDSHORT) ((ULONG)(*(PUSHORT)(PDSHORT)))
  1357. #define MI_INCREMENT_USED_PTES_BY_HANDLE(PDSHORT) \
  1358. ((*(PUSHORT)(PDSHORT)) += 1); \
  1359. ASSERT (((*(PUSHORT)(PDSHORT)) <= PTE_PER_PAGE))
  1360. #define MI_DECREMENT_USED_PTES_BY_HANDLE(PDSHORT) \
  1361. ((*(PUSHORT)(PDSHORT)) -= 1); \
  1362. ASSERT (((*(PUSHORT)(PDSHORT)) < PTE_PER_PAGE))
  1363. #endif
  1364. extern PFN_NUMBER MmDynamicPfn;
  1365. extern FAST_MUTEX MmDynamicMemoryMutex;
  1366. extern PFN_NUMBER MmSystemLockPagesCount;
  1367. #if DBG
  1368. #define MI_LOCK_ID_COUNTER_MAX 64
  1369. ULONG MiLockIds[MI_LOCK_ID_COUNTER_MAX];
  1370. #define MI_MARK_PFN_AS_LOCK_CHARGED(Pfn, CallerId) \
  1371. ASSERT (Pfn->u3.e1.LockCharged == 0); \
  1372. ASSERT (CallerId < MI_LOCK_ID_COUNTER_MAX); \
  1373. MiLockIds[CallerId] += 1; \
  1374. Pfn->u3.e1.LockCharged = 1;
  1375. #define MI_UNMARK_PFN_AS_LOCK_CHARGED(Pfn, CallerId) \
  1376. ASSERT (Pfn->u3.e1.LockCharged == 1); \
  1377. ASSERT (CallerId < MI_LOCK_ID_COUNTER_MAX); \
  1378. MiLockIds[CallerId] += 1; \
  1379. Pfn->u3.e1.LockCharged = 0;
  1380. #else
  1381. #define MI_MARK_PFN_AS_LOCK_CHARGED(Pfn, CallerId)
  1382. #define MI_UNMARK_PFN_AS_LOCK_CHARGED(Pfn, CallerId)
  1383. #endif
  1384. //++
  1385. //
  1386. // VOID
  1387. // MI_ADD_LOCKED_PAGE_CHARGE (
  1388. // IN PMMPFN Pfn
  1389. // )
  1390. //
  1391. // Routine Description:
  1392. //
  1393. // Charge the systemwide count of locked pages if this is the initial
  1394. // lock for this page (multiple concurrent locks are only charged once).
  1395. //
  1396. // Arguments:
  1397. //
  1398. // Pfn - the PFN index to operate on.
  1399. //
  1400. // Return Value:
  1401. //
  1402. // None.
  1403. //
  1404. //--
  1405. //
  1406. #define MI_ADD_LOCKED_PAGE_CHARGE(Pfn, CallerId) \
  1407. ASSERT (Pfn->u3.e2.ReferenceCount != 0); \
  1408. if (Pfn->u3.e2.ReferenceCount == 1) { \
  1409. if (Pfn->u2.ShareCount != 0) { \
  1410. ASSERT (Pfn->u3.e1.PageLocation == ActiveAndValid); \
  1411. MI_MARK_PFN_AS_LOCK_CHARGED(Pfn, CallerId); \
  1412. MmSystemLockPagesCount += 1; \
  1413. } \
  1414. else { \
  1415. ASSERT (Pfn->u3.e1.LockCharged == 1); \
  1416. } \
  1417. }
  1418. #define MI_ADD_LOCKED_PAGE_CHARGE_FOR_MODIFIED_PAGE(Pfn, CallerId) \
  1419. ASSERT (Pfn->u3.e1.PageLocation != ActiveAndValid); \
  1420. ASSERT (Pfn->u2.ShareCount == 0); \
  1421. if (Pfn->u3.e2.ReferenceCount == 0) { \
  1422. MI_MARK_PFN_AS_LOCK_CHARGED(Pfn, CallerId); \
  1423. MmSystemLockPagesCount += 1; \
  1424. }
  1425. #define MI_ADD_LOCKED_PAGE_CHARGE_FOR_TRANSITION_PAGE(Pfn, CallerId) \
  1426. ASSERT (Pfn->u3.e1.PageLocation == ActiveAndValid); \
  1427. ASSERT (Pfn->u2.ShareCount == 0); \
  1428. ASSERT (Pfn->u3.e2.ReferenceCount != 0); \
  1429. if (Pfn->u3.e2.ReferenceCount == 1) { \
  1430. MI_MARK_PFN_AS_LOCK_CHARGED(Pfn, CallerId); \
  1431. MmSystemLockPagesCount += 1; \
  1432. }
  1433. //++
  1434. //
  1435. // VOID
  1436. // MI_REMOVE_LOCKED_PAGE_CHARGE (
  1437. // IN PMMPFN Pfn
  1438. // )
  1439. //
  1440. // Routine Description:
  1441. //
  1442. // Remove the charge from the systemwide count of locked pages if this
  1443. // is the last lock for this page (multiple concurrent locks are only
  1444. // charged once).
  1445. //
  1446. // The PFN reference checks are carefully ordered so the most common case
  1447. // is handled first, the next most common case next, etc. The case of
  1448. // a reference count of 2 occurs more than 1000x (yes, 3 orders of
  1449. // magnitude) more than a reference count of 1. And reference counts of >2
  1450. // occur 3 orders of magnitude more frequently than reference counts of
  1451. // exactly 1.
  1452. //
  1453. // Arguments:
  1454. //
  1455. // Pfn - the PFN index to operate on.
  1456. //
  1457. // Return Value:
  1458. //
  1459. // None.
  1460. //
  1461. //--
  1462. //
  1463. #define MI_REMOVE_LOCKED_PAGE_CHARGE(Pfn, CallerId) \
  1464. ASSERT (Pfn->u3.e2.ReferenceCount != 0); \
  1465. if (Pfn->u3.e2.ReferenceCount == 2) { \
  1466. if (Pfn->u2.ShareCount >= 1) { \
  1467. ASSERT (Pfn->u3.e1.PageLocation == ActiveAndValid); \
  1468. MI_UNMARK_PFN_AS_LOCK_CHARGED(Pfn, CallerId); \
  1469. MmSystemLockPagesCount -= 1; \
  1470. } \
  1471. else { \
  1472. /* \
  1473. * There are multiple referencers to this page and the \
  1474. * page is no longer valid in any process address space. \
  1475. * The systemwide lock count can only be decremented \
  1476. * by the last dereference. \
  1477. */ \
  1478. NOTHING; \
  1479. } \
  1480. } \
  1481. else if (Pfn->u3.e2.ReferenceCount != 1) { \
  1482. /* \
  1483. * There are still multiple referencers to this page (it may \
  1484. * or may not be resident in any process address space). \
  1485. * Since the systemwide lock count can only be decremented \
  1486. * by the last dereference (and this is not it), no action \
  1487. * is taken here. \
  1488. */ \
  1489. ASSERT (Pfn->u3.e2.ReferenceCount > 2); \
  1490. NOTHING; \
  1491. } \
  1492. else { \
  1493. /* \
  1494. * This page has already been deleted from all process address \
  1495. * spaces. It is sitting in limbo (not on any list) awaiting \
  1496. * this last dereference. \
  1497. */ \
  1498. ASSERT (Pfn->u3.e2.ReferenceCount == 1); \
  1499. ASSERT (Pfn->u3.e1.PageLocation != ActiveAndValid); \
  1500. ASSERT (Pfn->u2.ShareCount == 0); \
  1501. MI_UNMARK_PFN_AS_LOCK_CHARGED(Pfn, CallerId); \
  1502. MmSystemLockPagesCount -= 1; \
  1503. }
  1504. //++
  1505. //
  1506. // VOID
  1507. // MI_REMOVE_LOCKED_PAGE_CHARGE_AND_DECREF (
  1508. // IN PMMPFN Pfn
  1509. // )
  1510. //
  1511. // Routine Description:
  1512. //
  1513. // Remove the charge from the systemwide count of locked pages if this
  1514. // is the last lock for this page (multiple concurrent locks are only
  1515. // charged once).
  1516. //
  1517. // The PFN reference checks are carefully ordered so the most common case
  1518. // is handled first, the next most common case next, etc. The case of
  1519. // a reference count of 2 occurs more than 1000x (yes, 3 orders of
  1520. // magnitude) more than a reference count of 1. And reference counts of >2
  1521. // occur 3 orders of magnitude more frequently than reference counts of
  1522. // exactly 1.
  1523. //
  1524. // The PFN reference count is then decremented.
  1525. //
  1526. // Arguments:
  1527. //
  1528. // Pfn - the PFN index to operate on.
  1529. //
  1530. // Return Value:
  1531. //
  1532. // None.
  1533. //
  1534. //--
  1535. //
  1536. #define MI_REMOVE_LOCKED_PAGE_CHARGE_AND_DECREF(Pfn, CallerId) \
  1537. ASSERT (Pfn->u3.e2.ReferenceCount != 0); \
  1538. if (Pfn->u3.e2.ReferenceCount == 2) { \
  1539. if (Pfn->u2.ShareCount >= 1) { \
  1540. ASSERT (Pfn->u3.e1.PageLocation == ActiveAndValid); \
  1541. MI_UNMARK_PFN_AS_LOCK_CHARGED(Pfn, CallerId); \
  1542. MmSystemLockPagesCount -= 1; \
  1543. } \
  1544. else { \
  1545. /* \
  1546. * There are multiple referencers to this page and the \
  1547. * page is no longer valid in any process address space. \
  1548. * The systemwide lock count can only be decremented \
  1549. * by the last dereference. \
  1550. */ \
  1551. NOTHING; \
  1552. } \
  1553. Pfn->u3.e2.ReferenceCount -= 1; \
  1554. } \
  1555. else if (Pfn->u3.e2.ReferenceCount != 1) { \
  1556. /* \
  1557. * There are still multiple referencers to this page (it may \
  1558. * or may not be resident in any process address space). \
  1559. * Since the systemwide lock count can only be decremented \
  1560. * by the last dereference (and this is not it), no action \
  1561. * is taken here. \
  1562. */ \
  1563. ASSERT (Pfn->u3.e2.ReferenceCount > 2); \
  1564. Pfn->u3.e2.ReferenceCount -= 1; \
  1565. } \
  1566. else { \
  1567. /* \
  1568. * This page has already been deleted from all process address \
  1569. * spaces. It is sitting in limbo (not on any list) awaiting \
  1570. * this last dereference. \
  1571. */ \
  1572. PFN_NUMBER _PageFrameIndex; \
  1573. ASSERT (Pfn->u3.e2.ReferenceCount == 1); \
  1574. ASSERT (Pfn->u3.e1.PageLocation != ActiveAndValid); \
  1575. ASSERT (Pfn->u2.ShareCount == 0); \
  1576. MI_UNMARK_PFN_AS_LOCK_CHARGED(Pfn, CallerId); \
  1577. MmSystemLockPagesCount -= 1; \
  1578. _PageFrameIndex = Pfn - MmPfnDatabase; \
  1579. MiDecrementReferenceCount (_PageFrameIndex); \
  1580. }
  1581. //++
  1582. //
  1583. // VOID
  1584. // MI_ZERO_WSINDEX (
  1585. // IN PMMPFN Pfn
  1586. // )
  1587. //
  1588. // Routine Description:
  1589. //
  1590. // Zero the Working Set Index field of the argument PFN entry.
  1591. // There is a subtlety here on systems where the WsIndex ULONG is
  1592. // overlaid with an Event pointer and sizeof(ULONG) != sizeof(PKEVENT).
  1593. // Note this will need to be updated if we ever decide to allocate bodies of
  1594. // thread objects on 4GB boundaries.
  1595. //
  1596. // Arguments:
  1597. //
  1598. // Pfn - the PFN index to operate on.
  1599. //
  1600. // Return Value:
  1601. //
  1602. // None.
  1603. //
  1604. //--
  1605. //
  1606. #define MI_ZERO_WSINDEX(Pfn) \
  1607. Pfn->u1.Event = NULL;
  1608. typedef enum _MMSHARE_TYPE {
  1609. Normal,
  1610. ShareCountOnly,
  1611. AndValid
  1612. } MMSHARE_TYPE;
  1613. typedef struct _MMWSLE_HASH {
  1614. PVOID Key;
  1615. WSLE_NUMBER Index;
  1616. } MMWSLE_HASH, *PMMWSLE_HASH;
  1617. //++
  1618. //
  1619. // WSLE_NUMBER
  1620. // MI_WSLE_HASH (
  1621. // IN ULONG_PTR VirtualAddress,
  1622. // IN PMMWSL WorkingSetList
  1623. // )
  1624. //
  1625. // Routine Description:
  1626. //
  1627. // Hash the address
  1628. //
  1629. // Arguments:
  1630. //
  1631. // VirtualAddress - the address to hash.
  1632. //
  1633. // WorkingSetList - the working set to hash the address into.
  1634. //
  1635. // Return Value:
  1636. //
  1637. // The hash key.
  1638. //
  1639. //--
  1640. //
  1641. #define MI_WSLE_HASH(Address, Wsl) \
  1642. ((WSLE_NUMBER)(((ULONG_PTR)PAGE_ALIGN(Address) >> (PAGE_SHIFT - 2)) % \
  1643. ((Wsl)->HashTableSize - 1)))
  1644. //
  1645. // Working Set List Entry.
  1646. //
  1647. typedef struct _MMWSLENTRY {
  1648. ULONG_PTR Valid : 1;
  1649. ULONG_PTR LockedInWs : 1;
  1650. ULONG_PTR LockedInMemory : 1;
  1651. ULONG_PTR Protection : 5;
  1652. ULONG_PTR SameProtectAsProto : 1;
  1653. ULONG_PTR Direct : 1;
  1654. ULONG_PTR Age : 2;
  1655. #if MM_VIRTUAL_PAGE_FILLER
  1656. ULONG_PTR Filler : MM_VIRTUAL_PAGE_FILLER;
  1657. #endif
  1658. ULONG_PTR VirtualPageNumber : MM_VIRTUAL_PAGE_SIZE;
  1659. } MMWSLENTRY;
  1660. typedef struct _MMWSLE {
  1661. union {
  1662. PVOID VirtualAddress;
  1663. ULONG_PTR Long;
  1664. MMWSLENTRY e1;
  1665. } u1;
  1666. } MMWSLE;
  1667. #define MI_GET_PROTECTION_FROM_WSLE(Wsl) ((Wsl)->u1.e1.Protection)
  1668. typedef MMWSLE *PMMWSLE;
  1669. //
  1670. // Working Set List. Must be quadword sized.
  1671. //
  1672. typedef struct _MMWSL {
  1673. SIZE_T Quota;
  1674. WSLE_NUMBER FirstFree;
  1675. WSLE_NUMBER FirstDynamic;
  1676. WSLE_NUMBER LastEntry;
  1677. WSLE_NUMBER NextSlot; // The next slot to trim
  1678. PMMWSLE Wsle;
  1679. WSLE_NUMBER LastInitializedWsle;
  1680. WSLE_NUMBER NonDirectCount;
  1681. PMMWSLE_HASH HashTable;
  1682. ULONG HashTableSize;
  1683. ULONG NumberOfCommittedPageTables;
  1684. PVOID HashTableStart;
  1685. PVOID HighestPermittedHashAddress;
  1686. ULONG NumberOfImageWaiters;
  1687. ULONG VadBitMapHint;
  1688. #if (_MI_PAGING_LEVELS >= 4)
  1689. PULONG CommittedPageTables;
  1690. ULONG NumberOfCommittedPageDirectories;
  1691. PULONG CommittedPageDirectories;
  1692. ULONG NumberOfCommittedPageDirectoryParents;
  1693. ULONG CommittedPageDirectoryParents[(MM_USER_PAGE_DIRECTORY_PARENT_PAGES + sizeof(ULONG)*8-1)/(sizeof(ULONG)*8)];
  1694. #elif (_MI_PAGING_LEVELS >= 3)
  1695. PULONG CommittedPageTables;
  1696. ULONG NumberOfCommittedPageDirectories;
  1697. ULONG CommittedPageDirectories[(MM_USER_PAGE_DIRECTORY_PAGES + sizeof(ULONG)*8-1)/(sizeof(ULONG)*8)];
  1698. #else
  1699. //
  1700. // This must be at the end.
  1701. // Not used in system cache or session working set lists.
  1702. //
  1703. USHORT UsedPageTableEntries[MM_USER_PAGE_TABLE_PAGES];
  1704. ULONG CommittedPageTables[MM_USER_PAGE_TABLE_PAGES/(sizeof(ULONG)*8)];
  1705. #endif
  1706. } MMWSL, *PMMWSL;
  1707. #if defined(_X86_)
  1708. extern PMMWSL MmWorkingSetList;
  1709. #endif
  1710. extern PKEVENT MiHighMemoryEvent;
  1711. extern PKEVENT MiLowMemoryEvent;
  1712. //
  1713. // The claim estimate of unused pages in a working set is limited
  1714. // to grow by this amount per estimation period.
  1715. //
  1716. #define MI_CLAIM_INCR 30
  1717. //
  1718. // The maximum number of different ages a page can be.
  1719. //
  1720. #define MI_USE_AGE_COUNT 4
  1721. #define MI_USE_AGE_MAX (MI_USE_AGE_COUNT - 1)
  1722. //
  1723. // If more than this "percentage" of the working set is estimated to
  1724. // be used then allow it to grow freely.
  1725. //
  1726. #define MI_REPLACEMENT_FREE_GROWTH_SHIFT 5
  1727. //
  1728. // If more than this "percentage" of the working set has been claimed
  1729. // then force replacement in low memory.
  1730. //
  1731. #define MI_REPLACEMENT_CLAIM_THRESHOLD_SHIFT 3
  1732. //
  1733. // If more than this "percentage" of the working set is estimated to
  1734. // be available then force replacement in low memory.
  1735. //
  1736. #define MI_REPLACEMENT_EAVAIL_THRESHOLD_SHIFT 3
  1737. //
  1738. // If while doing replacement a page is found of this age or older then
  1739. // replace it. Otherwise the oldest is selected.
  1740. //
  1741. #define MI_IMMEDIATE_REPLACEMENT_AGE 2
  1742. //
  1743. // When trimming, use these ages for different passes.
  1744. //
  1745. #define MI_MAX_TRIM_PASSES 4
  1746. #define MI_PASS0_TRIM_AGE 2
  1747. #define MI_PASS1_TRIM_AGE 1
  1748. #define MI_PASS2_TRIM_AGE 1
  1749. #define MI_PASS3_TRIM_AGE 1
  1750. #define MI_PASS4_TRIM_AGE 0
  1751. //
  1752. // If not a forced trim, trim pages older than this age.
  1753. //
  1754. #define MI_TRIM_AGE_THRESHOLD 2
  1755. //
  1756. // This "percentage" of a claim is up for grabs in a foreground process.
  1757. //
  1758. #define MI_FOREGROUND_CLAIM_AVAILABLE_SHIFT 3
  1759. //
  1760. // This "percentage" of a claim is up for grabs in a background process.
  1761. //
  1762. #define MI_BACKGROUND_CLAIM_AVAILABLE_SHIFT 1
  1763. //++
  1764. //
  1765. // DWORD
  1766. // MI_CALC_NEXT_VALID_ESTIMATION_SLOT (
  1767. // DWORD Previous,
  1768. // DWORD Minimum,
  1769. // DWORD Maximum,
  1770. // MI_NEXT_ESTIMATION_SLOT_CONST NextEstimationSlotConst,
  1771. // PMMWSLE Wsle
  1772. // )
  1773. //
  1774. // Routine Description:
  1775. //
  1776. // We iterate through the working set array in a non-sequential
  1777. // manner so that the sample is independent of any aging or trimming.
  1778. //
  1779. // This algorithm walks through the working set with a stride of
  1780. // 2^MiEstimationShift elements.
  1781. //
  1782. // Arguments:
  1783. //
  1784. // Previous - Last slot used
  1785. //
  1786. // Minimum - Minimum acceptable slot (ie. the first dynamic one)
  1787. //
  1788. // Maximum - max slot number + 1
  1789. //
  1790. // NextEstimationSlotConst - for this algorithm it contains the stride
  1791. //
  1792. // Wsle - the working set array
  1793. //
  1794. // Return Value:
  1795. //
  1796. // Next slot.
  1797. //
  1798. // Environment:
  1799. //
  1800. // Kernel mode, APCs disabled, working set lock held and PFN lock held.
  1801. //
  1802. //--
  1803. typedef struct _MI_NEXT_ESTIMATION_SLOT_CONST {
  1804. WSLE_NUMBER Stride;
  1805. } MI_NEXT_ESTIMATION_SLOT_CONST;
  1806. #define MI_CALC_NEXT_ESTIMATION_SLOT_CONST(NextEstimationSlotConst, WorkingSetList) \
  1807. (NextEstimationSlotConst).Stride = 1 << MiEstimationShift;
  1808. #define MI_NEXT_VALID_ESTIMATION_SLOT(Previous, StartEntry, Minimum, Maximum, NextEstimationSlotConst, Wsle) \
  1809. ASSERT(((Previous) >= Minimum) && ((Previous) <= Maximum)); \
  1810. ASSERT(((StartEntry) >= Minimum) && ((StartEntry) <= Maximum)); \
  1811. do { \
  1812. (Previous) += (NextEstimationSlotConst).Stride; \
  1813. if ((Previous) > Maximum) { \
  1814. (Previous) = Minimum + ((Previous + 1) & (NextEstimationSlotConst.Stride - 1)); \
  1815. StartEntry += 1; \
  1816. (Previous) = StartEntry; \
  1817. } \
  1818. if ((Previous) > Maximum || (Previous) < Minimum) { \
  1819. StartEntry = Minimum; \
  1820. (Previous) = StartEntry; \
  1821. } \
  1822. } while (Wsle[Previous].u1.e1.Valid == 0);
  1823. //++
  1824. //
  1825. // WSLE_NUMBER
  1826. // MI_NEXT_VALID_AGING_SLOT (
  1827. // DWORD Previous,
  1828. // DWORD Minimum,
  1829. // DWORD Maximum,
  1830. // PMMWSLE Wsle
  1831. // )
  1832. //
  1833. // Routine Description:
  1834. //
  1835. // This finds the next slot to valid slot to age. It walks
  1836. // through the slots sequentialy.
  1837. //
  1838. // Arguments:
  1839. //
  1840. // Previous - Last slot used
  1841. //
  1842. // Minimum - Minimum acceptable slot (ie. the first dynamic one)
  1843. //
  1844. // Maximum - Max slot number + 1
  1845. //
  1846. // Wsle - the working set array
  1847. //
  1848. // Return Value:
  1849. //
  1850. // None.
  1851. //
  1852. // Environment:
  1853. //
  1854. // Kernel mode, APCs disabled, working set lock held and PFN lock held.
  1855. //
  1856. //--
  1857. #define MI_NEXT_VALID_AGING_SLOT(Previous, Minimum, Maximum, Wsle) \
  1858. ASSERT(((Previous) >= Minimum) && ((Previous) <= Maximum)); \
  1859. do { \
  1860. (Previous) += 1; \
  1861. if ((Previous) > Maximum) { \
  1862. Previous = Minimum; \
  1863. } \
  1864. } while ((Wsle[Previous].u1.e1.Valid == 0));
  1865. //++
  1866. //
  1867. // ULONG
  1868. // MI_CALCULATE_USAGE_ESTIMATE (
  1869. // IN PULONG SampledAgeCounts.
  1870. // IN ULONG CounterShift
  1871. // )
  1872. //
  1873. // Routine Description:
  1874. //
  1875. // In Usage Estimation, we count the number of pages of each age in
  1876. // a sample. The function turns the SampledAgeCounts into an
  1877. // estimate of the unused pages.
  1878. //
  1879. // Arguments:
  1880. //
  1881. // SampledAgeCounts - counts of pages of each different age in the sample
  1882. //
  1883. // CounterShift - shift necessary to apply sample to entire WS
  1884. //
  1885. // Return Value:
  1886. //
  1887. // The number of pages to walk in the working set to get a good
  1888. // estimate of the number available.
  1889. //
  1890. //--
  1891. #define MI_CALCULATE_USAGE_ESTIMATE(SampledAgeCounts, CounterShift) \
  1892. (((SampledAgeCounts)[1] + \
  1893. (SampledAgeCounts)[2] + (SampledAgeCounts)[3]) \
  1894. << (CounterShift))
  1895. //++
  1896. //
  1897. // VOID
  1898. // MI_RESET_WSLE_AGE (
  1899. // IN PMMPTE PointerPte,
  1900. // IN PMMWSLE Wsle
  1901. // )
  1902. //
  1903. // Routine Description:
  1904. //
  1905. // Clear the age counter for the working set entry.
  1906. //
  1907. // Arguments:
  1908. //
  1909. // PointerPte - pointer to the working set list entry's PTE.
  1910. //
  1911. // Wsle - pointer to the working set list entry.
  1912. //
  1913. // Return Value:
  1914. //
  1915. // None.
  1916. //
  1917. //--
  1918. #define MI_RESET_WSLE_AGE(PointerPte, Wsle) \
  1919. (Wsle)->u1.e1.Age = 0;
  1920. //++
  1921. //
  1922. // ULONG
  1923. // MI_GET_WSLE_AGE (
  1924. // IN PMMPTE PointerPte,
  1925. // IN PMMWSLE Wsle
  1926. // )
  1927. //
  1928. // Routine Description:
  1929. //
  1930. // Clear the age counter for the working set entry.
  1931. //
  1932. // Arguments:
  1933. //
  1934. // PointerPte - pointer to the working set list entry's PTE
  1935. // Wsle - pointer to the working set list entry
  1936. //
  1937. // Return Value:
  1938. //
  1939. // Age group of the working set entry
  1940. //
  1941. //--
  1942. #define MI_GET_WSLE_AGE(PointerPte, Wsle) \
  1943. ((ULONG)((Wsle)->u1.e1.Age))
  1944. //++
  1945. //
  1946. // VOID
  1947. // MI_INC_WSLE_AGE (
  1948. // IN PMMPTE PointerPte,
  1949. // IN PMMWSLE Wsle,
  1950. // )
  1951. //
  1952. // Routine Description:
  1953. //
  1954. // Increment the age counter for the working set entry.
  1955. //
  1956. // Arguments:
  1957. //
  1958. // PointerPte - pointer to the working set list entry's PTE.
  1959. //
  1960. // Wsle - pointer to the working set list entry.
  1961. //
  1962. // Return Value:
  1963. //
  1964. // None
  1965. //
  1966. //--
  1967. #define MI_INC_WSLE_AGE(PointerPte, Wsle) \
  1968. if ((Wsle)->u1.e1.Age < 3) { \
  1969. (Wsle)->u1.e1.Age += 1; \
  1970. }
  1971. //++
  1972. //
  1973. // VOID
  1974. // MI_UPDATE_USE_ESTIMATE (
  1975. // IN PMMPTE PointerPte,
  1976. // IN PMMWSLE Wsle,
  1977. // IN ULONG *SampledAgeCounts
  1978. // )
  1979. //
  1980. // Routine Description:
  1981. //
  1982. // Update the sampled age counts.
  1983. //
  1984. // Arguments:
  1985. //
  1986. // PointerPte - pointer to the working set list entry's PTE.
  1987. //
  1988. // Wsle - pointer to the working set list entry.
  1989. //
  1990. // SampledAgeCounts - array of age counts to be updated.
  1991. //
  1992. // Return Value:
  1993. //
  1994. // None
  1995. //
  1996. //--
  1997. #define MI_UPDATE_USE_ESTIMATE(PointerPte, Wsle, SampledAgeCounts) \
  1998. (SampledAgeCounts)[(Wsle)->u1.e1.Age] += 1;
  1999. //++
  2000. //
  2001. // BOOLEAN
  2002. // MI_WS_GROWING_TOO_FAST (
  2003. // IN PMMSUPPORT VmSupport
  2004. // )
  2005. //
  2006. // Routine Description:
  2007. //
  2008. // Limit the growth rate of processes as the
  2009. // available memory approaches zero. Note the caller must ensure that
  2010. // MmAvailablePages is low enough so this calculation does not wrap.
  2011. //
  2012. // Arguments:
  2013. //
  2014. // VmSupport - a working set.
  2015. //
  2016. // Return Value:
  2017. //
  2018. // TRUE if the growth rate is too fast, FALSE otherwise.
  2019. //
  2020. //--
  2021. #define MI_WS_GROWING_TOO_FAST(VmSupport) \
  2022. ((VmSupport)->GrowthSinceLastEstimate > \
  2023. (((MI_CLAIM_INCR * (MmAvailablePages*MmAvailablePages)) / (64*64)) + 1))
  2024. #define SECTION_BASE_ADDRESS(_NtSection) \
  2025. (*((PVOID *)&(_NtSection)->PointerToRelocations))
  2026. #define SECTION_LOCK_COUNT_POINTER(_NtSection) \
  2027. ((PLONG)&(_NtSection)->NumberOfRelocations)
  2028. //
  2029. // Memory Management Object structures.
  2030. //
  2031. typedef enum _SECTION_CHECK_TYPE {
  2032. CheckDataSection,
  2033. CheckImageSection,
  2034. CheckUserDataSection,
  2035. CheckBothSection
  2036. } SECTION_CHECK_TYPE;
  2037. typedef struct _MMEXTEND_INFO {
  2038. UINT64 CommittedSize;
  2039. ULONG ReferenceCount;
  2040. } MMEXTEND_INFO, *PMMEXTEND_INFO;
  2041. typedef struct _SEGMENT {
  2042. struct _CONTROL_AREA *ControlArea;
  2043. ULONG TotalNumberOfPtes;
  2044. ULONG NonExtendedPtes;
  2045. ULONG WritableUserReferences;
  2046. UINT64 SizeOfSegment;
  2047. MMPTE SegmentPteTemplate;
  2048. SIZE_T NumberOfCommittedPages;
  2049. PMMEXTEND_INFO ExtendInfo;
  2050. PVOID SystemImageBase; // could replace with single bit in ldrmodlist
  2051. PVOID BasedAddress;
  2052. //
  2053. // The fields below are for image & pagefile-backed sections only.
  2054. // Common fields are above and new common entries must be added to
  2055. // both the SEGMENT and MAPPED_FILE_SEGMENT declarations.
  2056. //
  2057. union {
  2058. SIZE_T ImageCommitment; // for image-backed sections only
  2059. PEPROCESS CreatingProcess; // for pagefile-backed sections only
  2060. } u1;
  2061. union {
  2062. PSECTION_IMAGE_INFORMATION ImageInformation; // for images only
  2063. PVOID FirstMappedVa; // for pagefile-backed sections only
  2064. } u2;
  2065. PMMPTE PrototypePte;
  2066. MMPTE ThePtes[MM_PROTO_PTE_ALIGNMENT / PAGE_SIZE];
  2067. } SEGMENT, *PSEGMENT;
  2068. typedef struct _MAPPED_FILE_SEGMENT {
  2069. struct _CONTROL_AREA *ControlArea;
  2070. ULONG TotalNumberOfPtes;
  2071. ULONG NonExtendedPtes;
  2072. ULONG WritableUserReferences;
  2073. UINT64 SizeOfSegment;
  2074. MMPTE SegmentPteTemplate;
  2075. SIZE_T NumberOfCommittedPages;
  2076. PMMEXTEND_INFO ExtendInfo;
  2077. PVOID SystemImageBase; // could replace with single bit in ldrmodlist
  2078. PVOID BasedAddress;
  2079. struct _MSUBSECTION *LastSubsectionHint;
  2080. } MAPPED_FILE_SEGMENT, *PMAPPED_FILE_SEGMENT;
  2081. typedef struct _EVENT_COUNTER {
  2082. SINGLE_LIST_ENTRY ListEntry;
  2083. ULONG RefCount;
  2084. KEVENT Event;
  2085. } EVENT_COUNTER, *PEVENT_COUNTER;
  2086. typedef struct _MMSECTION_FLAGS {
  2087. unsigned BeingDeleted : 1;
  2088. unsigned BeingCreated : 1;
  2089. unsigned BeingPurged : 1;
  2090. unsigned NoModifiedWriting : 1;
  2091. unsigned FailAllIo : 1;
  2092. unsigned Image : 1;
  2093. unsigned Based : 1;
  2094. unsigned File : 1;
  2095. unsigned Networked : 1;
  2096. unsigned NoCache : 1;
  2097. unsigned PhysicalMemory : 1;
  2098. unsigned CopyOnWrite : 1;
  2099. unsigned Reserve : 1; // not a spare bit!
  2100. unsigned Commit : 1;
  2101. unsigned FloppyMedia : 1;
  2102. unsigned WasPurged : 1;
  2103. unsigned UserReference : 1;
  2104. unsigned GlobalMemory : 1;
  2105. unsigned DeleteOnClose : 1;
  2106. unsigned FilePointerNull : 1;
  2107. unsigned DebugSymbolsLoaded : 1;
  2108. unsigned SetMappedFileIoComplete : 1;
  2109. unsigned CollidedFlush : 1;
  2110. unsigned NoChange : 1;
  2111. unsigned HadUserReference : 1;
  2112. unsigned ImageMappedInSystemSpace : 1;
  2113. unsigned UserWritable : 1;
  2114. unsigned Accessed : 1;
  2115. unsigned GlobalOnlyPerSession : 1;
  2116. unsigned Rom : 1;
  2117. unsigned filler : 2;
  2118. } MMSECTION_FLAGS;
  2119. typedef struct _CONTROL_AREA { // must be quadword sized.
  2120. PSEGMENT Segment;
  2121. LIST_ENTRY DereferenceList;
  2122. ULONG NumberOfSectionReferences;
  2123. ULONG NumberOfPfnReferences;
  2124. ULONG NumberOfMappedViews;
  2125. USHORT NumberOfSubsections;
  2126. USHORT FlushInProgressCount;
  2127. ULONG NumberOfUserReferences;
  2128. union {
  2129. ULONG LongFlags;
  2130. MMSECTION_FLAGS Flags;
  2131. } u;
  2132. PFILE_OBJECT FilePointer;
  2133. PEVENT_COUNTER WaitingForDeletion;
  2134. USHORT ModifiedWriteCount;
  2135. USHORT NumberOfSystemCacheViews;
  2136. } CONTROL_AREA, *PCONTROL_AREA;
  2137. typedef struct _LARGE_CONTROL_AREA { // must be quadword sized.
  2138. PSEGMENT Segment;
  2139. LIST_ENTRY DereferenceList;
  2140. ULONG NumberOfSectionReferences;
  2141. ULONG NumberOfPfnReferences;
  2142. ULONG NumberOfMappedViews;
  2143. USHORT NumberOfSubsections;
  2144. USHORT FlushInProgressCount;
  2145. ULONG NumberOfUserReferences;
  2146. union {
  2147. ULONG LongFlags;
  2148. MMSECTION_FLAGS Flags;
  2149. } u;
  2150. PFILE_OBJECT FilePointer;
  2151. PEVENT_COUNTER WaitingForDeletion;
  2152. USHORT ModifiedWriteCount;
  2153. USHORT NumberOfSystemCacheViews;
  2154. PFN_NUMBER StartingFrame; // only used if Flags.Rom == 1.
  2155. LIST_ENTRY UserGlobalList;
  2156. ULONG SessionId;
  2157. } LARGE_CONTROL_AREA, *PLARGE_CONTROL_AREA;
  2158. typedef struct _MMSUBSECTION_FLAGS {
  2159. unsigned ReadOnly : 1;
  2160. unsigned ReadWrite : 1;
  2161. unsigned SubsectionStatic : 1;
  2162. unsigned GlobalMemory: 1;
  2163. unsigned Protection : 5;
  2164. unsigned LargePages : 1;
  2165. unsigned StartingSector4132 : 10; // 2 ** (42+12) == 4MB*4GB == 16K TB
  2166. unsigned SectorEndOffset : 12;
  2167. } MMSUBSECTION_FLAGS;
  2168. typedef struct _SUBSECTION { // Must start on quadword boundary and be quad sized
  2169. PCONTROL_AREA ControlArea;
  2170. union {
  2171. ULONG LongFlags;
  2172. MMSUBSECTION_FLAGS SubsectionFlags;
  2173. } u;
  2174. ULONG StartingSector;
  2175. ULONG NumberOfFullSectors; // (4GB-1) * 4K == 16TB-4K limit per subsection
  2176. PMMPTE SubsectionBase;
  2177. ULONG UnusedPtes;
  2178. ULONG PtesInSubsection;
  2179. struct _SUBSECTION *NextSubsection;
  2180. } SUBSECTION, *PSUBSECTION;
  2181. extern const ULONG MMSECT;
  2182. //
  2183. // Accesses to MMSUBSECTION_FLAGS2 are synchronized via the PFN lock
  2184. // (unlike MMSUBSECTION_FLAGS access which is not lock protected at all).
  2185. //
  2186. typedef struct _MMSUBSECTION_FLAGS2 {
  2187. unsigned SubsectionAccessed : 1;
  2188. unsigned SubsectionConverted : 1; // only needed for debug
  2189. unsigned Reserved : 30;
  2190. } MMSUBSECTION_FLAGS2;
  2191. //
  2192. // Mapped data file subsection structure. Not used for images
  2193. // or pagefile-backed shared memory.
  2194. //
  2195. typedef struct _MSUBSECTION { // Must start on quadword boundary and be quad sized
  2196. PCONTROL_AREA ControlArea;
  2197. union {
  2198. ULONG LongFlags;
  2199. MMSUBSECTION_FLAGS SubsectionFlags;
  2200. } u;
  2201. ULONG StartingSector;
  2202. ULONG NumberOfFullSectors; // (4GB-1) * 4K == 16TB-4K limit per subsection
  2203. PMMPTE SubsectionBase;
  2204. ULONG UnusedPtes;
  2205. ULONG PtesInSubsection;
  2206. struct _SUBSECTION *NextSubsection;
  2207. LIST_ENTRY DereferenceList;
  2208. ULONG_PTR NumberOfMappedViews;
  2209. union {
  2210. ULONG LongFlags2;
  2211. MMSUBSECTION_FLAGS2 SubsectionFlags2;
  2212. } u2;
  2213. } MSUBSECTION, *PMSUBSECTION;
  2214. #define MI_MAXIMUM_SECTION_SIZE ((UINT64)16*1024*1024*1024*1024*1024 - (1<<MM4K_SHIFT))
  2215. VOID
  2216. MiDecrementSubsections (
  2217. IN PSUBSECTION FirstSubsection,
  2218. IN PSUBSECTION LastSubsection OPTIONAL
  2219. );
  2220. NTSTATUS
  2221. MiAddViewsForSectionWithPfn (
  2222. IN PMSUBSECTION StartMappedSubsection,
  2223. IN ULONG LastPteOffset OPTIONAL
  2224. );
  2225. NTSTATUS
  2226. MiAddViewsForSection (
  2227. IN PMSUBSECTION MappedSubsection,
  2228. IN ULONG LastPteOffset OPTIONAL,
  2229. IN KIRQL OldIrql,
  2230. OUT PULONG Waited
  2231. );
  2232. LOGICAL
  2233. MiReferenceSubsection (
  2234. IN PMSUBSECTION MappedSubsection
  2235. );
  2236. VOID
  2237. MiRemoveViewsFromSection (
  2238. IN PMSUBSECTION StartMappedSubsection,
  2239. IN ULONG LastPteOffset OPTIONAL
  2240. );
  2241. VOID
  2242. MiRemoveViewsFromSectionWithPfn (
  2243. IN PMSUBSECTION StartMappedSubsection,
  2244. IN ULONG LastPteOffset OPTIONAL
  2245. );
  2246. VOID
  2247. MiSubsectionConsistent(
  2248. IN PSUBSECTION Subsection
  2249. );
  2250. #if DBG
  2251. #define MI_CHECK_SUBSECTION(_subsection) MiSubsectionConsistent((PSUBSECTION)(_subsection))
  2252. #else
  2253. #define MI_CHECK_SUBSECTION(_subsection)
  2254. #endif
  2255. //++
  2256. //ULONG
  2257. //Mi4KStartForSubsection (
  2258. // IN PLARGE_INTEGER address,
  2259. // IN OUT PSUBSECTION subsection
  2260. // );
  2261. //
  2262. // Routine Description:
  2263. //
  2264. // This macro sets into the specified subsection the supplied information
  2265. // indicating the start address (in 4K units) of this portion of the file.
  2266. //
  2267. // Arguments
  2268. //
  2269. // address - Supplies the 64-bit address (in 4K units) of the start of this
  2270. // portion of the file.
  2271. //
  2272. // subsection - Supplies the subsection address to store the address in.
  2273. //
  2274. // Return Value:
  2275. //
  2276. // None.
  2277. //
  2278. //--
  2279. #define Mi4KStartForSubsection(address, subsection) \
  2280. subsection->StartingSector = ((PLARGE_INTEGER)address)->LowPart; \
  2281. subsection->u.SubsectionFlags.StartingSector4132 = \
  2282. (((PLARGE_INTEGER)(address))->HighPart & 0x3ff);
  2283. //++
  2284. //ULONG
  2285. //Mi4KStartFromSubsection (
  2286. // IN OUT PLARGE_INTEGER address,
  2287. // IN PSUBSECTION subsection
  2288. // );
  2289. //
  2290. // Routine Description:
  2291. //
  2292. // This macro gets the start 4K offset from the specified subsection.
  2293. //
  2294. // Arguments
  2295. //
  2296. // address - Supplies the 64-bit address (in 4K units) to place the
  2297. // start of this subsection into.
  2298. //
  2299. // subsection - Supplies the subsection address to get the address from.
  2300. //
  2301. // Return Value:
  2302. //
  2303. // None.
  2304. //
  2305. //--
  2306. #define Mi4KStartFromSubsection(address, subsection) \
  2307. ((PLARGE_INTEGER)address)->LowPart = subsection->StartingSector; \
  2308. ((PLARGE_INTEGER)address)->HighPart = subsection->u.SubsectionFlags.StartingSector4132;
  2309. typedef struct _MMDEREFERENCE_SEGMENT_HEADER {
  2310. KSPIN_LOCK Lock;
  2311. KSEMAPHORE Semaphore;
  2312. LIST_ENTRY ListHead;
  2313. } MMDEREFERENCE_SEGMENT_HEADER;
  2314. //
  2315. // This entry is used for calling the segment dereference thread
  2316. // to perform page file expansion. It has a similar structure
  2317. // to a control area to allow either a control area or a page file
  2318. // expansion entry to be placed on the list. Note that for a control
  2319. // area the segment pointer is valid whereas for page file expansion
  2320. // it is null.
  2321. //
  2322. typedef struct _MMPAGE_FILE_EXPANSION {
  2323. PSEGMENT Segment;
  2324. LIST_ENTRY DereferenceList;
  2325. SIZE_T RequestedExpansionSize;
  2326. SIZE_T ActualExpansion;
  2327. KEVENT Event;
  2328. LONG InProgress;
  2329. ULONG PageFileNumber;
  2330. } MMPAGE_FILE_EXPANSION, *PMMPAGE_FILE_EXPANSION;
  2331. #define MI_EXTEND_ANY_PAGEFILE ((ULONG)-1)
  2332. #define MI_CONTRACT_PAGEFILES ((SIZE_T)-1)
  2333. typedef struct _MMWORKING_SET_EXPANSION_HEAD {
  2334. LIST_ENTRY ListHead;
  2335. } MMWORKING_SET_EXPANSION_HEAD;
  2336. #define SUBSECTION_READ_ONLY 1L
  2337. #define SUBSECTION_READ_WRITE 2L
  2338. #define SUBSECTION_COPY_ON_WRITE 4L
  2339. #define SUBSECTION_SHARE_ALLOW 8L
  2340. //
  2341. // The MMINPAGE_FLAGS relies on the fact that a pool allocation is always
  2342. // QUADWORD aligned so the low 3 bits are always available.
  2343. //
  2344. typedef struct _MMINPAGE_FLAGS {
  2345. ULONG_PTR Completed : 1;
  2346. ULONG_PTR Available1 : 1;
  2347. ULONG_PTR Available2 : 1;
  2348. #if defined (_WIN64)
  2349. ULONG_PTR PrefetchMdlHighBits : 61;
  2350. #else
  2351. ULONG_PTR PrefetchMdlHighBits : 29;
  2352. #endif
  2353. } MMINPAGE_FLAGS, *PMMINPAGE_FLAGS;
  2354. #define MI_EXTRACT_PREFETCH_MDL(_Support) ((PMDL)((ULONG_PTR)(_Support->u1.PrefetchMdl) & ~(sizeof(QUAD) - 1)))
  2355. typedef struct _MMINPAGE_SUPPORT {
  2356. KEVENT Event;
  2357. IO_STATUS_BLOCK IoStatus;
  2358. LARGE_INTEGER ReadOffset;
  2359. LONG WaitCount;
  2360. #if defined (_WIN64)
  2361. ULONG UsedPageTableEntries;
  2362. #endif
  2363. PETHREAD Thread;
  2364. PFILE_OBJECT FilePointer;
  2365. PMMPTE BasePte;
  2366. PMMPFN Pfn;
  2367. union {
  2368. MMINPAGE_FLAGS e1;
  2369. ULONG_PTR LongFlags;
  2370. PMDL PrefetchMdl; // Only used under _PREFETCH_
  2371. } u1;
  2372. MDL Mdl;
  2373. PFN_NUMBER Page[MM_MAXIMUM_READ_CLUSTER_SIZE + 1];
  2374. SINGLE_LIST_ENTRY ListEntry;
  2375. } MMINPAGE_SUPPORT, *PMMINPAGE_SUPPORT;
  2376. #define MI_PF_DUMMY_PAGE_PTE ((PMMPTE)0x23452345) // Only used by _PREFETCH_
  2377. //
  2378. // Address Node.
  2379. //
  2380. typedef struct _MMADDRESS_NODE {
  2381. ULONG_PTR StartingVpn;
  2382. ULONG_PTR EndingVpn;
  2383. struct _MMADDRESS_NODE *Parent;
  2384. struct _MMADDRESS_NODE *LeftChild;
  2385. struct _MMADDRESS_NODE *RightChild;
  2386. } MMADDRESS_NODE, *PMMADDRESS_NODE;
  2387. typedef struct _SECTION {
  2388. MMADDRESS_NODE Address;
  2389. PSEGMENT Segment;
  2390. LARGE_INTEGER SizeOfSection;
  2391. union {
  2392. ULONG LongFlags;
  2393. MMSECTION_FLAGS Flags;
  2394. } u;
  2395. ULONG InitialPageProtection;
  2396. } SECTION, *PSECTION;
  2397. //
  2398. // Banked memory descriptor. Pointed to by VAD which has
  2399. // the PhysicalMemory flags set and the Banked pointer field as
  2400. // non-NULL.
  2401. //
  2402. typedef struct _MMBANKED_SECTION {
  2403. PFN_NUMBER BasePhysicalPage;
  2404. PMMPTE BasedPte;
  2405. ULONG BankSize;
  2406. ULONG BankShift; //shift for PTEs to calculate bank number
  2407. PBANKED_SECTION_ROUTINE BankedRoutine;
  2408. PVOID Context;
  2409. PMMPTE CurrentMappedPte;
  2410. MMPTE BankTemplate[1];
  2411. } MMBANKED_SECTION, *PMMBANKED_SECTION;
  2412. //
  2413. // Virtual address descriptor
  2414. //
  2415. // ***** NOTE **********
  2416. // The first part of a virtual address descriptor is a MMADDRESS_NODE!!!
  2417. //
  2418. #if defined (_WIN64)
  2419. #define COMMIT_SIZE 51
  2420. #if ((COMMIT_SIZE + PAGE_SHIFT) < 63)
  2421. #error COMMIT_SIZE too small
  2422. #endif
  2423. #else
  2424. #define COMMIT_SIZE 19
  2425. #if ((COMMIT_SIZE + PAGE_SHIFT) < 31)
  2426. #error COMMIT_SIZE too small
  2427. #endif
  2428. #endif
  2429. #define MM_MAX_COMMIT (((ULONG_PTR) 1 << COMMIT_SIZE) - 1)
  2430. #define MM_VIEW_UNMAP 0
  2431. #define MM_VIEW_SHARE 1
  2432. typedef struct _MMVAD_FLAGS {
  2433. ULONG_PTR CommitCharge : COMMIT_SIZE; //limits system to 4k pages or bigger!
  2434. ULONG_PTR PhysicalMapping : 1;
  2435. ULONG_PTR ImageMap : 1;
  2436. ULONG_PTR UserPhysicalPages : 1;
  2437. ULONG_PTR NoChange : 1;
  2438. ULONG_PTR WriteWatch : 1;
  2439. ULONG_PTR Protection : 5;
  2440. ULONG_PTR LargePages : 1;
  2441. ULONG_PTR MemCommit: 1;
  2442. ULONG_PTR PrivateMemory : 1; //used to tell VAD from VAD_SHORT
  2443. } MMVAD_FLAGS;
  2444. typedef struct _MMVAD_FLAGS2 {
  2445. unsigned FileOffset : 24; // number of 64k units into file
  2446. unsigned SecNoChange : 1; // set if SEC_NOCHANGE specified
  2447. unsigned OneSecured : 1; // set if u3 field is a range
  2448. unsigned MultipleSecured : 1; // set if u3 field is a list head
  2449. unsigned ReadOnly : 1; // protected as ReadOnly
  2450. unsigned LongVad : 1; // set if VAD is a long VAD
  2451. unsigned ExtendableFile : 1;
  2452. unsigned Inherit : 1; //1 = ViewShare, 0 = ViewUnmap
  2453. unsigned CopyOnWrite : 1;
  2454. } MMVAD_FLAGS2;
  2455. typedef struct _MMADDRESS_LIST {
  2456. ULONG_PTR StartVpn;
  2457. ULONG_PTR EndVpn;
  2458. } MMADDRESS_LIST, *PMMADDRESS_LIST;
  2459. typedef struct _MMSECURE_ENTRY {
  2460. union {
  2461. ULONG_PTR LongFlags2;
  2462. MMVAD_FLAGS2 VadFlags2;
  2463. } u2;
  2464. ULONG_PTR StartVpn;
  2465. ULONG_PTR EndVpn;
  2466. LIST_ENTRY List;
  2467. } MMSECURE_ENTRY, *PMMSECURE_ENTRY;
  2468. typedef struct _ALIAS_VAD_INFO {
  2469. KAPC Apc;
  2470. ULONG NumberOfEntries;
  2471. ULONG MaximumEntries;
  2472. } ALIAS_VAD_INFO, *PALIAS_VAD_INFO;
  2473. typedef struct _ALIAS_VAD_INFO2 {
  2474. ULONG BaseAddress;
  2475. HANDLE SecureHandle;
  2476. } ALIAS_VAD_INFO2, *PALIAS_VAD_INFO2;
  2477. typedef struct _MMVAD {
  2478. ULONG_PTR StartingVpn;
  2479. ULONG_PTR EndingVpn;
  2480. struct _MMVAD *Parent;
  2481. struct _MMVAD *LeftChild;
  2482. struct _MMVAD *RightChild;
  2483. union {
  2484. ULONG_PTR LongFlags;
  2485. MMVAD_FLAGS VadFlags;
  2486. } u;
  2487. PCONTROL_AREA ControlArea;
  2488. PMMPTE FirstPrototypePte;
  2489. PMMPTE LastContiguousPte;
  2490. union {
  2491. ULONG LongFlags2;
  2492. MMVAD_FLAGS2 VadFlags2;
  2493. } u2;
  2494. } MMVAD, *PMMVAD;
  2495. typedef struct _MMVAD_LONG {
  2496. ULONG_PTR StartingVpn;
  2497. ULONG_PTR EndingVpn;
  2498. struct _MMVAD *Parent;
  2499. struct _MMVAD *LeftChild;
  2500. struct _MMVAD *RightChild;
  2501. union {
  2502. ULONG_PTR LongFlags;
  2503. MMVAD_FLAGS VadFlags;
  2504. } u;
  2505. PCONTROL_AREA ControlArea;
  2506. PMMPTE FirstPrototypePte;
  2507. PMMPTE LastContiguousPte;
  2508. union {
  2509. ULONG LongFlags2;
  2510. MMVAD_FLAGS2 VadFlags2;
  2511. } u2;
  2512. union {
  2513. LIST_ENTRY List;
  2514. MMADDRESS_LIST Secured;
  2515. } u3;
  2516. union {
  2517. PMMBANKED_SECTION Banked;
  2518. PMMEXTEND_INFO ExtendedInfo;
  2519. } u4;
  2520. #if defined(_MIALT4K_)
  2521. PALIAS_VAD_INFO AliasInformation;
  2522. #endif
  2523. } MMVAD_LONG, *PMMVAD_LONG;
  2524. typedef struct _MMVAD_SHORT {
  2525. ULONG_PTR StartingVpn;
  2526. ULONG_PTR EndingVpn;
  2527. struct _MMVAD *Parent;
  2528. struct _MMVAD *LeftChild;
  2529. struct _MMVAD *RightChild;
  2530. union {
  2531. ULONG_PTR LongFlags;
  2532. MMVAD_FLAGS VadFlags;
  2533. } u;
  2534. } MMVAD_SHORT, *PMMVAD_SHORT;
  2535. #define MI_GET_PROTECTION_FROM_VAD(_Vad) ((ULONG)(_Vad)->u.VadFlags.Protection)
  2536. #define MI_PHYSICAL_VIEW_AWE 0x1 // AWE region
  2537. #define MI_PHYSICAL_VIEW_PHYS 0x2 // Device\PhysicalMemory region
  2538. typedef struct _MI_PHYSICAL_VIEW {
  2539. LIST_ENTRY ListEntry;
  2540. PMMVAD Vad;
  2541. PCHAR StartVa;
  2542. PCHAR EndVa;
  2543. union {
  2544. ULONG_PTR LongFlags; // physical or AWE Vad identification
  2545. PRTL_BITMAP BitMap; // only if Vad->u.VadFlags.WriteWatch == 1
  2546. } u;
  2547. } MI_PHYSICAL_VIEW, *PMI_PHYSICAL_VIEW;
  2548. #define MI_PHYSICAL_VIEW_KEY 'vpmM'
  2549. #define MI_WRITEWATCH_VIEW_KEY 'wWmM'
  2550. //
  2551. // Stuff for support of Write Watch.
  2552. //
  2553. extern ULONG_PTR MiActiveWriteWatch;
  2554. VOID
  2555. MiCaptureWriteWatchDirtyBit (
  2556. IN PEPROCESS Process,
  2557. IN PVOID VirtualAddress
  2558. );
  2559. //
  2560. // Stuff for support of AWE (Address Windowing Extensions).
  2561. //
  2562. typedef struct _AWEINFO {
  2563. PRTL_BITMAP VadPhysicalPagesBitMap;
  2564. ULONG_PTR VadPhysicalPages;
  2565. //
  2566. // The PushLock is used to allow most of the NtMapUserPhysicalPages{Scatter}
  2567. // to execute in parallel as this is acquired shared for these calls.
  2568. // Exclusive acquisitions are used to protect maps against frees of the
  2569. // pages as well as to protect updates to the AweVadList. Collisions
  2570. // should be rare because the exclusive acquisitions should be rare.
  2571. //
  2572. PEX_PUSH_LOCK_CACHE_AWARE PushLock;
  2573. LIST_ENTRY AweVadList;
  2574. } AWEINFO, *PAWEINFO;
  2575. VOID
  2576. MiAweViewInserter (
  2577. IN PEPROCESS Process,
  2578. IN PMI_PHYSICAL_VIEW PhysicalView
  2579. );
  2580. VOID
  2581. MiAweViewRemover (
  2582. IN PEPROCESS Process,
  2583. IN PMMVAD Vad
  2584. );
  2585. //
  2586. // Stuff for support of POSIX Fork.
  2587. //
  2588. typedef struct _MMCLONE_BLOCK {
  2589. MMPTE ProtoPte;
  2590. LONG CloneRefCount;
  2591. } MMCLONE_BLOCK, *PMMCLONE_BLOCK;
  2592. typedef struct _MMCLONE_HEADER {
  2593. ULONG NumberOfPtes;
  2594. LONG NumberOfProcessReferences;
  2595. PMMCLONE_BLOCK ClonePtes;
  2596. } MMCLONE_HEADER, *PMMCLONE_HEADER;
  2597. typedef struct _MMCLONE_DESCRIPTOR {
  2598. ULONG_PTR StartingVpn;
  2599. ULONG_PTR EndingVpn;
  2600. struct _MMCLONE_DESCRIPTOR *Parent;
  2601. struct _MMCLONE_DESCRIPTOR *LeftChild;
  2602. struct _MMCLONE_DESCRIPTOR *RightChild;
  2603. ULONG NumberOfPtes;
  2604. PMMCLONE_HEADER CloneHeader;
  2605. LONG NumberOfReferences;
  2606. LONG FinalNumberOfReferences;
  2607. SIZE_T PagedPoolQuotaCharge;
  2608. } MMCLONE_DESCRIPTOR, *PMMCLONE_DESCRIPTOR;
  2609. //
  2610. // The following macro allocates and initializes a bitmap from the
  2611. // specified pool of the specified size.
  2612. //
  2613. // VOID
  2614. // MiCreateBitMap (
  2615. // OUT PRTL_BITMAP *BitMapHeader,
  2616. // IN SIZE_T SizeOfBitMap,
  2617. // IN POOL_TYPE PoolType
  2618. // );
  2619. //
  2620. #define MiCreateBitMap(BMH,S,P) { \
  2621. ULONG _S; \
  2622. ASSERT ((ULONG64)(S) < _4gb); \
  2623. _S = sizeof(RTL_BITMAP) + (ULONG)((((S) + 31) / 32) * 4); \
  2624. *(BMH) = (PRTL_BITMAP)ExAllocatePoolWithTag( (P), _S, ' mM'); \
  2625. if (*(BMH)) { \
  2626. RtlInitializeBitMap( *(BMH), (PULONG)((*(BMH))+1), (ULONG)(S)); \
  2627. } \
  2628. }
  2629. #define MiRemoveBitMap(BMH) { \
  2630. ExFreePool(*(BMH)); \
  2631. *(BMH) = NULL; \
  2632. }
  2633. #define MI_INITIALIZE_ZERO_MDL(MDL) { \
  2634. MDL->Next = (PMDL) NULL; \
  2635. MDL->MdlFlags = 0; \
  2636. MDL->StartVa = NULL; \
  2637. MDL->ByteOffset = 0; \
  2638. MDL->ByteCount = 0; \
  2639. }
  2640. //
  2641. // Page File structures.
  2642. //
  2643. typedef struct _MMMOD_WRITER_LISTHEAD {
  2644. LIST_ENTRY ListHead;
  2645. KEVENT Event;
  2646. } MMMOD_WRITER_LISTHEAD, *PMMMOD_WRITER_LISTHEAD;
  2647. typedef struct _MMMOD_WRITER_MDL_ENTRY {
  2648. LIST_ENTRY Links;
  2649. LARGE_INTEGER WriteOffset;
  2650. union {
  2651. IO_STATUS_BLOCK IoStatus;
  2652. LARGE_INTEGER LastByte;
  2653. } u;
  2654. PIRP Irp;
  2655. ULONG_PTR LastPageToWrite;
  2656. PMMMOD_WRITER_LISTHEAD PagingListHead;
  2657. PLIST_ENTRY CurrentList;
  2658. struct _MMPAGING_FILE *PagingFile;
  2659. PFILE_OBJECT File;
  2660. PCONTROL_AREA ControlArea;
  2661. PERESOURCE FileResource;
  2662. MDL Mdl;
  2663. PFN_NUMBER Page[1];
  2664. } MMMOD_WRITER_MDL_ENTRY, *PMMMOD_WRITER_MDL_ENTRY;
  2665. #define MM_PAGING_FILE_MDLS 2
  2666. typedef struct _MMPAGING_FILE {
  2667. PFN_NUMBER Size;
  2668. PFN_NUMBER MaximumSize;
  2669. PFN_NUMBER MinimumSize;
  2670. PFN_NUMBER FreeSpace;
  2671. PFN_NUMBER CurrentUsage;
  2672. PFN_NUMBER PeakUsage;
  2673. PFN_NUMBER Hint;
  2674. PFN_NUMBER HighestPage;
  2675. PMMMOD_WRITER_MDL_ENTRY Entry[MM_PAGING_FILE_MDLS];
  2676. PRTL_BITMAP Bitmap;
  2677. PFILE_OBJECT File;
  2678. UNICODE_STRING PageFileName;
  2679. ULONG PageFileNumber;
  2680. BOOLEAN Extended;
  2681. BOOLEAN HintSetToZero;
  2682. BOOLEAN BootPartition;
  2683. HANDLE FileHandle;
  2684. } MMPAGING_FILE, *PMMPAGING_FILE;
  2685. //
  2686. // System PTE structures.
  2687. //
  2688. typedef enum _MMSYSTEM_PTE_POOL_TYPE {
  2689. SystemPteSpace,
  2690. NonPagedPoolExpansion,
  2691. MaximumPtePoolTypes
  2692. } MMSYSTEM_PTE_POOL_TYPE;
  2693. typedef struct _MMFREE_POOL_ENTRY {
  2694. LIST_ENTRY List; // maintained free&chk, 1st entry only
  2695. PFN_NUMBER Size; // maintained free&chk, 1st entry only
  2696. ULONG Signature; // maintained chk only, all entries
  2697. struct _MMFREE_POOL_ENTRY *Owner; // maintained free&chk, all entries
  2698. } MMFREE_POOL_ENTRY, *PMMFREE_POOL_ENTRY;
  2699. typedef struct _MMLOCK_CONFLICT {
  2700. LIST_ENTRY List;
  2701. PETHREAD Thread;
  2702. } MMLOCK_CONFLICT, *PMMLOCK_CONFLICT;
  2703. //
  2704. // System view structures
  2705. //
  2706. typedef struct _MMVIEW {
  2707. ULONG_PTR Entry;
  2708. PCONTROL_AREA ControlArea;
  2709. } MMVIEW, *PMMVIEW;
  2710. //
  2711. // The MMSESSION structure represents kernel memory that is only valid on a
  2712. // per-session basis, thus the calling thread must be in the proper session
  2713. // to access this structure.
  2714. //
  2715. typedef struct _MMSESSION {
  2716. //
  2717. // Never refer to the SystemSpaceViewLock directly - always use the pointer
  2718. // following it or you will break support for multiple concurrent sessions.
  2719. //
  2720. FAST_MUTEX SystemSpaceViewLock;
  2721. //
  2722. // This points to the mutex above and is needed because the MMSESSION
  2723. // is mapped in session space on Hydra and the mutex needs to be globally
  2724. // visible for proper KeWaitForSingleObject & KeSetEvent operation.
  2725. //
  2726. PFAST_MUTEX SystemSpaceViewLockPointer;
  2727. PCHAR SystemSpaceViewStart;
  2728. PMMVIEW SystemSpaceViewTable;
  2729. ULONG SystemSpaceHashSize;
  2730. ULONG SystemSpaceHashEntries;
  2731. ULONG SystemSpaceHashKey;
  2732. PRTL_BITMAP SystemSpaceBitMap;
  2733. } MMSESSION, *PMMSESSION;
  2734. extern MMSESSION MmSession;
  2735. #define LOCK_SYSTEM_VIEW_SPACE(_Session) \
  2736. ExAcquireFastMutex (_Session->SystemSpaceViewLockPointer)
  2737. #define UNLOCK_SYSTEM_VIEW_SPACE(_Session) \
  2738. ExReleaseFastMutex (_Session->SystemSpaceViewLockPointer)
  2739. //
  2740. // List for flushing TBs singularly.
  2741. //
  2742. typedef struct _MMPTE_FLUSH_LIST {
  2743. ULONG Count;
  2744. PMMPTE FlushPte[MM_MAXIMUM_FLUSH_COUNT];
  2745. PVOID FlushVa[MM_MAXIMUM_FLUSH_COUNT];
  2746. } MMPTE_FLUSH_LIST, *PMMPTE_FLUSH_LIST;
  2747. typedef struct _LOCK_TRACKER {
  2748. LIST_ENTRY ListEntry;
  2749. PMDL Mdl;
  2750. PVOID StartVa;
  2751. PFN_NUMBER Count;
  2752. ULONG Offset;
  2753. ULONG Length;
  2754. PFN_NUMBER Page;
  2755. PVOID CallingAddress;
  2756. PVOID CallersCaller;
  2757. LIST_ENTRY GlobalListEntry;
  2758. ULONG Who;
  2759. PEPROCESS Process;
  2760. } LOCK_TRACKER, *PLOCK_TRACKER;
  2761. extern LOGICAL MmTrackLockedPages;
  2762. extern BOOLEAN MiTrackingAborted;
  2763. extern KSPIN_LOCK MiTrackLockedPagesLock;
  2764. typedef struct _LOCK_HEADER {
  2765. LIST_ENTRY ListHead;
  2766. PFN_NUMBER Count;
  2767. } LOCK_HEADER, *PLOCK_HEADER;
  2768. extern LOGICAL MmSnapUnloads;
  2769. #define MI_UNLOADED_DRIVERS 50
  2770. extern ULONG MmLastUnloadedDriver;
  2771. extern PUNLOADED_DRIVERS MmUnloadedDrivers;
  2772. VOID
  2773. MiInitMachineDependent (
  2774. IN PLOADER_PARAMETER_BLOCK LoaderBlock
  2775. );
  2776. VOID
  2777. MiReportPhysicalMemory (
  2778. VOID
  2779. );
  2780. extern PFN_NUMBER MiNumberOfCompressionPages;
  2781. NTSTATUS
  2782. MiArmCompressionInterrupt (
  2783. VOID
  2784. );
  2785. VOID
  2786. MiBuildPagedPool (
  2787. VOID
  2788. );
  2789. VOID
  2790. MiInitializeNonPagedPool (
  2791. VOID
  2792. );
  2793. LOGICAL
  2794. MiInitializeSystemSpaceMap (
  2795. PVOID Session OPTIONAL
  2796. );
  2797. VOID
  2798. MiFindInitializationCode (
  2799. OUT PVOID *StartVa,
  2800. OUT PVOID *EndVa
  2801. );
  2802. VOID
  2803. MiFreeInitializationCode (
  2804. IN PVOID StartVa,
  2805. IN PVOID EndVa
  2806. );
  2807. extern ULONG MiNonCachedCollisions;
  2808. //
  2809. // If /NOLOWMEM is used, this is set to the boundary PFN (pages below this
  2810. // value are not used whenever possible).
  2811. //
  2812. extern PFN_NUMBER MiNoLowMemory;
  2813. PVOID
  2814. MiAllocateLowMemory (
  2815. IN SIZE_T NumberOfBytes,
  2816. IN PFN_NUMBER LowestAcceptablePfn,
  2817. IN PFN_NUMBER HighestAcceptablePfn,
  2818. IN PFN_NUMBER BoundaryPfn,
  2819. IN PVOID CallingAddress,
  2820. IN MEMORY_CACHING_TYPE CacheType,
  2821. IN ULONG Tag
  2822. );
  2823. LOGICAL
  2824. MiFreeLowMemory (
  2825. IN PVOID BaseAddress,
  2826. IN ULONG Tag
  2827. );
  2828. //
  2829. // Move drivers out of the low 16mb that ntldr placed them at - this makes more
  2830. // memory below 16mb available for ISA-type drivers that cannot run without it.
  2831. //
  2832. extern LOGICAL MmMakeLowMemory;
  2833. VOID
  2834. MiRemoveLowPages (
  2835. IN ULONG RemovePhase
  2836. );
  2837. ULONG
  2838. MiSectionInitialization (
  2839. VOID
  2840. );
  2841. #define MI_MAX_DEREFERENCE_CHUNK (64 * 1024 / PAGE_SIZE)
  2842. typedef struct _MI_PFN_DEREFERENCE_CHUNK {
  2843. SINGLE_LIST_ENTRY ListEntry;
  2844. CSHORT Flags;
  2845. USHORT NumberOfPages;
  2846. PFN_NUMBER Pfns[MI_MAX_DEREFERENCE_CHUNK];
  2847. } MI_PFN_DEREFERENCE_CHUNK, *PMI_PFN_DEREFERENCE_CHUNK;
  2848. extern SLIST_HEADER MmPfnDereferenceSListHead;
  2849. extern PSINGLE_LIST_ENTRY MmPfnDeferredList;
  2850. #define MI_DEFER_PFN_HELD 0x1
  2851. #define MI_DEFER_DRAIN_LOCAL_ONLY 0x2
  2852. VOID
  2853. MiDeferredUnlockPages (
  2854. ULONG Flags
  2855. );
  2856. LOGICAL
  2857. MiFreeAllExpansionNonPagedPool (
  2858. IN LOGICAL PoolLockHeld
  2859. );
  2860. VOID
  2861. FASTCALL
  2862. MiDecrementReferenceCount (
  2863. IN PFN_NUMBER PageFrameIndex
  2864. );
  2865. //++
  2866. //VOID
  2867. //MiDecrementReferenceCountInline (
  2868. // IN PMMPFN PFN
  2869. // IN PFN_NUMBER FRAME
  2870. // );
  2871. //
  2872. // Routine Description:
  2873. //
  2874. // MiDecrementReferenceCountInline decrements the reference count inline,
  2875. // only calling MiDecrementReferenceCount if the count would go to zero
  2876. // which would cause the page to be released.
  2877. //
  2878. // Arguments:
  2879. //
  2880. // PFN - Supplies the PFN to decrement.
  2881. //
  2882. // FRAME - Supplies the frame matching the above PFN.
  2883. //
  2884. // Return Value:
  2885. //
  2886. // None.
  2887. //
  2888. // Environment:
  2889. //
  2890. // PFN lock held.
  2891. //
  2892. //--
  2893. #define MiDecrementReferenceCountInline(PFN, FRAME) \
  2894. MM_PFN_LOCK_ASSERT(); \
  2895. ASSERT (MI_PFN_ELEMENT(FRAME) == (PFN)); \
  2896. ASSERT ((FRAME) <= MmHighestPhysicalPage); \
  2897. ASSERT ((PFN)->u3.e2.ReferenceCount != 0); \
  2898. if ((PFN)->u3.e2.ReferenceCount != 1) { \
  2899. (PFN)->u3.e2.ReferenceCount -= 1; \
  2900. } \
  2901. else { \
  2902. MiDecrementReferenceCount (FRAME); \
  2903. }
  2904. VOID
  2905. FASTCALL
  2906. MiDecrementShareCount (
  2907. IN PFN_NUMBER PageFrameIndex
  2908. );
  2909. #define MiDecrementShareCountOnly(P) MiDecrementShareCount(P)
  2910. #define MiDecrementShareAndValidCount(P) MiDecrementShareCount(P)
  2911. //++
  2912. //VOID
  2913. //MiDecrementShareCountInline (
  2914. // IN PMMPFN PFN,
  2915. // IN PFN_NUMBER FRAME
  2916. // );
  2917. //
  2918. // Routine Description:
  2919. //
  2920. // MiDecrementShareCountInline decrements the share count inline,
  2921. // only calling MiDecrementShareCount if the count would go to zero
  2922. // which would cause the page to be released.
  2923. //
  2924. // Arguments:
  2925. //
  2926. // PFN - Supplies the PFN to decrement.
  2927. //
  2928. // FRAME - Supplies the frame matching the above PFN.
  2929. //
  2930. // Return Value:
  2931. //
  2932. // None.
  2933. //
  2934. // Environment:
  2935. //
  2936. // PFN lock held.
  2937. //
  2938. //--
  2939. #define MiDecrementShareCountInline(PFN, FRAME) \
  2940. MM_PFN_LOCK_ASSERT(); \
  2941. ASSERT (((FRAME) <= MmHighestPhysicalPage) && ((FRAME) > 0)); \
  2942. ASSERT (MI_PFN_ELEMENT(FRAME) == (PFN)); \
  2943. ASSERT ((PFN)->u2.ShareCount != 0); \
  2944. if ((PFN)->u3.e1.PageLocation != ActiveAndValid && (PFN)->u3.e1.PageLocation != StandbyPageList) { \
  2945. KeBugCheckEx (PFN_LIST_CORRUPT, 0x99, FRAME, (PFN)->u3.e1.PageLocation, 0); \
  2946. } \
  2947. if ((PFN)->u2.ShareCount != 1) { \
  2948. (PFN)->u2.ShareCount -= 1; \
  2949. PERFINFO_DECREFCNT((PFN), PERF_SOFT_TRIM, PERFINFO_LOG_TYPE_DECSHARCNT); \
  2950. ASSERT ((PFN)->u2.ShareCount < 0xF000000); \
  2951. } \
  2952. else { \
  2953. MiDecrementShareCount (FRAME); \
  2954. }
  2955. //
  2956. // Routines which operate on the Page Frame Database Lists
  2957. //
  2958. VOID
  2959. FASTCALL
  2960. MiInsertPageInList (
  2961. IN PMMPFNLIST ListHead,
  2962. IN PFN_NUMBER PageFrameIndex
  2963. );
  2964. VOID
  2965. FASTCALL
  2966. MiInsertPageInFreeList (
  2967. IN PFN_NUMBER PageFrameIndex
  2968. );
  2969. VOID
  2970. FASTCALL
  2971. MiInsertStandbyListAtFront (
  2972. IN PFN_NUMBER PageFrameIndex
  2973. );
  2974. PFN_NUMBER //PageFrameIndex
  2975. FASTCALL
  2976. MiRemovePageFromList (
  2977. IN PMMPFNLIST ListHead
  2978. );
  2979. VOID
  2980. FASTCALL
  2981. MiUnlinkPageFromList (
  2982. IN PMMPFN Pfn
  2983. );
  2984. VOID
  2985. MiUnlinkFreeOrZeroedPage (
  2986. IN PFN_NUMBER Page
  2987. );
  2988. VOID
  2989. FASTCALL
  2990. MiInsertFrontModifiedNoWrite (
  2991. IN PFN_NUMBER PageFrameIndex
  2992. );
  2993. #define MM_MEDIUM_LIMIT 32
  2994. #define MM_HIGH_LIMIT 128
  2995. ULONG
  2996. FASTCALL
  2997. MiEnsureAvailablePageOrWait (
  2998. IN PEPROCESS Process,
  2999. IN PVOID VirtualAddress
  3000. );
  3001. PFN_NUMBER
  3002. MiAllocatePfn (
  3003. IN PMMPTE PointerPte,
  3004. IN ULONG Protection
  3005. );
  3006. PFN_NUMBER
  3007. FASTCALL
  3008. MiRemoveAnyPage (
  3009. IN ULONG PageColor
  3010. );
  3011. PFN_NUMBER
  3012. FASTCALL
  3013. MiRemoveZeroPage (
  3014. IN ULONG PageColor
  3015. );
  3016. VOID
  3017. MiPurgeTransitionList (
  3018. VOID
  3019. );
  3020. PVOID
  3021. MiFindContiguousMemory (
  3022. IN PFN_NUMBER LowestPfn,
  3023. IN PFN_NUMBER HighestPfn,
  3024. IN PFN_NUMBER BoundaryPfn,
  3025. IN PFN_NUMBER SizeInPages,
  3026. IN MEMORY_CACHING_TYPE CacheType,
  3027. IN PVOID CallingAddress
  3028. );
  3029. PVOID
  3030. MiCheckForContiguousMemory (
  3031. IN PVOID BaseAddress,
  3032. IN PFN_NUMBER BaseAddressPages,
  3033. IN PFN_NUMBER SizeInPages,
  3034. IN PFN_NUMBER LowestPfn,
  3035. IN PFN_NUMBER HighestPfn,
  3036. IN PFN_NUMBER BoundaryPfn,
  3037. IN MI_PFN_CACHE_ATTRIBUTE CacheAttribute
  3038. );
  3039. //
  3040. // Routines which operate on the page frame database entry.
  3041. //
  3042. VOID
  3043. MiInitializePfn (
  3044. IN PFN_NUMBER PageFrameIndex,
  3045. IN PMMPTE PointerPte,
  3046. IN ULONG ModifiedState
  3047. );
  3048. VOID
  3049. MiInitializePfnForOtherProcess (
  3050. IN PFN_NUMBER PageFrameIndex,
  3051. IN PMMPTE PointerPte,
  3052. IN PFN_NUMBER ContainingPageFrame
  3053. );
  3054. VOID
  3055. MiInitializeCopyOnWritePfn (
  3056. IN PFN_NUMBER PageFrameIndex,
  3057. IN PMMPTE PointerPte,
  3058. IN WSLE_NUMBER WorkingSetIndex,
  3059. IN PVOID SessionSpace
  3060. );
  3061. VOID
  3062. MiInitializeTransitionPfn (
  3063. IN PFN_NUMBER PageFrameIndex,
  3064. IN PMMPTE PointerPte
  3065. );
  3066. extern SLIST_HEADER MmInPageSupportSListHead;
  3067. VOID
  3068. MiFreeInPageSupportBlock (
  3069. IN PMMINPAGE_SUPPORT Support
  3070. );
  3071. PMMINPAGE_SUPPORT
  3072. MiGetInPageSupportBlock (
  3073. IN LOGICAL PfnHeld,
  3074. IN PEPROCESS Process
  3075. );
  3076. //
  3077. // Routines which require a physical page to be mapped into hyperspace
  3078. // within the current process.
  3079. //
  3080. VOID
  3081. FASTCALL
  3082. MiZeroPhysicalPage (
  3083. IN PFN_NUMBER PageFrameIndex,
  3084. IN ULONG Color
  3085. );
  3086. VOID
  3087. FASTCALL
  3088. MiRestoreTransitionPte (
  3089. IN PFN_NUMBER PageFrameIndex
  3090. );
  3091. PSUBSECTION
  3092. MiGetSubsectionAndProtoFromPte (
  3093. IN PMMPTE PointerPte,
  3094. IN PMMPTE *ProtoPte
  3095. );
  3096. PVOID
  3097. MiMapPageInHyperSpace (
  3098. IN PEPROCESS Process,
  3099. IN PFN_NUMBER PageFrameIndex,
  3100. OUT PKIRQL OldIrql
  3101. );
  3102. PVOID
  3103. MiMapPageInHyperSpaceAtDpc (
  3104. IN PEPROCESS Process,
  3105. IN PFN_NUMBER PageFrameIndex
  3106. );
  3107. #define MiUnmapPageInZeroSpace(VA) \
  3108. MiGetPteAddress(VA)->u.Long = 0;
  3109. PVOID
  3110. MiMapImageHeaderInHyperSpace (
  3111. IN PFN_NUMBER PageFrameIndex
  3112. );
  3113. VOID
  3114. MiUnmapImageHeaderInHyperSpace (
  3115. VOID
  3116. );
  3117. VOID
  3118. MiUpdateImageHeaderPage (
  3119. IN PMMPTE PointerPte,
  3120. IN PFN_NUMBER PageFrameNumber,
  3121. IN PCONTROL_AREA ControlArea
  3122. );
  3123. PFN_NUMBER
  3124. MiGetPageForHeader (
  3125. VOID
  3126. );
  3127. VOID
  3128. MiRemoveImageHeaderPage (
  3129. IN PFN_NUMBER PageFrameNumber
  3130. );
  3131. PVOID
  3132. MiMapPageToZeroInHyperSpace (
  3133. IN PFN_NUMBER PageFrameIndex
  3134. );
  3135. NTSTATUS
  3136. MiGetWritablePagesInSection(
  3137. IN PSECTION Section,
  3138. OUT PULONG WritablePages
  3139. );
  3140. //
  3141. // Routines to obtain and release system PTEs.
  3142. //
  3143. PMMPTE
  3144. MiReserveSystemPtes (
  3145. IN ULONG NumberOfPtes,
  3146. IN MMSYSTEM_PTE_POOL_TYPE SystemPteType
  3147. );
  3148. PMMPTE
  3149. MiReserveAlignedSystemPtes (
  3150. IN ULONG NumberOfPtes,
  3151. IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType,
  3152. IN ULONG Alignment
  3153. );
  3154. VOID
  3155. MiReleaseSystemPtes (
  3156. IN PMMPTE StartingPte,
  3157. IN ULONG NumberOfPtes,
  3158. IN MMSYSTEM_PTE_POOL_TYPE SystemPteType
  3159. );
  3160. VOID
  3161. MiReleaseSplitSystemPtes (
  3162. IN PMMPTE StartingPte,
  3163. IN ULONG NumberOfPtes,
  3164. IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType
  3165. );
  3166. VOID
  3167. MiIncrementSystemPtes (
  3168. IN ULONG NumberOfPtes
  3169. );
  3170. LOGICAL
  3171. MiGetSystemPteAvailability (
  3172. IN ULONG NumberOfPtes,
  3173. IN MM_PAGE_PRIORITY Priority
  3174. );
  3175. VOID
  3176. MiIssueNoPtesBugcheck (
  3177. IN ULONG NumberOfPtes,
  3178. IN MMSYSTEM_PTE_POOL_TYPE SystemPteType
  3179. );
  3180. VOID
  3181. MiInitializeSystemPtes (
  3182. IN PMMPTE StartingPte,
  3183. IN ULONG NumberOfPtes,
  3184. IN MMSYSTEM_PTE_POOL_TYPE SystemPteType
  3185. );
  3186. NTSTATUS
  3187. MiAddMappedPtes (
  3188. IN PMMPTE FirstPte,
  3189. IN ULONG NumberOfPtes,
  3190. IN PCONTROL_AREA ControlArea
  3191. );
  3192. VOID
  3193. MiInitializeIoTrackers (
  3194. VOID
  3195. );
  3196. PVOID
  3197. MiMapSinglePage (
  3198. IN PVOID VirtualAddress OPTIONAL,
  3199. IN PFN_NUMBER PageFrameIndex,
  3200. IN MEMORY_CACHING_TYPE CacheType,
  3201. IN MM_PAGE_PRIORITY Priority
  3202. );
  3203. VOID
  3204. MiUnmapSinglePage (
  3205. IN PVOID BaseAddress
  3206. );
  3207. typedef struct _MM_PTE_MAPPING {
  3208. LIST_ENTRY ListEntry;
  3209. PVOID SystemVa;
  3210. PVOID SystemEndVa;
  3211. ULONG Protection;
  3212. } MM_PTE_MAPPING, *PMM_PTE_MAPPING;
  3213. extern LIST_ENTRY MmProtectedPteList;
  3214. extern KSPIN_LOCK MmProtectedPteLock;
  3215. LOGICAL
  3216. MiCheckSystemPteProtection (
  3217. IN ULONG_PTR StoreInstruction,
  3218. IN PVOID VirtualAddress
  3219. );
  3220. //
  3221. // Access Fault routines.
  3222. //
  3223. #define STATUS_ISSUE_PAGING_IO (0xC0033333)
  3224. NTSTATUS
  3225. MiDispatchFault (
  3226. IN ULONG_PTR FaultStatus,
  3227. IN PVOID VirtualAdress,
  3228. IN PMMPTE PointerPte,
  3229. IN PMMPTE PointerProtoPte,
  3230. IN PEPROCESS Process,
  3231. OUT PLOGICAL ApcNeeded
  3232. );
  3233. NTSTATUS
  3234. MiResolveDemandZeroFault (
  3235. IN PVOID VirtualAddress,
  3236. IN PMMPTE PointerPte,
  3237. IN PEPROCESS Process,
  3238. IN ULONG PrototypePte
  3239. );
  3240. NTSTATUS
  3241. MiResolveTransitionFault (
  3242. IN PVOID FaultingAddress,
  3243. IN PMMPTE PointerPte,
  3244. IN PEPROCESS Process,
  3245. IN ULONG PfnLockHeld,
  3246. OUT PLOGICAL ApcNeeded,
  3247. OUT PMMINPAGE_SUPPORT *InPageBlock
  3248. );
  3249. NTSTATUS
  3250. MiResolvePageFileFault (
  3251. IN PVOID FaultingAddress,
  3252. IN PMMPTE PointerPte,
  3253. IN PMMINPAGE_SUPPORT *ReadBlock,
  3254. IN PEPROCESS Process
  3255. );
  3256. NTSTATUS
  3257. MiResolveProtoPteFault (
  3258. IN ULONG_PTR StoreInstruction,
  3259. IN PVOID VirtualAddress,
  3260. IN PMMPTE PointerPte,
  3261. IN PMMPTE PointerProtoPte,
  3262. IN PMMINPAGE_SUPPORT *ReadBlock,
  3263. IN PEPROCESS Process,
  3264. OUT PLOGICAL ApcNeeded
  3265. );
  3266. NTSTATUS
  3267. MiResolveMappedFileFault (
  3268. IN PVOID FaultingAddress,
  3269. IN PMMPTE PointerPte,
  3270. IN PMMINPAGE_SUPPORT *ReadBlock,
  3271. IN PEPROCESS Process
  3272. );
  3273. VOID
  3274. MiAddValidPageToWorkingSet (
  3275. IN PVOID VirtualAddress,
  3276. IN PMMPTE PointerPte,
  3277. IN PMMPFN Pfn1,
  3278. IN ULONG WsleMask
  3279. );
  3280. NTSTATUS
  3281. MiWaitForInPageComplete (
  3282. IN PMMPFN Pfn,
  3283. IN PMMPTE PointerPte,
  3284. IN PVOID FaultingAddress,
  3285. IN PMMPTE PointerPteContents,
  3286. IN PMMINPAGE_SUPPORT InPageSupport,
  3287. IN PEPROCESS CurrentProcess
  3288. );
  3289. LOGICAL
  3290. FASTCALL
  3291. MiCopyOnWrite (
  3292. IN PVOID FaultingAddress,
  3293. IN PMMPTE PointerPte
  3294. );
  3295. VOID
  3296. MiSetDirtyBit (
  3297. IN PVOID FaultingAddress,
  3298. IN PMMPTE PointerPte,
  3299. IN ULONG PfnHeld
  3300. );
  3301. VOID
  3302. MiSetModifyBit (
  3303. IN PMMPFN Pfn
  3304. );
  3305. PMMPTE
  3306. MiFindActualFaultingPte (
  3307. IN PVOID FaultingAddress
  3308. );
  3309. VOID
  3310. MiInitializeReadInProgressSinglePfn (
  3311. IN PFN_NUMBER PageFrameIndex,
  3312. IN PMMPTE BasePte,
  3313. IN PKEVENT Event,
  3314. IN WSLE_NUMBER WorkingSetIndex
  3315. );
  3316. VOID
  3317. MiInitializeReadInProgressPfn (
  3318. IN PMDL Mdl,
  3319. IN PMMPTE BasePte,
  3320. IN PKEVENT Event,
  3321. IN WSLE_NUMBER WorkingSetIndex
  3322. );
  3323. NTSTATUS
  3324. MiAccessCheck (
  3325. IN PMMPTE PointerPte,
  3326. IN ULONG_PTR WriteOperation,
  3327. IN KPROCESSOR_MODE PreviousMode,
  3328. IN ULONG Protection,
  3329. IN BOOLEAN CallerHoldsPfnLock
  3330. );
  3331. NTSTATUS
  3332. FASTCALL
  3333. MiCheckForUserStackOverflow (
  3334. IN PVOID FaultingAddress
  3335. );
  3336. PMMPTE
  3337. MiCheckVirtualAddress (
  3338. IN PVOID VirtualAddress,
  3339. OUT PULONG ProtectCode
  3340. );
  3341. NTSTATUS
  3342. FASTCALL
  3343. MiCheckPdeForPagedPool (
  3344. IN PVOID VirtualAddress
  3345. );
  3346. //
  3347. // Routines which operate on an address tree.
  3348. //
  3349. PMMADDRESS_NODE
  3350. FASTCALL
  3351. MiGetNextNode (
  3352. IN PMMADDRESS_NODE Node
  3353. );
  3354. PMMADDRESS_NODE
  3355. FASTCALL
  3356. MiGetPreviousNode (
  3357. IN PMMADDRESS_NODE Node
  3358. );
  3359. PMMADDRESS_NODE
  3360. FASTCALL
  3361. MiGetFirstNode (
  3362. IN PMMADDRESS_NODE Root
  3363. );
  3364. PMMADDRESS_NODE
  3365. MiGetLastNode (
  3366. IN PMMADDRESS_NODE Root
  3367. );
  3368. VOID
  3369. FASTCALL
  3370. MiInsertNode (
  3371. IN PMMADDRESS_NODE Node,
  3372. IN OUT PMMADDRESS_NODE *Root
  3373. );
  3374. VOID
  3375. FASTCALL
  3376. MiRemoveNode (
  3377. IN PMMADDRESS_NODE Node,
  3378. IN OUT PMMADDRESS_NODE *Root
  3379. );
  3380. PMMADDRESS_NODE
  3381. FASTCALL
  3382. MiLocateAddressInTree (
  3383. IN ULONG_PTR Vpn,
  3384. IN PMMADDRESS_NODE *Root
  3385. );
  3386. PMMADDRESS_NODE
  3387. MiCheckForConflictingNode (
  3388. IN ULONG_PTR StartVpn,
  3389. IN ULONG_PTR EndVpn,
  3390. IN PMMADDRESS_NODE Root
  3391. );
  3392. NTSTATUS
  3393. MiFindEmptyAddressRangeInTree (
  3394. IN SIZE_T SizeOfRange,
  3395. IN ULONG_PTR Alignment,
  3396. IN PMMADDRESS_NODE Root,
  3397. OUT PMMADDRESS_NODE *PreviousVad,
  3398. OUT PVOID *Base
  3399. );
  3400. NTSTATUS
  3401. MiFindEmptyAddressRangeDownTree (
  3402. IN SIZE_T SizeOfRange,
  3403. IN PVOID HighestAddressToEndAt,
  3404. IN ULONG_PTR Alignment,
  3405. IN PMMADDRESS_NODE Root,
  3406. OUT PVOID *Base
  3407. );
  3408. VOID
  3409. NodeTreeWalk (
  3410. PMMADDRESS_NODE Start
  3411. );
  3412. //
  3413. // Routines which operate on the tree of virtual address descriptors.
  3414. //
  3415. NTSTATUS
  3416. MiInsertVad (
  3417. IN PMMVAD Vad
  3418. );
  3419. VOID
  3420. MiRemoveVad (
  3421. IN PMMVAD Vad
  3422. );
  3423. PMMVAD
  3424. FASTCALL
  3425. MiLocateAddress (
  3426. IN PVOID Vad
  3427. );
  3428. NTSTATUS
  3429. MiFindEmptyAddressRange (
  3430. IN SIZE_T SizeOfRange,
  3431. IN ULONG_PTR Alignment,
  3432. IN ULONG QuickCheck,
  3433. IN PVOID *Base
  3434. );
  3435. //
  3436. // Routines which operate on the clone tree structure.
  3437. //
  3438. NTSTATUS
  3439. MiCloneProcessAddressSpace (
  3440. IN PEPROCESS ProcessToClone,
  3441. IN PEPROCESS ProcessToInitialize,
  3442. IN PFN_NUMBER PdePhysicalPage,
  3443. IN PFN_NUMBER HyperPhysicalPage
  3444. );
  3445. ULONG
  3446. MiDecrementCloneBlockReference (
  3447. IN PMMCLONE_DESCRIPTOR CloneDescriptor,
  3448. IN PMMCLONE_BLOCK CloneBlock,
  3449. IN PEPROCESS CurrentProcess
  3450. );
  3451. LOGICAL
  3452. MiWaitForForkToComplete (
  3453. IN PEPROCESS CurrentProcess,
  3454. IN LOGICAL PfnHeld
  3455. );
  3456. //
  3457. // Routines which operate on the working set list.
  3458. //
  3459. WSLE_NUMBER
  3460. MiLocateAndReserveWsle (
  3461. IN PMMSUPPORT WsInfo
  3462. );
  3463. VOID
  3464. MiReleaseWsle (
  3465. IN WSLE_NUMBER WorkingSetIndex,
  3466. IN PMMSUPPORT WsInfo
  3467. );
  3468. VOID
  3469. MiUpdateWsle (
  3470. IN PWSLE_NUMBER DesiredIndex,
  3471. IN PVOID VirtualAddress,
  3472. IN PMMWSL WorkingSetList,
  3473. IN PMMPFN Pfn
  3474. );
  3475. VOID
  3476. MiInitializeWorkingSetList (
  3477. IN PEPROCESS CurrentProcess
  3478. );
  3479. VOID
  3480. MiGrowWsleHash (
  3481. IN PMMSUPPORT WsInfo
  3482. );
  3483. WSLE_NUMBER
  3484. MiTrimWorkingSet (
  3485. IN WSLE_NUMBER Reduction,
  3486. IN PMMSUPPORT WsInfo,
  3487. IN ULONG TrimAge
  3488. );
  3489. LOGICAL
  3490. MmTrimProcessMemory (
  3491. IN LOGICAL PurgeTransition
  3492. );
  3493. LOGICAL
  3494. MmTrimSessionMemory (
  3495. IN LOGICAL PurgeTransition
  3496. );
  3497. VOID
  3498. MiRemoveWorkingSetPages (
  3499. IN PMMWSL WorkingSetList,
  3500. IN PMMSUPPORT WsInfo
  3501. );
  3502. VOID
  3503. MiAgeAndEstimateAvailableInWorkingSet (
  3504. IN PMMSUPPORT VmSupport,
  3505. IN LOGICAL DoAging,
  3506. IN PWSLE_NUMBER WslesScanned,
  3507. IN OUT PPFN_NUMBER TotalClaim,
  3508. IN OUT PPFN_NUMBER TotalEstimatedAvailable
  3509. );
  3510. VOID
  3511. FASTCALL
  3512. MiInsertWsleHash (
  3513. IN WSLE_NUMBER Entry,
  3514. IN PMMWSL WorkingSetList
  3515. );
  3516. VOID
  3517. FASTCALL
  3518. MiRemoveWsle (
  3519. IN WSLE_NUMBER Entry,
  3520. IN PMMWSL WorkingSetList
  3521. );
  3522. WSLE_NUMBER
  3523. FASTCALL
  3524. MiLocateWsle (
  3525. IN PVOID VirtualAddress,
  3526. IN PMMWSL WorkingSetList,
  3527. IN WSLE_NUMBER WsPfnIndex
  3528. );
  3529. ULONG
  3530. MiFreeWsle (
  3531. IN WSLE_NUMBER WorkingSetIndex,
  3532. IN PMMSUPPORT WsInfo,
  3533. IN PMMPTE PointerPte
  3534. );
  3535. VOID
  3536. MiSwapWslEntries (
  3537. IN WSLE_NUMBER SwapEntry,
  3538. IN WSLE_NUMBER Entry,
  3539. IN PMMSUPPORT WsInfo
  3540. );
  3541. VOID
  3542. MiRemoveWsleFromFreeList (
  3543. IN WSLE_NUMBER Entry,
  3544. IN PMMWSLE Wsle,
  3545. IN PMMWSL WorkingSetList
  3546. );
  3547. ULONG
  3548. MiRemovePageFromWorkingSet (
  3549. IN PMMPTE PointerPte,
  3550. IN PMMPFN Pfn1,
  3551. IN PMMSUPPORT WsInfo
  3552. );
  3553. PFN_NUMBER
  3554. MiDeleteSystemPagableVm (
  3555. IN PMMPTE PointerPte,
  3556. IN PFN_NUMBER NumberOfPtes,
  3557. IN MMPTE NewPteValue,
  3558. IN LOGICAL SessionAllocation,
  3559. OUT PPFN_NUMBER ResidentPages OPTIONAL
  3560. );
  3561. VOID
  3562. MiLockCode (
  3563. IN PMMPTE FirstPte,
  3564. IN PMMPTE LastPte,
  3565. IN ULONG LockType
  3566. );
  3567. PKLDR_DATA_TABLE_ENTRY
  3568. MiLookupDataTableEntry (
  3569. IN PVOID AddressWithinSection,
  3570. IN ULONG ResourceHeld
  3571. );
  3572. //
  3573. // Routines which perform working set management.
  3574. //
  3575. VOID
  3576. MiObtainFreePages (
  3577. VOID
  3578. );
  3579. VOID
  3580. MiModifiedPageWriter (
  3581. IN PVOID StartContext
  3582. );
  3583. VOID
  3584. MiMappedPageWriter (
  3585. IN PVOID StartContext
  3586. );
  3587. LOGICAL
  3588. MiIssuePageExtendRequest (
  3589. IN PMMPAGE_FILE_EXPANSION PageExtend
  3590. );
  3591. VOID
  3592. MiIssuePageExtendRequestNoWait (
  3593. IN PFN_NUMBER SizeInPages
  3594. );
  3595. SIZE_T
  3596. MiExtendPagingFiles (
  3597. IN PMMPAGE_FILE_EXPANSION PageExpand
  3598. );
  3599. VOID
  3600. MiContractPagingFiles (
  3601. VOID
  3602. );
  3603. VOID
  3604. MiAttemptPageFileReduction (
  3605. VOID
  3606. );
  3607. LOGICAL
  3608. MiCancelWriteOfMappedPfn (
  3609. IN PFN_NUMBER PageToStop
  3610. );
  3611. //
  3612. // Routines to delete address space.
  3613. //
  3614. VOID
  3615. MiDeletePteRange (
  3616. IN PEPROCESS Process,
  3617. IN PMMPTE PointerPte,
  3618. IN PMMPTE LastPte,
  3619. IN LOGICAL AddressSpaceDeletion
  3620. );
  3621. VOID
  3622. MiDeleteVirtualAddresses (
  3623. IN PUCHAR StartingAddress,
  3624. IN PUCHAR EndingAddress,
  3625. IN ULONG AddressSpaceDeletion,
  3626. IN PMMVAD Vad
  3627. );
  3628. ULONG
  3629. MiDeletePte (
  3630. IN PMMPTE PointerPte,
  3631. IN PVOID VirtualAddress,
  3632. IN ULONG AddressSpaceDeletion,
  3633. IN PEPROCESS CurrentProcess,
  3634. IN PMMPTE PrototypePte,
  3635. IN PMMPTE_FLUSH_LIST PteFlushList OPTIONAL
  3636. );
  3637. VOID
  3638. MiDeletePageTablesForPhysicalRange (
  3639. IN PVOID StartingAddress,
  3640. IN PVOID EndingAddress
  3641. );
  3642. VOID
  3643. MiFlushPteList (
  3644. IN PMMPTE_FLUSH_LIST PteFlushList,
  3645. IN ULONG AllProcessors,
  3646. IN MMPTE FillPte
  3647. );
  3648. ULONG
  3649. FASTCALL
  3650. MiReleasePageFileSpace (
  3651. IN MMPTE PteContents
  3652. );
  3653. VOID
  3654. FASTCALL
  3655. MiReleaseConfirmedPageFileSpace (
  3656. IN MMPTE PteContents
  3657. );
  3658. VOID
  3659. FASTCALL
  3660. MiUpdateModifiedWriterMdls (
  3661. IN ULONG PageFileNumber
  3662. );
  3663. PVOID
  3664. MiAllocateAweInfo (
  3665. VOID
  3666. );
  3667. VOID
  3668. MiRemoveUserPhysicalPagesVad (
  3669. IN PMMVAD_SHORT FoundVad
  3670. );
  3671. VOID
  3672. MiCleanPhysicalProcessPages (
  3673. IN PEPROCESS Process
  3674. );
  3675. VOID
  3676. MiPhysicalViewRemover (
  3677. IN PEPROCESS Process,
  3678. IN PMMVAD Vad
  3679. );
  3680. VOID
  3681. MiPhysicalViewAdjuster (
  3682. IN PEPROCESS Process,
  3683. IN PMMVAD OldVad,
  3684. IN PMMVAD NewVad
  3685. );
  3686. LOGICAL
  3687. MiIsPhysicalMemoryAddress (
  3688. IN PFN_NUMBER PageFrameIndex,
  3689. IN OUT PULONG Hint,
  3690. IN LOGICAL PfnLockNeeded
  3691. );
  3692. //
  3693. // MM_SYSTEM_PAGE_COLOR - MmSystemPageColor
  3694. //
  3695. // This variable is updated frequently, on MP systems we keep
  3696. // a separate system color per processor to avoid cache line
  3697. // thrashing.
  3698. //
  3699. #if defined(NT_UP)
  3700. #define MI_SYSTEM_PAGE_COLOR MmSystemPageColor
  3701. #else
  3702. #define MI_SYSTEM_PAGE_COLOR (KeGetCurrentPrcb()->PageColor)
  3703. #endif
  3704. #if defined(MI_MULTINODE)
  3705. extern PKNODE KeNodeBlock[];
  3706. #define MI_NODE_FROM_COLOR(c) \
  3707. (KeNodeBlock[(c) >> MmSecondaryColorNodeShift])
  3708. #define MI_GET_COLOR_FROM_LIST_ENTRY(index,pfn) \
  3709. ((ULONG)(((pfn)->u3.e1.PageColor << MmSecondaryColorNodeShift) | \
  3710. MI_GET_SECONDARY_COLOR((index),(pfn))))
  3711. #define MI_ADJUST_COLOR_FOR_NODE(c,n) ((c) | (n)->Color)
  3712. #define MI_CURRENT_NODE_COLOR (KeGetCurrentNode()->MmShiftedColor)
  3713. #define MiRemoveZeroPageIfAny(c) \
  3714. (KeGetCurrentNode()->FreeCount[ZeroedPageList] ? MiRemoveZeroPage(c) : 0)
  3715. #define MI_GET_PAGE_COLOR_NODE(n) \
  3716. (((MI_SYSTEM_PAGE_COLOR++) & MmSecondaryColorMask) | \
  3717. KeNodeBlock[n]->MmShiftedColor)
  3718. #else
  3719. #define MI_NODE_FROM_COLOR(c)
  3720. #define MI_GET_COLOR_FROM_LIST_ENTRY(index,pfn) \
  3721. ((ULONG)MI_GET_SECONDARY_COLOR((index),(pfn)))
  3722. #define MI_ADJUST_COLOR_FOR_NODE(c,n) (c)
  3723. #define MI_CURRENT_NODE_COLOR 0
  3724. #define MiRemoveZeroPageIfAny(COLOR) \
  3725. (MmFreePagesByColor[ZeroedPageList][COLOR].Flink != MM_EMPTY_LIST) ? \
  3726. MiRemoveZeroPage(COLOR) : 0
  3727. #define MI_GET_PAGE_COLOR_NODE(n) \
  3728. ((MI_SYSTEM_PAGE_COLOR++) & MmSecondaryColorMask)
  3729. #endif
  3730. FORCEINLINE
  3731. PFN_NUMBER
  3732. MiRemoveZeroPageMayReleaseLocks (
  3733. IN ULONG Color,
  3734. IN KIRQL OldIrql
  3735. )
  3736. /*++
  3737. Routine Description:
  3738. This routine returns a zeroed page.
  3739. It may release and reacquire the PFN lock to do so, as well as mapping
  3740. the page in hyperspace to perform the actual zeroing if necessary.
  3741. Environment:
  3742. Kernel mode. PFN lock held, hyperspace lock NOT held.
  3743. --*/
  3744. {
  3745. PFN_NUMBER PageFrameIndex;
  3746. PageFrameIndex = MiRemoveZeroPageIfAny (Color);
  3747. if (PageFrameIndex == 0) {
  3748. PageFrameIndex = MiRemoveAnyPage (Color);
  3749. UNLOCK_PFN (OldIrql);
  3750. MiZeroPhysicalPage (PageFrameIndex, Color);
  3751. LOCK_PFN (OldIrql);
  3752. }
  3753. return PageFrameIndex;
  3754. }
  3755. //
  3756. // General support routines.
  3757. //
  3758. #if (_MI_PAGING_LEVELS <= 3)
  3759. //++
  3760. //PMMPTE
  3761. //MiGetPxeAddress (
  3762. // IN PVOID va
  3763. // );
  3764. //
  3765. // Routine Description:
  3766. //
  3767. // MiGetPxeAddress returns the address of the extended page directory parent
  3768. // entry which maps the given virtual address. This is one level above the
  3769. // page parent directory.
  3770. //
  3771. // Arguments
  3772. //
  3773. // Va - Supplies the virtual address to locate the PXE for.
  3774. //
  3775. // Return Value:
  3776. //
  3777. // The address of the PXE.
  3778. //
  3779. //--
  3780. #define MiGetPxeAddress(va) ((PMMPTE)0)
  3781. //++
  3782. //LOGICAL
  3783. //MiIsPteOnPxeBoundary (
  3784. // IN PVOID PTE
  3785. // );
  3786. //
  3787. // Routine Description:
  3788. //
  3789. // MiIsPteOnPxeBoundary returns TRUE if the PTE is
  3790. // on an extended page directory parent entry boundary.
  3791. //
  3792. // Arguments
  3793. //
  3794. // PTE - Supplies the PTE to check.
  3795. //
  3796. // Return Value:
  3797. //
  3798. // TRUE if on a boundary, FALSE if not.
  3799. //
  3800. //--
  3801. #define MiIsPteOnPxeBoundary(PTE) (FALSE)
  3802. #endif
  3803. #if (_MI_PAGING_LEVELS <= 2)
  3804. //++
  3805. //PMMPTE
  3806. //MiGetPpeAddress (
  3807. // IN PVOID va
  3808. // );
  3809. //
  3810. // Routine Description:
  3811. //
  3812. // MiGetPpeAddress returns the address of the page directory parent entry
  3813. // which maps the given virtual address. This is one level above the
  3814. // page directory.
  3815. //
  3816. // Arguments
  3817. //
  3818. // Va - Supplies the virtual address to locate the PPE for.
  3819. //
  3820. // Return Value:
  3821. //
  3822. // The address of the PPE.
  3823. //
  3824. //--
  3825. #define MiGetPpeAddress(va) ((PMMPTE)0)
  3826. //++
  3827. //LOGICAL
  3828. //MiIsPteOnPpeBoundary (
  3829. // IN PVOID VA
  3830. // );
  3831. //
  3832. // Routine Description:
  3833. //
  3834. // MiIsPteOnPpeBoundary returns TRUE if the PTE is
  3835. // on a page directory parent entry boundary.
  3836. //
  3837. // Arguments
  3838. //
  3839. // VA - Supplies the virtual address to check.
  3840. //
  3841. // Return Value:
  3842. //
  3843. // TRUE if on a boundary, FALSE if not.
  3844. //
  3845. //--
  3846. #define MiIsPteOnPpeBoundary(PTE) (FALSE)
  3847. #endif
  3848. ULONG
  3849. MiDoesPdeExistAndMakeValid (
  3850. IN PMMPTE PointerPde,
  3851. IN PEPROCESS TargetProcess,
  3852. IN LOGICAL PfnLockHeld,
  3853. OUT PULONG Waited
  3854. );
  3855. #if (_MI_PAGING_LEVELS >= 3)
  3856. #define MiDoesPpeExistAndMakeValid(PPE, PROCESS, PFNLOCKHELD, WAITED) \
  3857. MiDoesPdeExistAndMakeValid(PPE, PROCESS, PFNLOCKHELD, WAITED)
  3858. #else
  3859. #define MiDoesPpeExistAndMakeValid(PPE, PROCESS, PFNLOCKHELD, WAITED) 1
  3860. #endif
  3861. #if (_MI_PAGING_LEVELS >= 4)
  3862. #define MiDoesPxeExistAndMakeValid(PXE, PROCESS, PFNLOCKHELD, WAITED) \
  3863. MiDoesPdeExistAndMakeValid(PXE, PROCESS, PFNLOCKHELD, WAITED)
  3864. #else
  3865. #define MiDoesPxeExistAndMakeValid(PXE, PROCESS, PFNLOCKHELD, WAITED) 1
  3866. #endif
  3867. VOID
  3868. MiMakePdeExistAndMakeValid (
  3869. IN PMMPTE PointerPde,
  3870. IN PEPROCESS TargetProcess,
  3871. IN LOGICAL PfnLockHeld
  3872. );
  3873. #if (_MI_PAGING_LEVELS >= 4)
  3874. VOID
  3875. MiMakePxeExistAndMakeValid (
  3876. IN PMMPTE PointerPpe,
  3877. IN PEPROCESS TargetProcess,
  3878. IN LOGICAL PfnLockHeld
  3879. );
  3880. #else
  3881. #define MiMakePxeExistAndMakeValid(PDE, PROCESS, PFNLOCKHELD)
  3882. #endif
  3883. #if (_MI_PAGING_LEVELS >= 3)
  3884. VOID
  3885. MiMakePpeExistAndMakeValid (
  3886. IN PMMPTE PointerPpe,
  3887. IN PEPROCESS TargetProcess,
  3888. IN LOGICAL PfnLockHeld
  3889. );
  3890. #else
  3891. #define MiMakePpeExistAndMakeValid(PDE, PROCESS, PFNLOCKHELD)
  3892. #endif
  3893. ULONG
  3894. FASTCALL
  3895. MiMakeSystemAddressValid (
  3896. IN PVOID VirtualAddress,
  3897. IN PEPROCESS CurrentProcess
  3898. );
  3899. ULONG
  3900. FASTCALL
  3901. MiMakeSystemAddressValidPfnWs (
  3902. IN PVOID VirtualAddress,
  3903. IN PEPROCESS CurrentProcess OPTIONAL
  3904. );
  3905. ULONG
  3906. FASTCALL
  3907. MiMakeSystemAddressValidPfnSystemWs (
  3908. IN PVOID VirtualAddress
  3909. );
  3910. ULONG
  3911. FASTCALL
  3912. MiMakeSystemAddressValidPfn (
  3913. IN PVOID VirtualAddress
  3914. );
  3915. ULONG
  3916. FASTCALL
  3917. MiLockPagedAddress (
  3918. IN PVOID VirtualAddress,
  3919. IN ULONG PfnLockHeld
  3920. );
  3921. VOID
  3922. FASTCALL
  3923. MiUnlockPagedAddress (
  3924. IN PVOID VirtualAddress,
  3925. IN ULONG PfnLockHeld
  3926. );
  3927. ULONG
  3928. FASTCALL
  3929. MiIsPteDecommittedPage (
  3930. IN PMMPTE PointerPte
  3931. );
  3932. ULONG
  3933. FASTCALL
  3934. MiIsProtectionCompatible (
  3935. IN ULONG OldProtect,
  3936. IN ULONG NewProtect
  3937. );
  3938. ULONG
  3939. FASTCALL
  3940. MiIsPteProtectionCompatible (
  3941. IN ULONG OldPteProtection,
  3942. IN ULONG NewProtect
  3943. );
  3944. ULONG
  3945. FASTCALL
  3946. MiMakeProtectionMask (
  3947. IN ULONG Protect
  3948. );
  3949. ULONG
  3950. MiIsEntireRangeCommitted (
  3951. IN PVOID StartingAddress,
  3952. IN PVOID EndingAddress,
  3953. IN PMMVAD Vad,
  3954. IN PEPROCESS Process
  3955. );
  3956. ULONG
  3957. MiIsEntireRangeDecommitted (
  3958. IN PVOID StartingAddress,
  3959. IN PVOID EndingAddress,
  3960. IN PMMVAD Vad,
  3961. IN PEPROCESS Process
  3962. );
  3963. LOGICAL
  3964. MiCheckProtoPtePageState (
  3965. IN PMMPTE PrototypePte,
  3966. IN LOGICAL PfnLockHeld,
  3967. OUT PLOGICAL DroppedPfnLock
  3968. );
  3969. //++
  3970. //PMMPTE
  3971. //MiGetProtoPteAddress (
  3972. // IN PMMPTE VAD,
  3973. // IN PVOID VA
  3974. // );
  3975. //
  3976. // Routine Description:
  3977. //
  3978. // MiGetProtoPteAddress returns a pointer to the prototype PTE which
  3979. // is mapped by the given virtual address descriptor and address within
  3980. // the virtual address descriptor.
  3981. //
  3982. // Arguments
  3983. //
  3984. // VAD - Supplies a pointer to the virtual address descriptor that contains
  3985. // the VA.
  3986. //
  3987. // VPN - Supplies the virtual page number.
  3988. //
  3989. // Return Value:
  3990. //
  3991. // A pointer to the proto PTE which corresponds to the VA.
  3992. //
  3993. //--
  3994. #define MiGetProtoPteAddress(VAD,VPN) \
  3995. ((((((VPN) - (VAD)->StartingVpn) << PTE_SHIFT) + \
  3996. (ULONG_PTR)(VAD)->FirstPrototypePte) <= (ULONG_PTR)(VAD)->LastContiguousPte) ? \
  3997. ((PMMPTE)(((((VPN) - (VAD)->StartingVpn) << PTE_SHIFT) + \
  3998. (ULONG_PTR)(VAD)->FirstPrototypePte))) : \
  3999. MiGetProtoPteAddressExtended ((VAD),(VPN)))
  4000. PMMPTE
  4001. FASTCALL
  4002. MiGetProtoPteAddressExtended (
  4003. IN PMMVAD Vad,
  4004. IN ULONG_PTR Vpn
  4005. );
  4006. PSUBSECTION
  4007. FASTCALL
  4008. MiLocateSubsection (
  4009. IN PMMVAD Vad,
  4010. IN ULONG_PTR Vpn
  4011. );
  4012. VOID
  4013. MiInitializeSystemCache (
  4014. IN ULONG MinimumWorkingSet,
  4015. IN ULONG MaximumWorkingSet
  4016. );
  4017. VOID
  4018. MiAdjustWorkingSetManagerParameters(
  4019. IN LOGICAL WorkStation
  4020. );
  4021. VOID
  4022. MiNotifyMemoryEvents (
  4023. VOID
  4024. );
  4025. extern PFN_NUMBER MmLowMemoryThreshold;
  4026. extern PFN_NUMBER MmHighMemoryThreshold;
  4027. //
  4028. // Section support
  4029. //
  4030. VOID
  4031. FASTCALL
  4032. MiInsertBasedSection (
  4033. IN PSECTION Section
  4034. );
  4035. NTSTATUS
  4036. MiMapViewOfPhysicalSection (
  4037. IN PCONTROL_AREA ControlArea,
  4038. IN PEPROCESS Process,
  4039. IN PVOID *CapturedBase,
  4040. IN PLARGE_INTEGER SectionOffset,
  4041. IN PSIZE_T CapturedViewSize,
  4042. IN ULONG ProtectionMask,
  4043. IN ULONG_PTR ZeroBits,
  4044. IN ULONG AllocationType,
  4045. IN LOGICAL WriteCombined
  4046. );
  4047. NTSTATUS
  4048. MiMapViewOfDataSection (
  4049. IN PCONTROL_AREA ControlArea,
  4050. IN PEPROCESS Process,
  4051. IN PVOID *CapturedBase,
  4052. IN PLARGE_INTEGER SectionOffset,
  4053. IN PSIZE_T CapturedViewSize,
  4054. IN PSECTION Section,
  4055. IN SECTION_INHERIT InheritDisposition,
  4056. IN ULONG ProtectionMask,
  4057. IN SIZE_T CommitSize,
  4058. IN ULONG_PTR ZeroBits,
  4059. IN ULONG AllocationType
  4060. );
  4061. NTSTATUS
  4062. MiUnmapViewOfSection (
  4063. IN PEPROCESS Process,
  4064. IN PVOID BaseAddress,
  4065. IN LOGICAL AddressSpaceMutexHeld
  4066. );
  4067. VOID
  4068. MiRemoveImageSectionObject(
  4069. IN PFILE_OBJECT File,
  4070. IN PCONTROL_AREA ControlArea
  4071. );
  4072. VOID
  4073. MiAddSystemPtes(
  4074. IN PMMPTE StartingPte,
  4075. IN ULONG NumberOfPtes,
  4076. IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType
  4077. );
  4078. VOID
  4079. MiRemoveMappedView (
  4080. IN PEPROCESS CurrentProcess,
  4081. IN PMMVAD Vad
  4082. );
  4083. VOID
  4084. MiSegmentDelete (
  4085. PSEGMENT Segment
  4086. );
  4087. VOID
  4088. MiSectionDelete (
  4089. IN PVOID Object
  4090. );
  4091. VOID
  4092. MiDereferenceSegmentThread (
  4093. IN PVOID StartContext
  4094. );
  4095. NTSTATUS
  4096. MiCreateImageFileMap (
  4097. IN PFILE_OBJECT File,
  4098. OUT PSEGMENT *Segment
  4099. );
  4100. NTSTATUS
  4101. MiCreateDataFileMap (
  4102. IN PFILE_OBJECT File,
  4103. OUT PSEGMENT *Segment,
  4104. IN PUINT64 MaximumSize,
  4105. IN ULONG SectionPageProtection,
  4106. IN ULONG AllocationAttributes,
  4107. IN ULONG IgnoreFileSizing
  4108. );
  4109. NTSTATUS
  4110. MiCreatePagingFileMap (
  4111. OUT PSEGMENT *Segment,
  4112. IN PUINT64 MaximumSize,
  4113. IN ULONG ProtectionMask,
  4114. IN ULONG AllocationAttributes
  4115. );
  4116. VOID
  4117. MiPurgeSubsectionInternal (
  4118. IN PSUBSECTION Subsection,
  4119. IN ULONG PteOffset
  4120. );
  4121. VOID
  4122. MiPurgeImageSection (
  4123. IN PCONTROL_AREA ControlArea,
  4124. IN PEPROCESS Process
  4125. );
  4126. VOID
  4127. MiCleanSection (
  4128. IN PCONTROL_AREA ControlArea,
  4129. IN LOGICAL DirtyDataPagesOk
  4130. );
  4131. VOID
  4132. MiDereferenceControlArea (
  4133. IN PCONTROL_AREA ControlArea
  4134. );
  4135. VOID
  4136. MiCheckControlArea (
  4137. IN PCONTROL_AREA ControlArea,
  4138. IN PEPROCESS CurrentProcess,
  4139. IN KIRQL PreviousIrql
  4140. );
  4141. LOGICAL
  4142. MiCheckPurgeAndUpMapCount (
  4143. IN PCONTROL_AREA ControlArea
  4144. );
  4145. VOID
  4146. MiCheckForControlAreaDeletion (
  4147. IN PCONTROL_AREA ControlArea
  4148. );
  4149. LOGICAL
  4150. MiCheckControlAreaStatus (
  4151. IN SECTION_CHECK_TYPE SectionCheckType,
  4152. IN PSECTION_OBJECT_POINTERS SectionObjectPointers,
  4153. IN ULONG DelayClose,
  4154. OUT PCONTROL_AREA *ControlArea,
  4155. OUT PKIRQL OldIrql
  4156. );
  4157. extern SLIST_HEADER MmEventCountSListHead;
  4158. PEVENT_COUNTER
  4159. MiGetEventCounter (
  4160. VOID
  4161. );
  4162. VOID
  4163. MiFreeEventCounter (
  4164. IN PEVENT_COUNTER Support
  4165. );
  4166. ULONG
  4167. MiCanFileBeTruncatedInternal (
  4168. IN PSECTION_OBJECT_POINTERS SectionPointer,
  4169. IN PLARGE_INTEGER NewFileSize OPTIONAL,
  4170. IN LOGICAL BlockNewViews,
  4171. OUT PKIRQL PreviousIrql
  4172. );
  4173. #define STATUS_MAPPED_WRITER_COLLISION (0xC0033333)
  4174. NTSTATUS
  4175. MiFlushSectionInternal (
  4176. IN PMMPTE StartingPte,
  4177. IN PMMPTE FinalPte,
  4178. IN PSUBSECTION FirstSubsection,
  4179. IN PSUBSECTION LastSubsection,
  4180. IN ULONG Synchronize,
  4181. IN LOGICAL WriteInProgressOk,
  4182. OUT PIO_STATUS_BLOCK IoStatus
  4183. );
  4184. //
  4185. // protection stuff...
  4186. //
  4187. NTSTATUS
  4188. MiProtectVirtualMemory (
  4189. IN PEPROCESS Process,
  4190. IN PVOID *CapturedBase,
  4191. IN PSIZE_T CapturedRegionSize,
  4192. IN ULONG Protect,
  4193. IN PULONG LastProtect
  4194. );
  4195. ULONG
  4196. MiGetPageProtection (
  4197. IN PMMPTE PointerPte,
  4198. IN PEPROCESS Process,
  4199. IN LOGICAL PteCapturedToLocalStack
  4200. );
  4201. NTSTATUS
  4202. MiSetProtectionOnSection (
  4203. IN PEPROCESS Process,
  4204. IN PMMVAD Vad,
  4205. IN PVOID StartingAddress,
  4206. IN PVOID EndingAddress,
  4207. IN ULONG NewProtect,
  4208. OUT PULONG CapturedOldProtect,
  4209. IN ULONG DontCharge,
  4210. OUT PULONG Locked
  4211. );
  4212. NTSTATUS
  4213. MiCheckSecuredVad (
  4214. IN PMMVAD Vad,
  4215. IN PVOID Base,
  4216. IN ULONG_PTR Size,
  4217. IN ULONG ProtectionMask
  4218. );
  4219. HANDLE
  4220. MiSecureVirtualMemory (
  4221. IN PVOID Address,
  4222. IN SIZE_T Size,
  4223. IN ULONG ProbeMode,
  4224. IN LOGICAL AddressSpaceMutexHeld
  4225. );
  4226. VOID
  4227. MiUnsecureVirtualMemory (
  4228. IN HANDLE SecureHandle,
  4229. IN LOGICAL AddressSpaceMutexHeld
  4230. );
  4231. ULONG
  4232. MiChangeNoAccessForkPte (
  4233. IN PMMPTE PointerPte,
  4234. IN ULONG ProtectionMask
  4235. );
  4236. VOID
  4237. MiSetImageProtect (
  4238. IN PSEGMENT Segment,
  4239. IN ULONG Protection
  4240. );
  4241. //
  4242. // Routines for charging quota and commitment.
  4243. //
  4244. VOID
  4245. MiTrimSegmentCache (
  4246. VOID
  4247. );
  4248. VOID
  4249. MiInitializeCommitment (
  4250. VOID
  4251. );
  4252. LOGICAL
  4253. FASTCALL
  4254. MiChargeCommitment (
  4255. IN SIZE_T QuotaCharge,
  4256. IN PEPROCESS Process OPTIONAL
  4257. );
  4258. LOGICAL
  4259. FASTCALL
  4260. MiChargeCommitmentCantExpand (
  4261. IN SIZE_T QuotaCharge,
  4262. IN ULONG MustSucceed
  4263. );
  4264. LOGICAL
  4265. FASTCALL
  4266. MiChargeTemporaryCommitmentForReduction (
  4267. IN SIZE_T QuotaCharge
  4268. );
  4269. #if defined (_MI_DEBUG_COMMIT_LEAKS)
  4270. VOID
  4271. FASTCALL
  4272. MiReturnCommitment (
  4273. IN SIZE_T QuotaCharge
  4274. );
  4275. #else
  4276. #define MiReturnCommitment(_QuotaCharge) \
  4277. ASSERT ((SSIZE_T)(_QuotaCharge) >= 0); \
  4278. ASSERT (MmTotalCommittedPages >= (_QuotaCharge)); \
  4279. InterlockedExchangeAddSizeT (&MmTotalCommittedPages, 0-((SIZE_T)(_QuotaCharge))); \
  4280. MM_TRACK_COMMIT (MM_DBG_COMMIT_RETURN_NORMAL, (_QuotaCharge));
  4281. #endif
  4282. VOID
  4283. MiCauseOverCommitPopup (
  4284. VOID
  4285. );
  4286. extern SIZE_T MmPeakCommitment;
  4287. extern SIZE_T MmTotalCommitLimitMaximum;
  4288. SIZE_T
  4289. MiCalculatePageCommitment (
  4290. IN PVOID StartingAddress,
  4291. IN PVOID EndingAddress,
  4292. IN PMMVAD Vad,
  4293. IN PEPROCESS Process
  4294. );
  4295. VOID
  4296. MiReturnPageTablePageCommitment (
  4297. IN PVOID StartingAddress,
  4298. IN PVOID EndingAddress,
  4299. IN PEPROCESS CurrentProcess,
  4300. IN PMMVAD PreviousVad,
  4301. IN PMMVAD NextVad
  4302. );
  4303. VOID
  4304. MiFlushAllPages (
  4305. VOID
  4306. );
  4307. VOID
  4308. MiModifiedPageWriterTimerDispatch (
  4309. IN PKDPC Dpc,
  4310. IN PVOID DeferredContext,
  4311. IN PVOID SystemArgument1,
  4312. IN PVOID SystemArgument2
  4313. );
  4314. LONGLONG
  4315. MiStartingOffset(
  4316. IN PSUBSECTION Subsection,
  4317. IN PMMPTE PteAddress
  4318. );
  4319. LARGE_INTEGER
  4320. MiEndingOffset(
  4321. IN PSUBSECTION Subsection
  4322. );
  4323. VOID
  4324. MiReloadBootLoadedDrivers (
  4325. IN PLOADER_PARAMETER_BLOCK LoaderBlock
  4326. );
  4327. LOGICAL
  4328. MiInitializeLoadedModuleList (
  4329. IN PLOADER_PARAMETER_BLOCK LoaderBlock
  4330. );
  4331. extern ULONG MmSpecialPoolTag;
  4332. extern PVOID MmSpecialPoolStart;
  4333. extern PVOID MmSpecialPoolEnd;
  4334. extern PVOID MmSessionSpecialPoolStart;
  4335. extern PVOID MmSessionSpecialPoolEnd;
  4336. LOGICAL
  4337. MiInitializeSpecialPool (
  4338. IN POOL_TYPE PoolType
  4339. );
  4340. LOGICAL
  4341. MiIsSpecialPoolAddressNonPaged (
  4342. IN PVOID VirtualAddress
  4343. );
  4344. #if defined (_WIN64)
  4345. LOGICAL
  4346. MiInitializeSessionSpecialPool (
  4347. VOID
  4348. );
  4349. VOID
  4350. MiDeleteSessionSpecialPool (
  4351. VOID
  4352. );
  4353. #endif
  4354. #if defined (_X86_)
  4355. LOGICAL
  4356. MiRecoverSpecialPtes (
  4357. IN ULONG NumberOfPtes
  4358. );
  4359. #endif
  4360. VOID
  4361. MiEnableRandomSpecialPool (
  4362. IN LOGICAL Enable
  4363. );
  4364. LOGICAL
  4365. MiTriageSystem (
  4366. IN PLOADER_PARAMETER_BLOCK LoaderBlock
  4367. );
  4368. LOGICAL
  4369. MiTriageAddDrivers (
  4370. IN PLOADER_PARAMETER_BLOCK LoaderBlock
  4371. );
  4372. LOGICAL
  4373. MiTriageVerifyDriver (
  4374. IN PKLDR_DATA_TABLE_ENTRY DataTableEntry
  4375. );
  4376. extern ULONG MmTriageActionTaken;
  4377. #if defined (_WIN64)
  4378. #define MM_SPECIAL_POOL_PTES ((1024 * 1024) / sizeof (MMPTE))
  4379. #else
  4380. #define MM_SPECIAL_POOL_PTES (24 * PTE_PER_PAGE)
  4381. #endif
  4382. #define MI_SUSPECT_DRIVER_BUFFER_LENGTH 512
  4383. extern WCHAR MmVerifyDriverBuffer[];
  4384. extern ULONG MmVerifyDriverBufferLength;
  4385. extern ULONG MmVerifyDriverLevel;
  4386. extern LOGICAL MmDontVerifyRandomDrivers;
  4387. extern LOGICAL MmSnapUnloads;
  4388. extern LOGICAL MmProtectFreedNonPagedPool;
  4389. extern ULONG MmEnforceWriteProtection;
  4390. extern LOGICAL MmTrackLockedPages;
  4391. extern ULONG MmTrackPtes;
  4392. #define VI_POOL_FREELIST_END ((ULONG_PTR)-1)
  4393. typedef struct _VI_POOL_ENTRY_INUSE {
  4394. PVOID VirtualAddress;
  4395. PVOID CallingAddress;
  4396. SIZE_T NumberOfBytes;
  4397. ULONG_PTR Tag;
  4398. } VI_POOL_ENTRY_INUSE, *PVI_POOL_ENTRY_INUSE;
  4399. typedef struct _VI_POOL_ENTRY {
  4400. union {
  4401. VI_POOL_ENTRY_INUSE InUse;
  4402. ULONG_PTR FreeListNext;
  4403. };
  4404. } VI_POOL_ENTRY, *PVI_POOL_ENTRY;
  4405. #define MI_VERIFIER_ENTRY_SIGNATURE 0x98761940
  4406. typedef struct _MI_VERIFIER_DRIVER_ENTRY {
  4407. LIST_ENTRY Links;
  4408. ULONG Loads;
  4409. ULONG Unloads;
  4410. UNICODE_STRING BaseName;
  4411. PVOID StartAddress;
  4412. PVOID EndAddress;
  4413. #define VI_VERIFYING_DIRECTLY 0x1
  4414. #define VI_VERIFYING_INVERSELY 0x2
  4415. #define VI_DISABLE_VERIFICATION 0x4
  4416. ULONG Flags;
  4417. ULONG_PTR Signature;
  4418. ULONG_PTR Reserved;
  4419. KSPIN_LOCK VerifierPoolLock;
  4420. PVI_POOL_ENTRY PoolHash;
  4421. ULONG_PTR PoolHashSize;
  4422. ULONG_PTR PoolHashFree;
  4423. ULONG_PTR PoolHashReserved;
  4424. ULONG CurrentPagedPoolAllocations;
  4425. ULONG CurrentNonPagedPoolAllocations;
  4426. ULONG PeakPagedPoolAllocations;
  4427. ULONG PeakNonPagedPoolAllocations;
  4428. SIZE_T PagedBytes;
  4429. SIZE_T NonPagedBytes;
  4430. SIZE_T PeakPagedBytes;
  4431. SIZE_T PeakNonPagedBytes;
  4432. } MI_VERIFIER_DRIVER_ENTRY, *PMI_VERIFIER_DRIVER_ENTRY;
  4433. typedef struct _MI_VERIFIER_POOL_HEADER {
  4434. ULONG_PTR ListIndex;
  4435. PMI_VERIFIER_DRIVER_ENTRY Verifier;
  4436. } MI_VERIFIER_POOL_HEADER, *PMI_VERIFIER_POOL_HEADER;
  4437. typedef struct _MM_DRIVER_VERIFIER_DATA {
  4438. ULONG Level;
  4439. ULONG RaiseIrqls;
  4440. ULONG AcquireSpinLocks;
  4441. ULONG SynchronizeExecutions;
  4442. ULONG AllocationsAttempted;
  4443. ULONG AllocationsSucceeded;
  4444. ULONG AllocationsSucceededSpecialPool;
  4445. ULONG AllocationsWithNoTag;
  4446. ULONG TrimRequests;
  4447. ULONG Trims;
  4448. ULONG AllocationsFailed;
  4449. ULONG AllocationsFailedDeliberately;
  4450. ULONG Loads;
  4451. ULONG Unloads;
  4452. ULONG UnTrackedPool;
  4453. ULONG UserTrims;
  4454. ULONG CurrentPagedPoolAllocations;
  4455. ULONG CurrentNonPagedPoolAllocations;
  4456. ULONG PeakPagedPoolAllocations;
  4457. ULONG PeakNonPagedPoolAllocations;
  4458. SIZE_T PagedBytes;
  4459. SIZE_T NonPagedBytes;
  4460. SIZE_T PeakPagedBytes;
  4461. SIZE_T PeakNonPagedBytes;
  4462. ULONG BurstAllocationsFailedDeliberately;
  4463. ULONG SessionTrims;
  4464. ULONG Reserved[2];
  4465. } MM_DRIVER_VERIFIER_DATA, *PMM_DRIVER_VERIFIER_DATA;
  4466. LOGICAL
  4467. MiInitializeDriverVerifierList (
  4468. IN PLOADER_PARAMETER_BLOCK LoaderBlock
  4469. );
  4470. LOGICAL
  4471. MiInitializeVerifyingComponents (
  4472. IN PLOADER_PARAMETER_BLOCK LoaderBlock
  4473. );
  4474. LOGICAL
  4475. MiApplyDriverVerifier (
  4476. IN PKLDR_DATA_TABLE_ENTRY,
  4477. IN PMI_VERIFIER_DRIVER_ENTRY Verifier
  4478. );
  4479. VOID
  4480. MiReApplyVerifierToLoadedModules(
  4481. IN PLIST_ENTRY ModuleListHead
  4482. );
  4483. VOID
  4484. MiVerifyingDriverUnloading (
  4485. IN PKLDR_DATA_TABLE_ENTRY DataTableEntry
  4486. );
  4487. VOID
  4488. MiVerifierCheckThunks (
  4489. IN PKLDR_DATA_TABLE_ENTRY DataTableEntry
  4490. );
  4491. extern ULONG MiActiveVerifierThunks;
  4492. extern LIST_ENTRY MiSuspectDriverList;
  4493. extern ULONG MiVerifierThunksAdded;
  4494. VOID
  4495. MiEnableKernelVerifier (
  4496. VOID
  4497. );
  4498. extern LOGICAL KernelVerifier;
  4499. extern MM_DRIVER_VERIFIER_DATA MmVerifierData;
  4500. #define MI_FREED_SPECIAL_POOL_SIGNATURE 0x98764321
  4501. #define MI_STACK_BYTES 1024
  4502. typedef struct _MI_FREED_SPECIAL_POOL {
  4503. POOL_HEADER OverlaidPoolHeader;
  4504. MI_VERIFIER_POOL_HEADER OverlaidVerifierPoolHeader;
  4505. ULONG Signature;
  4506. ULONG TickCount;
  4507. ULONG NumberOfBytesRequested;
  4508. ULONG Pagable;
  4509. PVOID VirtualAddress;
  4510. PVOID StackPointer;
  4511. ULONG StackBytes;
  4512. PETHREAD Thread;
  4513. UCHAR StackData[MI_STACK_BYTES];
  4514. } MI_FREED_SPECIAL_POOL, *PMI_FREED_SPECIAL_POOL;
  4515. #define MM_DBG_COMMIT_NONPAGED_POOL_EXPANSION 0
  4516. #define MM_DBG_COMMIT_PAGED_POOL_PAGETABLE 1
  4517. #define MM_DBG_COMMIT_PAGED_POOL_PAGES 2
  4518. #define MM_DBG_COMMIT_SESSION_POOL_PAGE_TABLES 3
  4519. #define MM_DBG_COMMIT_ALLOCVM1 4
  4520. #define MM_DBG_COMMIT_ALLOCVM_SEGMENT 5
  4521. #define MM_DBG_COMMIT_IMAGE 6
  4522. #define MM_DBG_COMMIT_PAGEFILE_BACKED_SHMEM 7
  4523. #define MM_DBG_COMMIT_INDEPENDENT_PAGES 8
  4524. #define MM_DBG_COMMIT_CONTIGUOUS_PAGES 9
  4525. #define MM_DBG_COMMIT_MDL_PAGES 0xA
  4526. #define MM_DBG_COMMIT_NONCACHED_PAGES 0xB
  4527. #define MM_DBG_COMMIT_MAPVIEW_DATA 0xC
  4528. #define MM_DBG_COMMIT_FILL_SYSTEM_DIRECTORY 0xD
  4529. #define MM_DBG_COMMIT_EXTRA_SYSTEM_PTES 0xE
  4530. #define MM_DBG_COMMIT_DRIVER_PAGING_AT_INIT 0xF
  4531. #define MM_DBG_COMMIT_PAGEFILE_FULL 0x10
  4532. #define MM_DBG_COMMIT_PROCESS_CREATE 0x11
  4533. #define MM_DBG_COMMIT_KERNEL_STACK_CREATE 0x12
  4534. #define MM_DBG_COMMIT_SET_PROTECTION 0x13
  4535. #define MM_DBG_COMMIT_SESSION_CREATE 0x14
  4536. #define MM_DBG_COMMIT_SESSION_IMAGE_PAGES 0x15
  4537. #define MM_DBG_COMMIT_SESSION_PAGETABLE_PAGES 0x16
  4538. #define MM_DBG_COMMIT_SESSION_SHARED_IMAGE 0x17
  4539. #define MM_DBG_COMMIT_DRIVER_PAGES 0x18
  4540. #define MM_DBG_COMMIT_INSERT_VAD 0x19
  4541. #define MM_DBG_COMMIT_SESSION_WS_INIT 0x1A
  4542. #define MM_DBG_COMMIT_SESSION_ADDITIONAL_WS_PAGES 0x1B
  4543. #define MM_DBG_COMMIT_SESSION_ADDITIONAL_WS_HASHPAGES 0x1C
  4544. #define MM_DBG_COMMIT_SPECIAL_POOL_PAGES 0x1D
  4545. #define MM_DBG_COMMIT_SPECIAL_POOL_MAPPING_PAGES 0x1E
  4546. #define MM_DBG_COMMIT_SMALL 0x1F
  4547. #define MM_DBG_COMMIT_EXTRA_WS_PAGES 0x20
  4548. #define MM_DBG_COMMIT_EXTRA_INITIAL_SESSION_WS_PAGES 0x21
  4549. #define MM_DBG_COMMIT_ALLOCVM_PROCESS 0x22
  4550. #define MM_DBG_COMMIT_INSERT_VAD_PT 0x23
  4551. #define MM_DBG_COMMIT_ALLOCVM_PROCESS2 0x24
  4552. #define MM_DBG_COMMIT_CHARGE_NORMAL 0x25
  4553. #define MM_DBG_COMMIT_CHARGE_CAUSE_POPUP 0x26
  4554. #define MM_DBG_COMMIT_CHARGE_CANT_EXPAND 0x27
  4555. #define MM_DBG_COMMIT_RETURN_NONPAGED_POOL_EXPANSION 0x40
  4556. #define MM_DBG_COMMIT_RETURN_PAGED_POOL_PAGES 0x41
  4557. #define MM_DBG_COMMIT_RETURN_SESSION_DATAPAGE 0x42
  4558. #define MM_DBG_COMMIT_RETURN_ALLOCVM_SEGMENT 0x43
  4559. #define MM_DBG_COMMIT_RETURN_ALLOCVM2 0x44
  4560. #define MM_DBG_COMMIT_RETURN_IMAGE_NO_LARGE_CA 0x46
  4561. #define MM_DBG_COMMIT_RETURN_PTE_RANGE 0x47
  4562. #define MM_DBG_COMMIT_RETURN_NTFREEVM1 0x48
  4563. #define MM_DBG_COMMIT_RETURN_NTFREEVM2 0x49
  4564. #define MM_DBG_COMMIT_RETURN_INDEPENDENT_PAGES 0x4A
  4565. #define MM_DBG_COMMIT_RETURN_AWE_EXCESS 0x4B
  4566. #define MM_DBG_COMMIT_RETURN_MDL_PAGES 0x4C
  4567. #define MM_DBG_COMMIT_RETURN_NONCACHED_PAGES 0x4D
  4568. #define MM_DBG_COMMIT_RETURN_SESSION_CREATE_FAILURE 0x4E
  4569. #define MM_DBG_COMMIT_RETURN_PAGETABLES 0x4F
  4570. #define MM_DBG_COMMIT_RETURN_PROTECTION 0x50
  4571. #define MM_DBG_COMMIT_RETURN_SEGMENT_DELETE1 0x51
  4572. #define MM_DBG_COMMIT_RETURN_SEGMENT_DELETE2 0x52
  4573. #define MM_DBG_COMMIT_RETURN_PAGEFILE_FULL 0x53
  4574. #define MM_DBG_COMMIT_RETURN_SESSION_DEREFERENCE 0x54
  4575. #define MM_DBG_COMMIT_RETURN_VAD 0x55
  4576. #define MM_DBG_COMMIT_RETURN_PROCESS_CREATE_FAILURE1 0x56
  4577. #define MM_DBG_COMMIT_RETURN_PROCESS_DELETE 0x57
  4578. #define MM_DBG_COMMIT_RETURN_PROCESS_CLEAN_PAGETABLES 0x58
  4579. #define MM_DBG_COMMIT_RETURN_KERNEL_STACK_DELETE 0x59
  4580. #define MM_DBG_COMMIT_RETURN_SESSION_DRIVER_LOAD_FAILURE1 0x5A
  4581. #define MM_DBG_COMMIT_RETURN_DRIVER_INIT_CODE 0x5B
  4582. #define MM_DBG_COMMIT_RETURN_DRIVER_UNLOAD 0x5C
  4583. #define MM_DBG_COMMIT_RETURN_DRIVER_UNLOAD1 0x5D
  4584. #define MM_DBG_COMMIT_RETURN_NORMAL 0x5E
  4585. #define MM_DBG_COMMIT_RETURN_PF_FULL_EXTEND 0x5F
  4586. #define MM_DBG_COMMIT_RETURN_EXTENDED 0x60
  4587. #define MM_DBG_COMMIT_RETURN_SEGMENT_DELETE3 0x61
  4588. #if 0
  4589. #define MM_COMMIT_COUNTER_MAX 0x80
  4590. #define MM_TRACK_COMMIT(_index, bump) \
  4591. if (_index >= MM_COMMIT_COUNTER_MAX) { \
  4592. DbgPrint("Mm: Invalid commit counter %d %d\n", _index, MM_COMMIT_COUNTER_MAX); \
  4593. DbgBreakPoint(); \
  4594. } \
  4595. else { \
  4596. InterlockedExchangeAddSizeT (&MmTrackCommit[_index], bump); \
  4597. }
  4598. #define MM_TRACK_COMMIT_REDUCTION(_index, bump) \
  4599. if (_index >= MM_COMMIT_COUNTER_MAX) { \
  4600. DbgPrint("Mm: Invalid commit counter %d %d\n", _index, MM_COMMIT_COUNTER_MAX); \
  4601. DbgBreakPoint(); \
  4602. } \
  4603. else { \
  4604. InterlockedExchangeAddSizeT (&MmTrackCommit[_index], 0 - (bump)); \
  4605. }
  4606. extern SIZE_T MmTrackCommit[MM_COMMIT_COUNTER_MAX];
  4607. #define MI_INCREMENT_TOTAL_PROCESS_COMMIT(_charge) InterlockedExchangeAddSizeT (&MmTotalProcessCommit, (_charge));
  4608. #else
  4609. #define MM_TRACK_COMMIT(_index, bump)
  4610. #define MM_TRACK_COMMIT_REDUCTION(_index, bump)
  4611. #define MI_INCREMENT_TOTAL_PROCESS_COMMIT(_charge)
  4612. #endif
  4613. #define MM_BUMP_COUNTER_MAX 60
  4614. extern SIZE_T MmResTrack[MM_BUMP_COUNTER_MAX];
  4615. #define MM_BUMP_COUNTER(_index, bump) \
  4616. ASSERT (_index < MM_BUMP_COUNTER_MAX); \
  4617. InterlockedExchangeAddSizeT (&MmResTrack[_index], (SIZE_T)(bump));
  4618. extern ULONG MiSpecialPagesNonPaged;
  4619. extern ULONG MiSpecialPagesNonPagedMaximum;
  4620. //++
  4621. //PFN_NUMBER
  4622. //MI_NONPAGABLE_MEMORY_AVAILABLE(
  4623. // VOID
  4624. // );
  4625. //
  4626. // Routine Description:
  4627. //
  4628. // This routine lets callers know how many pages can be charged against
  4629. // the resident available, factoring in earlier Mm promises that
  4630. // may not have been redeemed at this point (ie: nonpaged pool expansion,
  4631. // etc, that must be honored at a later point if requested).
  4632. //
  4633. // Arguments
  4634. //
  4635. // None.
  4636. //
  4637. // Return Value:
  4638. //
  4639. // The number of currently available pages in the resident available.
  4640. //
  4641. // N.B. This is a signed quantity and can be negative.
  4642. //
  4643. //--
  4644. #define MI_NONPAGABLE_MEMORY_AVAILABLE() \
  4645. ((SPFN_NUMBER) \
  4646. (MmResidentAvailablePages - \
  4647. MmSystemLockPagesCount))
  4648. extern ULONG MmLargePageMinimum;
  4649. //
  4650. // hack stuff for testing.
  4651. //
  4652. VOID
  4653. MiDumpValidAddresses (
  4654. VOID
  4655. );
  4656. VOID
  4657. MiDumpPfn ( VOID );
  4658. VOID
  4659. MiDumpWsl ( VOID );
  4660. VOID
  4661. MiFormatPte (
  4662. IN PMMPTE PointerPte
  4663. );
  4664. VOID
  4665. MiCheckPfn ( VOID );
  4666. VOID
  4667. MiCheckPte ( VOID );
  4668. VOID
  4669. MiFormatPfn (
  4670. IN PMMPFN PointerPfn
  4671. );
  4672. extern const MMPTE ZeroPte;
  4673. extern const MMPTE ZeroKernelPte;
  4674. extern const MMPTE ValidKernelPteLocal;
  4675. extern MMPTE ValidKernelPte;
  4676. extern MMPTE ValidKernelPde;
  4677. extern const MMPTE ValidKernelPdeLocal;
  4678. extern const MMPTE ValidUserPte;
  4679. extern const MMPTE ValidPtePte;
  4680. extern const MMPTE ValidPdePde;
  4681. extern MMPTE DemandZeroPde;
  4682. extern const MMPTE DemandZeroPte;
  4683. extern MMPTE KernelPrototypePte;
  4684. extern const MMPTE TransitionPde;
  4685. extern MMPTE PrototypePte;
  4686. extern const MMPTE NoAccessPte;
  4687. extern ULONG_PTR MmSubsectionBase;
  4688. extern ULONG_PTR MmSubsectionTopPage;
  4689. extern ULONG ExpMultiUserTS;
  4690. //
  4691. // Virtual alignment for PTEs (machine specific) minimum value is
  4692. // 4k maximum value is 64k. The maximum value can be raised by
  4693. // changing the MM_PROTO_PTE_ALIGNMENT constant and adding more
  4694. // reserved mapping PTEs in hyperspace.
  4695. //
  4696. //
  4697. // Total number of physical pages on the system.
  4698. //
  4699. extern PFN_COUNT MmNumberOfPhysicalPages;
  4700. //
  4701. // Lowest physical page number on the system.
  4702. //
  4703. extern PFN_NUMBER MmLowestPhysicalPage;
  4704. //
  4705. // Highest physical page number on the system.
  4706. //
  4707. extern PFN_NUMBER MmHighestPhysicalPage;
  4708. //
  4709. // Highest possible physical page number in the system.
  4710. //
  4711. extern PFN_NUMBER MmHighestPossiblePhysicalPage;
  4712. #if defined (_WIN64)
  4713. #define MI_DTC_MAX_PAGES ((PFN_NUMBER)(((ULONG64)128 * 1024 * 1024 * 1024) >> PAGE_SHIFT))
  4714. #define MI_DTC_BOOTED_3GB_MAX_PAGES MI_DTC_MAX_PAGES
  4715. #define MI_ADS_MAX_PAGES ((PFN_NUMBER)(((ULONG64)64 * 1024 * 1024 * 1024) >> PAGE_SHIFT))
  4716. #define MI_DEFAULT_MAX_PAGES ((PFN_NUMBER)(((ULONG64)16 * 1024 * 1024 * 1024) >> PAGE_SHIFT))
  4717. #else
  4718. #define MI_DTC_MAX_PAGES ((PFN_NUMBER)(((ULONG64)64 * 1024 * 1024 * 1024) >> PAGE_SHIFT))
  4719. #define MI_DTC_BOOTED_3GB_MAX_PAGES ((PFN_NUMBER)(((ULONG64)16 * 1024 * 1024 * 1024) >> PAGE_SHIFT))
  4720. #define MI_ADS_MAX_PAGES ((PFN_NUMBER)(((ULONG64)32 * 1024 * 1024 * 1024) >> PAGE_SHIFT))
  4721. #define MI_DEFAULT_MAX_PAGES ((PFN_NUMBER)(((ULONG64)4 * 1024 * 1024 * 1024) >> PAGE_SHIFT))
  4722. #endif
  4723. #define MI_BLADE_MAX_PAGES ((PFN_NUMBER)(((ULONG64)2 * 1024 * 1024 * 1024) >> PAGE_SHIFT))
  4724. //
  4725. // Total number of available pages on the system. This
  4726. // is the sum of the pages on the zeroed, free and standby lists.
  4727. //
  4728. extern PFN_COUNT MmAvailablePages;
  4729. //
  4730. // Total number of free pages to base working set trimming on.
  4731. //
  4732. extern PFN_NUMBER MmMoreThanEnoughFreePages;
  4733. //
  4734. // Total number physical pages which would be usable if every process
  4735. // was at it's minimum working set size. This value is initialized
  4736. // at system initialization to MmAvailablePages - MM_FLUID_PHYSICAL_PAGES.
  4737. // Everytime a thread is created, the kernel stack is subtracted from
  4738. // this and every time a process is created, the minimum working set
  4739. // is subtracted from this. If the value would become negative, the
  4740. // operation (create process/kernel stack/ adjust working set) fails.
  4741. // The PFN LOCK must be owned to manipulate this value.
  4742. //
  4743. extern SPFN_NUMBER MmResidentAvailablePages;
  4744. //
  4745. // The total number of pages which would be removed from working sets
  4746. // if every working set was at its minimum.
  4747. //
  4748. extern PFN_NUMBER MmPagesAboveWsMinimum;
  4749. //
  4750. // The total number of pages which would be removed from working sets
  4751. // if every working set above its maximum was at its maximum.
  4752. //
  4753. extern PFN_NUMBER MmPagesAboveWsMaximum;
  4754. //
  4755. // If memory is becoming short and MmPagesAboveWsMinimum is
  4756. // greater than MmPagesAboveWsThreshold, trim working sets.
  4757. //
  4758. extern PFN_NUMBER MmPagesAboveWsThreshold;
  4759. //
  4760. // The number of pages to add to a working set if there are ample
  4761. // available pages and the working set is below its maximum.
  4762. //
  4763. extern PFN_NUMBER MmWorkingSetSizeIncrement;
  4764. //
  4765. // The number of pages to extend the maximum working set size by
  4766. // if the working set at its maximum and there are ample available pages.
  4767. extern PFN_NUMBER MmWorkingSetSizeExpansion;
  4768. extern ULONG MmPlentyFreePages;
  4769. //
  4770. // The number of pages required to be freed by working set reduction
  4771. // before working set reduction is attempted.
  4772. //
  4773. extern PFN_NUMBER MmWsAdjustThreshold;
  4774. //
  4775. // The number of pages available to allow the working set to be
  4776. // expanded above its maximum.
  4777. //
  4778. extern PFN_NUMBER MmWsExpandThreshold;
  4779. //
  4780. // The total number of pages to reduce by working set trimming.
  4781. //
  4782. extern PFN_NUMBER MmWsTrimReductionGoal;
  4783. extern LONG MiDelayPageFaults;
  4784. extern PMMPFN MmPfnDatabase;
  4785. extern MMPFNLIST MmZeroedPageListHead;
  4786. extern MMPFNLIST MmFreePageListHead;
  4787. extern MMPFNLIST MmStandbyPageListHead;
  4788. extern MMPFNLIST MmRomPageListHead;
  4789. extern MMPFNLIST MmModifiedPageListHead;
  4790. extern MMPFNLIST MmModifiedNoWritePageListHead;
  4791. extern MMPFNLIST MmBadPageListHead;
  4792. extern PMMPFNLIST MmPageLocationList[NUMBER_OF_PAGE_LISTS];
  4793. extern MMPFNLIST MmModifiedPageListByColor[MM_MAXIMUM_NUMBER_OF_COLORS];
  4794. //
  4795. // Mask for isolating secondary color from physical page number.
  4796. //
  4797. extern ULONG MmSecondaryColorMask;
  4798. //
  4799. // Mask for isolating node color from combined node and secondary
  4800. // color.
  4801. //
  4802. extern ULONG MmSecondaryColorNodeMask;
  4803. //
  4804. // Width of MmSecondaryColorMask in bits. In multi node systems,
  4805. // the node number is combined with the secondary color to make up
  4806. // the page color.
  4807. //
  4808. extern UCHAR MmSecondaryColorNodeShift;
  4809. //
  4810. // Event for available pages, set means pages are available.
  4811. //
  4812. extern KEVENT MmAvailablePagesEvent;
  4813. extern KEVENT MmAvailablePagesEventMedium;
  4814. extern KEVENT MmAvailablePagesEventHigh;
  4815. //
  4816. // Event for the zeroing page thread.
  4817. //
  4818. extern KEVENT MmZeroingPageEvent;
  4819. //
  4820. // Boolean to indicate if the zeroing page thread is currently
  4821. // active. This is set to true when the zeroing page event is
  4822. // set and set to false when the zeroing page thread is done
  4823. // zeroing all the pages on the free list.
  4824. //
  4825. extern BOOLEAN MmZeroingPageThreadActive;
  4826. //
  4827. // Minimum number of free pages before zeroing page thread starts.
  4828. //
  4829. extern PFN_NUMBER MmMinimumFreePagesToZero;
  4830. //
  4831. // Global event to synchronize mapped writing with cleaning segments.
  4832. //
  4833. extern KEVENT MmMappedFileIoComplete;
  4834. //
  4835. // Hyper space items.
  4836. //
  4837. extern PMMPTE MmFirstReservedMappingPte;
  4838. extern PMMPTE MmLastReservedMappingPte;
  4839. //
  4840. // System space sizes - MmNonPagedSystemStart to MM_NON_PAGED_SYSTEM_END
  4841. // defines the ranges of PDEs which must be copied into a new process's
  4842. // address space.
  4843. //
  4844. extern PVOID MmNonPagedSystemStart;
  4845. extern PCHAR MmSystemSpaceViewStart;
  4846. extern LOGICAL MmProtectFreedNonPagedPool;
  4847. //
  4848. // Pool sizes.
  4849. //
  4850. extern SIZE_T MmSizeOfNonPagedPoolInBytes;
  4851. extern SIZE_T MmMinimumNonPagedPoolSize;
  4852. extern SIZE_T MmDefaultMaximumNonPagedPool;
  4853. extern ULONG MmMaximumNonPagedPoolPercent;
  4854. extern ULONG MmMinAdditionNonPagedPoolPerMb;
  4855. extern ULONG MmMaxAdditionNonPagedPoolPerMb;
  4856. extern SIZE_T MmSizeOfPagedPoolInBytes;
  4857. extern SIZE_T MmMaximumNonPagedPoolInBytes;
  4858. extern PFN_NUMBER MmAllocatedNonPagedPool;
  4859. extern PVOID MmNonPagedPoolExpansionStart;
  4860. extern ULONG MmExpandedPoolBitPosition;
  4861. extern PFN_NUMBER MmNumberOfFreeNonPagedPool;
  4862. extern ULONG MmNumberOfSystemPtes;
  4863. extern ULONG MiRequestedSystemPtes;
  4864. extern ULONG MmTotalFreeSystemPtes[MaximumPtePoolTypes];
  4865. extern PMMPTE MmSystemPagePtes;
  4866. extern ULONG MmSystemPageDirectory[];
  4867. extern SIZE_T MmHeapSegmentReserve;
  4868. extern SIZE_T MmHeapSegmentCommit;
  4869. extern SIZE_T MmHeapDeCommitTotalFreeThreshold;
  4870. extern SIZE_T MmHeapDeCommitFreeBlockThreshold;
  4871. #define MI_MAX_FREE_LIST_HEADS 4
  4872. extern LIST_ENTRY MmNonPagedPoolFreeListHead[MI_MAX_FREE_LIST_HEADS];
  4873. //
  4874. // Counter for flushes of the entire TB.
  4875. //
  4876. extern ULONG MmFlushCounter;
  4877. //
  4878. // Pool start and end.
  4879. //
  4880. extern PVOID MmNonPagedPoolStart;
  4881. extern PVOID MmNonPagedPoolEnd;
  4882. extern PVOID MmPagedPoolStart;
  4883. extern PVOID MmPagedPoolEnd;
  4884. //
  4885. // Pool bit maps and other related structures.
  4886. //
  4887. typedef struct _MM_PAGED_POOL_INFO {
  4888. PRTL_BITMAP PagedPoolAllocationMap;
  4889. PRTL_BITMAP EndOfPagedPoolBitmap;
  4890. PRTL_BITMAP PagedPoolLargeSessionAllocationMap; // HYDRA only
  4891. PMMPTE FirstPteForPagedPool;
  4892. PMMPTE LastPteForPagedPool;
  4893. PMMPTE NextPdeForPagedPoolExpansion;
  4894. ULONG PagedPoolHint;
  4895. SIZE_T PagedPoolCommit;
  4896. SIZE_T AllocatedPagedPool;
  4897. } MM_PAGED_POOL_INFO, *PMM_PAGED_POOL_INFO;
  4898. extern MM_PAGED_POOL_INFO MmPagedPoolInfo;
  4899. extern PVOID MmPageAlignedPoolBase[2];
  4900. extern PRTL_BITMAP VerifierLargePagedPoolMap;
  4901. //
  4902. // MmFirstFreeSystemPte contains the offset from the
  4903. // Nonpaged system base to the first free system PTE.
  4904. // Note, that an offset of zero indicates an empty list.
  4905. //
  4906. extern MMPTE MmFirstFreeSystemPte[MaximumPtePoolTypes];
  4907. extern ULONG_PTR MiSystemViewStart;
  4908. //
  4909. // System cache sizes.
  4910. //
  4911. //extern MMSUPPORT MmSystemCacheWs;
  4912. extern PMMWSL MmSystemCacheWorkingSetList;
  4913. extern PMMWSLE MmSystemCacheWsle;
  4914. extern PVOID MmSystemCacheStart;
  4915. extern PVOID MmSystemCacheEnd;
  4916. extern PFN_NUMBER MmSystemCacheWsMinimum;
  4917. extern PFN_NUMBER MmSystemCacheWsMaximum;
  4918. //
  4919. // Virtual alignment for PTEs (machine specific) minimum value is
  4920. // 0 (no alignment) maximum value is 64k. The maximum value can be raised by
  4921. // changing the MM_PROTO_PTE_ALIGNMENT constant and adding more
  4922. // reserved mapping PTEs in hyperspace.
  4923. //
  4924. extern ULONG MmAliasAlignment;
  4925. //
  4926. // Mask to AND with virtual address to get an offset to go
  4927. // with the alignment. This value is page aligned.
  4928. //
  4929. extern ULONG MmAliasAlignmentOffset;
  4930. //
  4931. // Mask to and with PTEs to determine if the alias mapping is compatible.
  4932. // This value is usually (MmAliasAlignment - 1)
  4933. //
  4934. extern ULONG MmAliasAlignmentMask;
  4935. //
  4936. // Cells to track unused thread kernel stacks to avoid TB flushes
  4937. // every time a thread terminates.
  4938. //
  4939. extern ULONG MmMaximumDeadKernelStacks;
  4940. extern SLIST_HEADER MmDeadStackSListHead;
  4941. //
  4942. // MmSystemPteBase contains the address of 1 PTE before
  4943. // the first free system PTE (zero indicates an empty list).
  4944. // The value of this field does not change once set.
  4945. //
  4946. extern PMMPTE MmSystemPteBase;
  4947. //
  4948. // Root of system space virtual address descriptors. These define
  4949. // the pagable portion of the system.
  4950. //
  4951. extern PMMVAD MmVirtualAddressDescriptorRoot;
  4952. extern PMMADDRESS_NODE MmSectionBasedRoot;
  4953. extern PVOID MmHighSectionBase;
  4954. //
  4955. // Section commit mutex.
  4956. //
  4957. extern FAST_MUTEX MmSectionCommitMutex;
  4958. //
  4959. // Section base address mutex.
  4960. //
  4961. extern FAST_MUTEX MmSectionBasedMutex;
  4962. //
  4963. // Resource for section extension.
  4964. //
  4965. extern ERESOURCE MmSectionExtendResource;
  4966. extern ERESOURCE MmSectionExtendSetResource;
  4967. //
  4968. // Inpage cluster sizes for executable pages (set based on memory size).
  4969. //
  4970. extern ULONG MmDataClusterSize;
  4971. extern ULONG MmCodeClusterSize;
  4972. //
  4973. // Pagefile creation mutex.
  4974. //
  4975. extern FAST_MUTEX MmPageFileCreationLock;
  4976. //
  4977. // Event to set when first paging file is created.
  4978. //
  4979. extern PKEVENT MmPagingFileCreated;
  4980. //
  4981. // Paging file debug information.
  4982. //
  4983. extern ULONG_PTR MmPagingFileDebug[];
  4984. //
  4985. // Spinlock which guards PFN database. This spinlock is used by
  4986. // memory management for accessing the PFN database. The I/O
  4987. // system makes use of it for unlocking pages during I/O complete.
  4988. //
  4989. // extern KSPIN_LOCK MmPfnLock;
  4990. //
  4991. // Spinlock which guards the working set list for the system shared
  4992. // address space (paged pool, system cache, pagable drivers).
  4993. //
  4994. extern ERESOURCE MmSystemWsLock;
  4995. //
  4996. // Spin lock for allowing working set expansion.
  4997. //
  4998. extern KSPIN_LOCK MmExpansionLock;
  4999. //
  5000. // To prevent optimizations.
  5001. //
  5002. extern MMPTE GlobalPte;
  5003. //
  5004. // Page color for system working set.
  5005. //
  5006. extern ULONG MmSystemPageColor;
  5007. extern ULONG MmSecondaryColors;
  5008. extern ULONG MmProcessColorSeed;
  5009. //
  5010. // Set from ntos\config\CMDAT3.C Used by customers to disable paging
  5011. // of executive on machines with lots of memory. Worth a few TPS on a
  5012. // data base server.
  5013. //
  5014. #define MM_SYSTEM_CODE_LOCKED_DOWN 0x1
  5015. #define MM_PAGED_POOL_LOCKED_DOWN 0x2
  5016. extern ULONG MmDisablePagingExecutive;
  5017. //
  5018. // For debugging.
  5019. #if DBG
  5020. extern ULONG MmDebug;
  5021. #endif
  5022. //
  5023. // Unused segment management
  5024. //
  5025. extern MMDEREFERENCE_SEGMENT_HEADER MmDereferenceSegmentHeader;
  5026. extern LIST_ENTRY MmUnusedSegmentList;
  5027. extern LIST_ENTRY MmUnusedSubsectionList;
  5028. extern KEVENT MmUnusedSegmentCleanup;
  5029. extern ULONG MmConsumedPoolPercentage;
  5030. extern ULONG MmUnusedSegmentCount;
  5031. extern ULONG MmUnusedSubsectionCount;
  5032. extern ULONG MmUnusedSubsectionCountPeak;
  5033. extern SIZE_T MiUnusedSubsectionPagedPool;
  5034. extern SIZE_T MiUnusedSubsectionPagedPoolPeak;
  5035. #define MI_UNUSED_SUBSECTIONS_COUNT_INSERT(_MappedSubsection) \
  5036. MmUnusedSubsectionCount += 1; \
  5037. if (MmUnusedSubsectionCount > MmUnusedSubsectionCountPeak) { \
  5038. MmUnusedSubsectionCountPeak = MmUnusedSubsectionCount; \
  5039. } \
  5040. MiUnusedSubsectionPagedPool += EX_REAL_POOL_USAGE((_MappedSubsection->PtesInSubsection + _MappedSubsection->UnusedPtes) * sizeof (MMPTE)); \
  5041. if (MiUnusedSubsectionPagedPool > MiUnusedSubsectionPagedPoolPeak) { \
  5042. MiUnusedSubsectionPagedPoolPeak = MiUnusedSubsectionPagedPool; \
  5043. } \
  5044. #define MI_UNUSED_SUBSECTIONS_COUNT_REMOVE(_MappedSubsection) \
  5045. MmUnusedSubsectionCount -= 1; \
  5046. MiUnusedSubsectionPagedPool -= EX_REAL_POOL_USAGE((_MappedSubsection->PtesInSubsection + _MappedSubsection->UnusedPtes) * sizeof (MMPTE));
  5047. #define MI_FILESYSTEM_NONPAGED_POOL_CHARGE 150
  5048. #define MI_FILESYSTEM_PAGED_POOL_CHARGE 1024
  5049. //++
  5050. //LOGICAL
  5051. //MI_UNUSED_SEGMENTS_SURPLUS (
  5052. // IN PVOID va
  5053. // );
  5054. //
  5055. // Routine Description:
  5056. //
  5057. // This routine determines whether a surplus of unused
  5058. // segments exist. If so, the caller can initiate a trim to free pool.
  5059. //
  5060. // Arguments
  5061. //
  5062. // None.
  5063. //
  5064. // Return Value:
  5065. //
  5066. // TRUE if unused segment trimming should be initiated, FALSE if not.
  5067. //
  5068. //--
  5069. #define MI_UNUSED_SEGMENTS_SURPLUS() \
  5070. (((ULONG)((MmPagedPoolInfo.AllocatedPagedPool * 100) / (MmSizeOfPagedPoolInBytes >> PAGE_SHIFT)) > MmConsumedPoolPercentage) || \
  5071. ((ULONG)((MmAllocatedNonPagedPool * 100) / (MmMaximumNonPagedPoolInBytes >> PAGE_SHIFT)) > MmConsumedPoolPercentage))
  5072. VOID
  5073. MiConvertStaticSubsections (
  5074. IN PCONTROL_AREA ControlArea
  5075. );
  5076. //++
  5077. //VOID
  5078. //MI_INSERT_UNUSED_SEGMENT (
  5079. // IN PCONTROL_AREA _ControlArea
  5080. // );
  5081. //
  5082. // Routine Description:
  5083. //
  5084. // This routine inserts a control area into the unused segment list,
  5085. // also managing the associated pool charges.
  5086. //
  5087. // Arguments
  5088. //
  5089. // _ControlArea - Supplies the control area to obtain the pool charges from.
  5090. //
  5091. // Return Value:
  5092. //
  5093. // None.
  5094. //
  5095. //--
  5096. #define MI_INSERT_UNUSED_SEGMENT(_ControlArea) \
  5097. { \
  5098. MM_PFN_LOCK_ASSERT(); \
  5099. if ((_ControlArea->u.Flags.Image == 0) && \
  5100. (_ControlArea->FilePointer != NULL) && \
  5101. (_ControlArea->u.Flags.PhysicalMemory == 0)) { \
  5102. MiConvertStaticSubsections(_ControlArea); \
  5103. } \
  5104. InsertTailList (&MmUnusedSegmentList, &_ControlArea->DereferenceList); \
  5105. MmUnusedSegmentCount += 1; \
  5106. }
  5107. //++
  5108. //VOID
  5109. //MI_UNUSED_SEGMENTS_REMOVE_CHARGE (
  5110. // IN PCONTROL_AREA _ControlArea
  5111. // );
  5112. //
  5113. // Routine Description:
  5114. //
  5115. // This routine manages pool charges during removals of segments from
  5116. // the unused segment list.
  5117. //
  5118. // Arguments
  5119. //
  5120. // _ControlArea - Supplies the control area to obtain the pool charges from.
  5121. //
  5122. // Return Value:
  5123. //
  5124. // None.
  5125. //
  5126. //--
  5127. #define MI_UNUSED_SEGMENTS_REMOVE_CHARGE(_ControlArea) \
  5128. { \
  5129. MM_PFN_LOCK_ASSERT(); \
  5130. MmUnusedSegmentCount -= 1; \
  5131. }
  5132. //
  5133. // List heads
  5134. //
  5135. extern MMWORKING_SET_EXPANSION_HEAD MmWorkingSetExpansionHead;
  5136. extern MMPAGE_FILE_EXPANSION MmAttemptForCantExtend;
  5137. //
  5138. // Paging files
  5139. //
  5140. extern MMMOD_WRITER_LISTHEAD MmPagingFileHeader;
  5141. extern MMMOD_WRITER_LISTHEAD MmMappedFileHeader;
  5142. extern PMMPAGING_FILE MmPagingFile[MAX_PAGE_FILES];
  5143. extern LIST_ENTRY MmFreePagingSpaceLow;
  5144. extern ULONG MmNumberOfActiveMdlEntries;
  5145. extern ULONG MmNumberOfPagingFiles;
  5146. extern KEVENT MmModifiedPageWriterEvent;
  5147. extern KEVENT MmCollidedFlushEvent;
  5148. extern KEVENT MmCollidedLockEvent;
  5149. //
  5150. // Total number of committed pages.
  5151. //
  5152. extern SIZE_T MmTotalCommittedPages;
  5153. extern SIZE_T MmTotalCommitLimit;
  5154. extern SIZE_T MmOverCommit;
  5155. extern SIZE_T MmSharedCommit;
  5156. // #define _MI_DEBUG_DATA 1 // Uncomment this for data logging
  5157. #if defined (_MI_DEBUG_DATA)
  5158. #define MI_DATA_BACKTRACE_LENGTH 8
  5159. typedef struct _MI_DATA_TRACES {
  5160. PETHREAD Thread;
  5161. PMMPFN Pfn;
  5162. PMMPTE PointerPte;
  5163. MMPFN PfnData;
  5164. ULONG CallerId;
  5165. ULONG DataInThePage[2];
  5166. PVOID StackTrace [MI_DATA_BACKTRACE_LENGTH];
  5167. } MI_DATA_TRACES, *PMI_DATA_TRACES;
  5168. extern LONG MiDataIndex;
  5169. extern ULONG MiTrackData;
  5170. extern PMI_DATA_TRACES MiDataTraces;
  5171. VOID
  5172. FORCEINLINE
  5173. MiSnapData (
  5174. IN PMMPFN Pfn,
  5175. IN PMMPTE PointerPte,
  5176. IN ULONG CallerId
  5177. )
  5178. {
  5179. KIRQL OldIrql;
  5180. PVOID Va;
  5181. PMI_DATA_TRACES Information;
  5182. ULONG Index;
  5183. ULONG Hash;
  5184. PEPROCESS CurrentProcess;
  5185. if (MiDataTraces == NULL) {
  5186. return;
  5187. }
  5188. Index = InterlockedIncrement(&MiDataIndex);
  5189. Index &= (MiTrackData - 1);
  5190. Information = &MiDataTraces[Index];
  5191. Information->Thread = PsGetCurrentThread ();
  5192. Information->Pfn = Pfn;
  5193. Information->PointerPte = PointerPte;
  5194. Information->PfnData = *Pfn;
  5195. Information->CallerId = CallerId;
  5196. CurrentProcess = PsGetCurrentProcess ();
  5197. Va = MiMapPageInHyperSpace (CurrentProcess, Pfn - MmPfnDatabase, &OldIrql);
  5198. RtlCopyMemory (&Information->DataInThePage[0],
  5199. Va,
  5200. sizeof (Information->DataInThePage));
  5201. MiUnmapPageInHyperSpace (CurrentProcess, Va, OldIrql);
  5202. RtlZeroMemory (&Information->StackTrace[0], MI_DATA_BACKTRACE_LENGTH * sizeof(PVOID)); \
  5203. RtlCaptureStackBackTrace (0, MI_DATA_BACKTRACE_LENGTH, Information->StackTrace, &Hash);
  5204. }
  5205. #define MI_SNAP_DATA(_Pfn, _Pte, _CallerId) MiSnapData(_Pfn, _Pte, _CallerId)
  5206. #else
  5207. #define MI_SNAP_DATA(_Pfn, _Pte, _CallerId)
  5208. #endif
  5209. //
  5210. // Modified page writer.
  5211. //
  5212. extern PFN_NUMBER MmMinimumFreePages;
  5213. extern PFN_NUMBER MmFreeGoal;
  5214. extern PFN_NUMBER MmModifiedPageMaximum;
  5215. extern PFN_NUMBER MmModifiedPageMinimum;
  5216. extern ULONG MmModifiedWriteClusterSize;
  5217. extern ULONG MmMinimumFreeDiskSpace;
  5218. extern ULONG MmPageFileExtension;
  5219. extern ULONG MmMinimumPageFileReduction;
  5220. extern LARGE_INTEGER MiModifiedPageLife;
  5221. extern BOOLEAN MiTimerPending;
  5222. extern KEVENT MiMappedPagesTooOldEvent;
  5223. extern KDPC MiModifiedPageWriterTimerDpc;
  5224. extern KTIMER MiModifiedPageWriterTimer;
  5225. //
  5226. // System process working set sizes.
  5227. //
  5228. extern PFN_NUMBER MmSystemProcessWorkingSetMin;
  5229. extern PFN_NUMBER MmSystemProcessWorkingSetMax;
  5230. extern PFN_NUMBER MmMinimumWorkingSetSize;
  5231. //
  5232. // Support for debugger's mapping physical memory.
  5233. //
  5234. extern PMMPTE MmDebugPte;
  5235. extern PMMPTE MmCrashDumpPte;
  5236. extern ULONG MiOverCommitCallCount;
  5237. //
  5238. // Event tracing routines
  5239. //
  5240. extern PPAGE_FAULT_NOTIFY_ROUTINE MmPageFaultNotifyRoutine;
  5241. extern SIZE_T MmSystemViewSize;
  5242. VOID
  5243. FASTCALL
  5244. MiIdentifyPfn (
  5245. IN PMMPFN Pfn1,
  5246. OUT PMMPFN_IDENTITY PfnIdentity
  5247. );
  5248. #if defined (_WIN64)
  5249. #define InterlockedExchangeAddSizeT(a, b) InterlockedExchangeAdd64((PLONGLONG)a, b)
  5250. #else
  5251. #define InterlockedExchangeAddSizeT(a, b) InterlockedExchangeAdd((PLONG)(a), b)
  5252. #endif
  5253. //
  5254. // This is a special value loaded into an EPROCESS pointer to indicate that
  5255. // the action underway is for a Hydra session, not really the current process.
  5256. // (Any value could be used here that is not a valid system pointer or NULL).
  5257. //
  5258. #define HYDRA_PROCESS ((PEPROCESS)1)
  5259. #define PREFETCH_PROCESS ((PEPROCESS)2)
  5260. #define MI_SESSION_SPACE_STRUCT_SIZE MM_ALLOCATION_GRANULARITY
  5261. #if defined (_WIN64)
  5262. /*++
  5263. Virtual memory layout of session space when loaded down from
  5264. 0x2000.0002.0000.0000 (IA64) or FFFF.F980.0000.0000 (AMD64) :
  5265. Note that the sizes of mapped views, paged pool & images are registry tunable.
  5266. +------------------------------------+
  5267. 2000.0002.0000.0000 | |
  5268. | win32k.sys & video drivers |
  5269. | (16MB) |
  5270. | |
  5271. +------------------------------------+
  5272. 2000.0001.FF00.0000 | |
  5273. | MM_SESSION_SPACE & Session WSLs |
  5274. | (16MB) |
  5275. | |
  5276. 2000.0001.FEFF.0000 +------------------------------------+
  5277. | |
  5278. | ... |
  5279. | |
  5280. +------------------------------------+
  5281. 2000.0001.FE80.0000 | |
  5282. | Mapped Views for this session |
  5283. | (104MB) |
  5284. | |
  5285. +------------------------------------+
  5286. 2000.0001.F800.0000 | |
  5287. | Paged Pool for this session |
  5288. | (64MB) |
  5289. | |
  5290. 2000.0001.F400.0000 +------------------------------------+
  5291. | Special Pool for this session |
  5292. | (64MB) |
  5293. | |
  5294. 2000.0000.0000.0000 +------------------------------------+
  5295. --*/
  5296. #define MI_SESSION_SPACE_WS_SIZE ((ULONG_PTR)(16*1024*1024) - MI_SESSION_SPACE_STRUCT_SIZE)
  5297. #define MI_SESSION_DEFAULT_IMAGE_SIZE ((ULONG_PTR)(16*1024*1024))
  5298. #define MI_SESSION_DEFAULT_VIEW_SIZE ((ULONG_PTR)(104*1024*1024))
  5299. #define MI_SESSION_DEFAULT_POOL_SIZE ((ULONG_PTR)(64*1024*1024))
  5300. #define MI_SESSION_SPACE_MAXIMUM_TOTAL_SIZE (MM_VA_MAPPED_BY_PPE)
  5301. #else
  5302. /*++
  5303. Virtual memory layout of session space when loaded down from 0xC0000000.
  5304. Note that the sizes of mapped views, paged pool and images are registry
  5305. tunable on 32-bit systems (if NOT booted /3GB, as 3GB has very limited
  5306. address space).
  5307. +------------------------------------+
  5308. C0000000 | |
  5309. | win32k.sys, video drivers and any |
  5310. | rebased NT4 printer drivers. |
  5311. | |
  5312. | (8MB) |
  5313. | |
  5314. +------------------------------------+
  5315. BF800000 | |
  5316. | MM_SESSION_SPACE & Session WSLs |
  5317. | (4MB) |
  5318. | |
  5319. +------------------------------------+
  5320. BF400000 | |
  5321. | Mapped views for this session |
  5322. | (20MB by default, but is |
  5323. | registry configurable) |
  5324. | |
  5325. +------------------------------------+
  5326. BE000000 | |
  5327. | Paged pool for this session |
  5328. | (16MB by default, but is |
  5329. | registry configurable) |
  5330. | |
  5331. BD000000 +------------------------------------+
  5332. --*/
  5333. #define MI_SESSION_SPACE_WS_SIZE (4*1024*1024 - MI_SESSION_SPACE_STRUCT_SIZE)
  5334. #define MI_SESSION_DEFAULT_IMAGE_SIZE (8*1024*1024)
  5335. #define MI_SESSION_DEFAULT_VIEW_SIZE (20*1024*1024)
  5336. #define MI_SESSION_DEFAULT_POOL_SIZE (16*1024*1024)
  5337. #define MI_SESSION_SPACE_MAXIMUM_TOTAL_SIZE \
  5338. (MM_SYSTEM_CACHE_END_EXTRA - MM_KSEG2_BASE)
  5339. #endif
  5340. #define MI_SESSION_SPACE_DEFAULT_TOTAL_SIZE \
  5341. (MI_SESSION_DEFAULT_IMAGE_SIZE + \
  5342. MI_SESSION_SPACE_STRUCT_SIZE + \
  5343. MI_SESSION_SPACE_WS_SIZE + \
  5344. MI_SESSION_DEFAULT_VIEW_SIZE + \
  5345. MI_SESSION_DEFAULT_POOL_SIZE)
  5346. extern ULONG_PTR MmSessionBase;
  5347. extern PMMPTE MiSessionBasePte;
  5348. extern PMMPTE MiSessionLastPte;
  5349. extern ULONG_PTR MiSessionSpaceWs;
  5350. extern ULONG_PTR MiSessionViewStart;
  5351. extern SIZE_T MmSessionViewSize;
  5352. extern ULONG_PTR MiSessionImageStart;
  5353. extern ULONG_PTR MiSessionImageEnd;
  5354. extern SIZE_T MmSessionImageSize;
  5355. extern PMMPTE MiSessionImagePteStart;
  5356. extern PMMPTE MiSessionImagePteEnd;
  5357. extern ULONG_PTR MiSessionPoolStart;
  5358. extern ULONG_PTR MiSessionPoolEnd;
  5359. extern SIZE_T MmSessionPoolSize;
  5360. extern ULONG_PTR MiSessionSpaceEnd;
  5361. extern ULONG MiSessionSpacePageTables;
  5362. //
  5363. // The number of page table pages required to map all of session space.
  5364. //
  5365. #define MI_SESSION_SPACE_MAXIMUM_PAGE_TABLES \
  5366. (MI_SESSION_SPACE_MAXIMUM_TOTAL_SIZE / MM_VA_MAPPED_BY_PDE)
  5367. extern SIZE_T MmSessionSize; // size of the entire session space.
  5368. //
  5369. // Macros to determine if a given address lies in the specified session range.
  5370. //
  5371. #define MI_IS_SESSION_IMAGE_ADDRESS(VirtualAddress) \
  5372. ((PVOID)(VirtualAddress) >= (PVOID)MiSessionImageStart && (PVOID)(VirtualAddress) < (PVOID)(MiSessionImageEnd))
  5373. #define MI_IS_SESSION_POOL_ADDRESS(VirtualAddress) \
  5374. ((PVOID)(VirtualAddress) >= (PVOID)MiSessionPoolStart && (PVOID)(VirtualAddress) < (PVOID)MiSessionPoolEnd)
  5375. #define MI_IS_SESSION_ADDRESS(_VirtualAddress) \
  5376. ((PVOID)(_VirtualAddress) >= (PVOID)MmSessionBase && (PVOID)(_VirtualAddress) < (PVOID)(MiSessionSpaceEnd))
  5377. #define MI_IS_SESSION_PTE(_Pte) \
  5378. ((PMMPTE)(_Pte) >= MiSessionBasePte && (PMMPTE)(_Pte) < MiSessionLastPte)
  5379. #define MI_IS_SESSION_IMAGE_PTE(_Pte) \
  5380. ((PMMPTE)(_Pte) >= MiSessionImagePteStart && (PMMPTE)(_Pte) < MiSessionImagePteEnd)
  5381. #define SESSION_GLOBAL(_Session) (_Session->GlobalVirtualAddress)
  5382. #define MM_DBG_SESSION_INITIAL_PAGETABLE_ALLOC 0
  5383. #define MM_DBG_SESSION_INITIAL_PAGETABLE_FREE_RACE 1
  5384. #define MM_DBG_SESSION_INITIAL_PAGE_ALLOC 2
  5385. #define MM_DBG_SESSION_INITIAL_PAGE_FREE_FAIL1 3
  5386. #define MM_DBG_SESSION_INITIAL_PAGETABLE_FREE_FAIL1 4
  5387. #define MM_DBG_SESSION_WS_PAGE_FREE 5
  5388. #define MM_DBG_SESSION_PAGETABLE_ALLOC 6
  5389. #define MM_DBG_SESSION_SYSMAPPED_PAGES_ALLOC 7
  5390. #define MM_DBG_SESSION_WS_PAGETABLE_ALLOC 8
  5391. #define MM_DBG_SESSION_PAGEDPOOL_PAGETABLE_ALLOC 9
  5392. #define MM_DBG_SESSION_PAGEDPOOL_PAGETABLE_FREE_FAIL1 10
  5393. #define MM_DBG_SESSION_WS_PAGE_ALLOC 11
  5394. #define MM_DBG_SESSION_WS_PAGE_ALLOC_GROWTH 12
  5395. #define MM_DBG_SESSION_INITIAL_PAGE_FREE 13
  5396. #define MM_DBG_SESSION_PAGETABLE_FREE 14
  5397. #define MM_DBG_SESSION_PAGEDPOOL_PAGETABLE_ALLOC1 15
  5398. #define MM_DBG_SESSION_DRIVER_PAGES_LOCKED 16
  5399. #define MM_DBG_SESSION_DRIVER_PAGES_UNLOCKED 17
  5400. #define MM_DBG_SESSION_WS_HASHPAGE_ALLOC 18
  5401. #define MM_DBG_SESSION_SYSMAPPED_PAGES_COMMITTED 19
  5402. #define MM_DBG_SESSION_COMMIT_PAGEDPOOL_PAGES 30
  5403. #define MM_DBG_SESSION_COMMIT_DELETE_VM_RETURN 31
  5404. #define MM_DBG_SESSION_COMMIT_POOL_FREED 32
  5405. #define MM_DBG_SESSION_COMMIT_IMAGE_UNLOAD 33
  5406. #define MM_DBG_SESSION_COMMIT_IMAGELOAD_FAILED1 34
  5407. #define MM_DBG_SESSION_COMMIT_IMAGELOAD_FAILED2 35
  5408. #define MM_DBG_SESSION_COMMIT_IMAGELOAD_NOACCESS 36
  5409. #define MM_DBG_SESSION_NP_LOCK_CODE1 38
  5410. #define MM_DBG_SESSION_NP_LOCK_CODE2 39
  5411. #define MM_DBG_SESSION_NP_SESSION_CREATE 40
  5412. #define MM_DBG_SESSION_NP_PAGETABLE_ALLOC 41
  5413. #define MM_DBG_SESSION_NP_POOL_CREATE 42
  5414. #define MM_DBG_SESSION_NP_COMMIT_IMAGE 43
  5415. #define MM_DBG_SESSION_NP_COMMIT_IMAGE_PT 44
  5416. #define MM_DBG_SESSION_NP_INIT_WS 45
  5417. #define MM_DBG_SESSION_NP_WS_GROW 46
  5418. #define MM_DBG_SESSION_NP_HASH_GROW 47
  5419. #define MM_DBG_SESSION_NP_PAGE_DRIVER 48
  5420. #define MM_DBG_SESSION_NP_POOL_CREATE_FAILED 49
  5421. #define MM_DBG_SESSION_NP_WS_PAGE_FREE 50
  5422. #define MM_DBG_SESSION_NP_SESSION_DESTROY 51
  5423. #define MM_DBG_SESSION_NP_SESSION_PTDESTROY 52
  5424. #define MM_DBG_SESSION_NP_DELVA 53
  5425. #if DBG
  5426. #define MM_SESS_COUNTER_MAX 54
  5427. #define MM_BUMP_SESS_COUNTER(_index, bump) \
  5428. if (_index >= MM_SESS_COUNTER_MAX) { \
  5429. DbgPrint("Mm: Invalid bump counter %d %d\n", _index, MM_SESS_COUNTER_MAX); \
  5430. DbgBreakPoint(); \
  5431. } \
  5432. MmSessionSpace->Debug[_index] += (bump);
  5433. typedef struct _MM_SESSION_MEMORY_COUNTERS {
  5434. SIZE_T NonPagablePages;
  5435. SIZE_T CommittedPages;
  5436. } MM_SESSION_MEMORY_COUNTERS, *PMM_SESSION_MEMORY_COUNTERS;
  5437. #define MM_SESS_MEMORY_COUNTER_MAX 8
  5438. #define MM_SNAP_SESS_MEMORY_COUNTERS(_index) \
  5439. if (_index >= MM_SESS_MEMORY_COUNTER_MAX) { \
  5440. DbgPrint("Mm: Invalid session mem counter %d %d\n", _index, MM_SESS_MEMORY_COUNTER_MAX); \
  5441. DbgBreakPoint(); \
  5442. } \
  5443. else { \
  5444. MmSessionSpace->Debug2[_index].NonPagablePages = MmSessionSpace->NonPagablePages; \
  5445. MmSessionSpace->Debug2[_index].CommittedPages = MmSessionSpace->CommittedPages; \
  5446. }
  5447. #else
  5448. #define MM_BUMP_SESS_COUNTER(_index, bump)
  5449. #define MM_SNAP_SESS_MEMORY_COUNTERS(_index)
  5450. #endif
  5451. #define MM_SESSION_FAILURE_NO_IDS 0
  5452. #define MM_SESSION_FAILURE_NO_COMMIT 1
  5453. #define MM_SESSION_FAILURE_NO_RESIDENT 2
  5454. #define MM_SESSION_FAILURE_RACE_DETECTED 3
  5455. #define MM_SESSION_FAILURE_NO_SYSPTES 4
  5456. #define MM_SESSION_FAILURE_NO_PAGED_POOL 5
  5457. #define MM_SESSION_FAILURE_NO_NONPAGED_POOL 6
  5458. #define MM_SESSION_FAILURE_NO_IMAGE_VA_SPACE 7
  5459. #define MM_SESSION_FAILURE_NO_SESSION_PAGED_POOL 8
  5460. #define MM_SESSION_FAILURE_NO_AVAILABLE 9
  5461. #define MM_SESSION_FAILURE_CAUSES 10
  5462. ULONG MmSessionFailureCauses[MM_SESSION_FAILURE_CAUSES];
  5463. #define MM_BUMP_SESSION_FAILURES(_index) MmSessionFailureCauses[_index] += 1;
  5464. typedef struct _MM_SESSION_SPACE_FLAGS {
  5465. ULONG Initialized : 1;
  5466. ULONG Filler0 : 1;
  5467. ULONG WorkingSetInserted : 1;
  5468. ULONG SessionListInserted : 1;
  5469. ULONG HasWsLock : 1;
  5470. ULONG DeletePending : 1;
  5471. ULONG Filler : 26;
  5472. } MM_SESSION_SPACE_FLAGS;
  5473. //
  5474. // The session space data structure - allocated per session and only visible at
  5475. // MM_SESSION_SPACE_BASE when in the context of a process from the session.
  5476. // This virtual address space is rotated at context switch time when switching
  5477. // from a process in session A to a process in session B. This rotation is
  5478. // useful for things like providing paged pool per session so many sessions
  5479. // won't exhaust the VA space which backs the system global pool.
  5480. //
  5481. // A kernel PTE is also allocated to double map this page so that global
  5482. // pointers can be maintained to provide system access from any process context.
  5483. // This is needed for things like mutexes and WSL chains.
  5484. //
  5485. typedef struct _MM_SESSION_SPACE {
  5486. ULONG ReferenceCount;
  5487. union {
  5488. ULONG LongFlags;
  5489. MM_SESSION_SPACE_FLAGS Flags;
  5490. } u;
  5491. ULONG SessionId;
  5492. //
  5493. // All the page tables for session space use this as their parent.
  5494. // Note that it's not really a page directory - it's really a page
  5495. // table page itself (the one used to map this very structure).
  5496. //
  5497. // This provides a reference to something that won't go away and
  5498. // is relevant regardless of which process within the session is current.
  5499. //
  5500. PFN_NUMBER SessionPageDirectoryIndex;
  5501. //
  5502. // This is a pointer in global system address space, used to make various
  5503. // fields that can be referenced from any process visible from any process
  5504. // context. This is for things like mutexes, WSL chains, etc.
  5505. //
  5506. struct _MM_SESSION_SPACE *GlobalVirtualAddress;
  5507. //
  5508. // This is the list of the processes in this group that have
  5509. // session space entries.
  5510. //
  5511. LIST_ENTRY ProcessList;
  5512. //
  5513. // Pool allocation counts - these are always valid.
  5514. //
  5515. SIZE_T NonPagedPoolBytes;
  5516. SIZE_T PagedPoolBytes;
  5517. ULONG NonPagedPoolAllocations;
  5518. ULONG PagedPoolAllocations;
  5519. //
  5520. // This is the count of non paged allocations to support this session
  5521. // space. This includes the session structure page table and data pages,
  5522. // WSL page table and data pages, session pool page table pages and session
  5523. // image page table pages. These are all charged against
  5524. // MmResidentAvailable.
  5525. //
  5526. SIZE_T NonPagablePages;
  5527. //
  5528. // This is the count of pages in this session that have been charged against
  5529. // the systemwide commit. This includes all the NonPagablePages plus the
  5530. // data pages they typically map.
  5531. //
  5532. SIZE_T CommittedPages;
  5533. LARGE_INTEGER LastProcessSwappedOutTime;
  5534. #if (_MI_PAGING_LEVELS >= 3)
  5535. //
  5536. // The page directory that maps session space is saved here so
  5537. // trimmers can attach.
  5538. //
  5539. MMPTE PageDirectory;
  5540. #else
  5541. //
  5542. // The second level page tables that map session space are shared
  5543. // by all processes in the session.
  5544. //
  5545. PMMPTE PageTables;
  5546. #endif
  5547. //
  5548. // Session space paged pool support.
  5549. //
  5550. FAST_MUTEX PagedPoolMutex;
  5551. //
  5552. // Start of session paged pool virtual space.
  5553. //
  5554. PVOID PagedPoolStart;
  5555. //
  5556. // Current end of pool virtual space. Can be extended to the
  5557. // end of the session space.
  5558. //
  5559. PVOID PagedPoolEnd;
  5560. //
  5561. // PTE pointers for pool.
  5562. //
  5563. PMMPTE PagedPoolBasePde;
  5564. MM_PAGED_POOL_INFO PagedPoolInfo;
  5565. ULONG Color;
  5566. ULONG ProcessOutSwapCount;
  5567. //
  5568. // This is the list of system images currently valid in
  5569. // this session space. This information is in addition
  5570. // to the module global information in PsLoadedModuleList.
  5571. //
  5572. LIST_ENTRY ImageList;
  5573. //
  5574. // The system PTE self-map entry.
  5575. //
  5576. PMMPTE GlobalPteEntry;
  5577. ULONG CopyOnWriteCount;
  5578. ULONG SessionPoolAllocationFailures[4];
  5579. //
  5580. // The count of "known attachers and the associated event.
  5581. //
  5582. ULONG AttachCount;
  5583. KEVENT AttachEvent;
  5584. PEPROCESS LastProcess;
  5585. //
  5586. // Working set information.
  5587. //
  5588. MMSUPPORT Vm;
  5589. PMMWSLE Wsle;
  5590. ERESOURCE WsLock; // owned by WorkingSetLockOwner
  5591. //
  5592. // This chain is in global system addresses (not session VAs) and can
  5593. // be walked from any system context, ie: for WSL trimming.
  5594. //
  5595. LIST_ENTRY WsListEntry;
  5596. //
  5597. // Support for mapping system views into session space. Each desktop
  5598. // allocates a 3MB heap and the global system view space is only 48M
  5599. // total. This would limit us to only 20-30 users - rotating the
  5600. // system view space with each session removes this limitation.
  5601. //
  5602. MMSESSION Session;
  5603. //
  5604. // This is the driver object entry for WIN32K.SYS
  5605. //
  5606. // It is not a real driver object, but contained here
  5607. // for information such as the DriverUnload routine.
  5608. //
  5609. DRIVER_OBJECT Win32KDriverObject;
  5610. PETHREAD WorkingSetLockOwner;
  5611. //
  5612. // Pool descriptor for less than 1 page allocations.
  5613. //
  5614. POOL_DESCRIPTOR PagedPool;
  5615. //
  5616. // This is generally decremented in process delete (not clean) so that
  5617. // the session data page and mapping PTE can finally be freed when this
  5618. // reaches zero. smss is the only process that decrements it in other
  5619. // places as smss never exits.
  5620. //
  5621. LONG ProcessReferenceToSession;
  5622. LCID LocaleId;
  5623. #if defined (_WIN64)
  5624. //
  5625. // NT64 has enough virtual address space to support per-session special
  5626. // pool.
  5627. //
  5628. PMMPTE SpecialPoolFirstPte;
  5629. PMMPTE SpecialPoolLastPte;
  5630. PMMPTE NextPdeForSpecialPoolExpansion;
  5631. PMMPTE LastPdeForSpecialPoolExpansion;
  5632. PFN_NUMBER SpecialPagesInUse;
  5633. #endif
  5634. #if defined(_IA64_)
  5635. REGION_MAP_INFO SessionMapInfo;
  5636. PFN_NUMBER PageDirectoryParentPage;
  5637. #endif
  5638. #if DBG
  5639. ULONG Debug[MM_SESS_COUNTER_MAX];
  5640. MM_SESSION_MEMORY_COUNTERS Debug2[MM_SESS_MEMORY_COUNTER_MAX];
  5641. #endif
  5642. } MM_SESSION_SPACE, *PMM_SESSION_SPACE;
  5643. extern PMM_SESSION_SPACE MmSessionSpace;
  5644. extern ULONG MiSessionCount;
  5645. //
  5646. // This could be improved to just flush the non-global TB entries.
  5647. //
  5648. #define MI_FLUSH_SESSION_TB() KeFlushEntireTb (TRUE, TRUE);
  5649. //
  5650. // The default number of pages for the session working set minimum & maximum.
  5651. //
  5652. #define MI_SESSION_SPACE_WORKING_SET_MINIMUM 20
  5653. #define MI_SESSION_SPACE_WORKING_SET_MAXIMUM 384
  5654. NTSTATUS
  5655. MiSessionCommitPageTables (
  5656. PVOID StartVa,
  5657. PVOID EndVa
  5658. );
  5659. NTSTATUS
  5660. MiInitializeAndChargePfn (
  5661. OUT PPFN_NUMBER PageFrameIndex,
  5662. IN PMMPTE PointerPde,
  5663. IN PFN_NUMBER ContainingPageFrame,
  5664. IN LOGICAL SessionAllocation
  5665. );
  5666. VOID
  5667. MiSessionPageTableRelease (
  5668. IN PFN_NUMBER PageFrameIndex
  5669. );
  5670. NTSTATUS
  5671. MiInitializeSessionPool (
  5672. VOID
  5673. );
  5674. VOID
  5675. MiCheckSessionPoolAllocations (
  5676. VOID
  5677. );
  5678. VOID
  5679. MiFreeSessionPoolBitMaps (
  5680. VOID
  5681. );
  5682. VOID
  5683. MiDetachSession (
  5684. VOID
  5685. );
  5686. VOID
  5687. MiAttachSession (
  5688. IN PMM_SESSION_SPACE SessionGlobal
  5689. );
  5690. PVOID
  5691. MiAttachToSecureProcessInSession (
  5692. IN PRKAPC_STATE ApcState
  5693. );
  5694. VOID
  5695. MiDetachFromSecureProcessInSession (
  5696. IN PVOID OpaqueSession,
  5697. IN PRKAPC_STATE ApcState
  5698. );
  5699. VOID
  5700. MiReleaseProcessReferenceToSessionDataPage (
  5701. PMM_SESSION_SPACE SessionGlobal
  5702. );
  5703. #define MM_SET_SESSION_RESOURCE_OWNER(_Thread) \
  5704. ASSERT (MmSessionSpace->WorkingSetLockOwner == NULL); \
  5705. MmSessionSpace->WorkingSetLockOwner = _Thread;
  5706. #define MM_CLEAR_SESSION_RESOURCE_OWNER() \
  5707. ASSERT (MmSessionSpace->WorkingSetLockOwner == PsGetCurrentThread()); \
  5708. MmSessionSpace->WorkingSetLockOwner = NULL;
  5709. #define MM_SESSION_SPACE_WS_LOCK_ASSERT() \
  5710. ASSERT (MmSessionSpace->WorkingSetLockOwner == PsGetCurrentThread())
  5711. #define LOCK_SESSION_SPACE_WS(OLDIRQL,_Thread) \
  5712. ASSERT (KeGetCurrentIrql() <= APC_LEVEL); \
  5713. KeRaiseIrql(APC_LEVEL,&OLDIRQL); \
  5714. ExAcquireResourceExclusiveLite(&MmSessionSpace->WsLock, TRUE); \
  5715. MM_SET_SESSION_RESOURCE_OWNER (_Thread);
  5716. #define UNLOCK_SESSION_SPACE_WS(OLDIRQL) \
  5717. MM_CLEAR_SESSION_RESOURCE_OWNER (); \
  5718. ExReleaseResourceLite (&MmSessionSpace->WsLock); \
  5719. KeLowerIrql (OLDIRQL); \
  5720. ASSERT (KeGetCurrentIrql() <= APC_LEVEL);
  5721. extern PMMPTE MiHighestUserPte;
  5722. extern PMMPTE MiHighestUserPde;
  5723. #if (_MI_PAGING_LEVELS >= 4)
  5724. extern PMMPTE MiHighestUserPpe;
  5725. extern PMMPTE MiHighestUserPxe;
  5726. #endif
  5727. NTSTATUS
  5728. MiEmptyWorkingSet (
  5729. IN PMMSUPPORT WsInfo,
  5730. IN LOGICAL WaitOk
  5731. );
  5732. //++
  5733. //ULONG
  5734. //MiGetPdeSessionIndex (
  5735. // IN PVOID va
  5736. // );
  5737. //
  5738. // Routine Description:
  5739. //
  5740. // MiGetPdeSessionIndex returns the session structure index for the PDE
  5741. // will (or does) map the given virtual address.
  5742. //
  5743. // Arguments
  5744. //
  5745. // Va - Supplies the virtual address to locate the PDE index for.
  5746. //
  5747. // Return Value:
  5748. //
  5749. // The index of the PDE entry.
  5750. //
  5751. //--
  5752. #define MiGetPdeSessionIndex(va) ((ULONG)(((ULONG_PTR)(va) - (ULONG_PTR)MmSessionBase) >> PDI_SHIFT))
  5753. //
  5754. // Session space contains the image loader and tracker, virtual
  5755. // address allocator, paged pool allocator, system view image mappings,
  5756. // and working set for kernel mode virtual addresses that are instanced
  5757. // for groups of processes in a Session process group. This
  5758. // process group is identified by a SessionId.
  5759. //
  5760. // Each Session process group's loaded kernel modules, paged pool
  5761. // allocations, working set, and mapped system views are separate from
  5762. // other Session process groups, even though they have the same
  5763. // virtual addresses.
  5764. //
  5765. // This is to support the Hydra multi-user Windows NT system by
  5766. // replicating WIN32K.SYS, and its complement of video and printer drivers,
  5767. // desktop heaps, memory allocations, etc.
  5768. //
  5769. //
  5770. // Structure linked into a session space structure to describe
  5771. // which system images in PsLoadedModuleTable and
  5772. // SESSION_DRIVER_GLOBAL_LOAD_ADDRESS's
  5773. // have been allocated for the current session space.
  5774. //
  5775. // The reference count tracks the number of loads of this image within
  5776. // this session.
  5777. //
  5778. typedef struct _IMAGE_ENTRY_IN_SESSION {
  5779. LIST_ENTRY Link;
  5780. PVOID Address;
  5781. PVOID LastAddress;
  5782. ULONG ImageCountInThisSession;
  5783. PMMPTE PrototypePtes;
  5784. PKLDR_DATA_TABLE_ENTRY DataTableEntry;
  5785. } IMAGE_ENTRY_IN_SESSION, *PIMAGE_ENTRY_IN_SESSION;
  5786. extern LIST_ENTRY MiSessionWsList;
  5787. NTSTATUS
  5788. FASTCALL
  5789. MiCheckPdeForSessionSpace(
  5790. IN PVOID VirtualAddress
  5791. );
  5792. NTSTATUS
  5793. MiShareSessionImage (
  5794. IN PSECTION Section,
  5795. IN OUT PSIZE_T ViewSize
  5796. );
  5797. VOID
  5798. MiSessionWideInitializeAddresses (
  5799. VOID
  5800. );
  5801. NTSTATUS
  5802. MiSessionWideReserveImageAddress (
  5803. IN PUNICODE_STRING pImageName,
  5804. IN PSECTION Section,
  5805. IN ULONG_PTR Alignment,
  5806. OUT PVOID *ppAddr,
  5807. OUT PLOGICAL pAlreadyLoaded
  5808. );
  5809. VOID
  5810. MiInitializeSessionIds (
  5811. VOID
  5812. );
  5813. VOID
  5814. MiInitializeSessionWsSupport(
  5815. VOID
  5816. );
  5817. VOID
  5818. MiSessionAddProcess (
  5819. IN PEPROCESS NewProcess
  5820. );
  5821. VOID
  5822. MiSessionRemoveProcess (
  5823. VOID
  5824. );
  5825. NTSTATUS
  5826. MiRemovePsLoadedModule(
  5827. PKLDR_DATA_TABLE_ENTRY DataTableEntry
  5828. );
  5829. NTSTATUS
  5830. MiRemoveImageSessionWide(
  5831. IN PVOID BaseAddr
  5832. );
  5833. NTSTATUS
  5834. MiDeleteSessionVirtualAddresses(
  5835. IN PVOID VirtualAddress,
  5836. IN SIZE_T NumberOfBytes
  5837. );
  5838. NTSTATUS
  5839. MiUnloadSessionImageByForce (
  5840. IN SIZE_T NumberOfPtes,
  5841. IN PVOID ImageBase
  5842. );
  5843. NTSTATUS
  5844. MiSessionWideGetImageSize(
  5845. IN PVOID BaseAddress,
  5846. OUT PSIZE_T NumberOfBytes OPTIONAL,
  5847. OUT PSIZE_T CommitPages OPTIONAL
  5848. );
  5849. PIMAGE_ENTRY_IN_SESSION
  5850. MiSessionLookupImage (
  5851. IN PVOID BaseAddress
  5852. );
  5853. NTSTATUS
  5854. MiSessionCommitImagePages(
  5855. PVOID BaseAddr,
  5856. SIZE_T Size
  5857. );
  5858. VOID
  5859. MiSessionUnloadAllImages (
  5860. VOID
  5861. );
  5862. VOID
  5863. MiFreeSessionSpaceMap (
  5864. VOID
  5865. );
  5866. NTSTATUS
  5867. MiSessionInitializeWorkingSetList (
  5868. VOID
  5869. );
  5870. VOID
  5871. MiSessionUnlinkWorkingSet (
  5872. VOID
  5873. );
  5874. NTSTATUS
  5875. MiSessionCopyOnWrite (
  5876. IN PVOID FaultingAddress,
  5877. IN PMMPTE PointerPte
  5878. );
  5879. VOID
  5880. MiSessionOutSwapProcess (
  5881. IN PEPROCESS Process
  5882. );
  5883. VOID
  5884. MiSessionInSwapProcess (
  5885. IN PEPROCESS Process
  5886. );
  5887. #if !defined (_X86PAE_)
  5888. #define MI_GET_DIRECTORY_FRAME_FROM_PROCESS(_Process) \
  5889. MI_GET_PAGE_FRAME_FROM_PTE((PMMPTE)(&((_Process)->Pcb.DirectoryTableBase[0])))
  5890. #define MI_GET_HYPER_PAGE_TABLE_FRAME_FROM_PROCESS(_Process) \
  5891. MI_GET_PAGE_FRAME_FROM_PTE((PMMPTE)(&((_Process)->Pcb.DirectoryTableBase[1])))
  5892. #else
  5893. #define MI_GET_DIRECTORY_FRAME_FROM_PROCESS(_Process) \
  5894. (MI_GET_PAGE_FRAME_FROM_PTE(((PMMPTE)((_Process)->PaeTop)) + PD_PER_SYSTEM - 1))
  5895. #define MI_GET_HYPER_PAGE_TABLE_FRAME_FROM_PROCESS(_Process) \
  5896. ((PFN_NUMBER)((_Process)->Pcb.DirectoryTableBase[1]))
  5897. #endif
  5898. #if defined(_MIALT4K_)
  5899. NTSTATUS
  5900. MiSetCopyPagesFor4kPage (
  5901. IN PEPROCESS Process,
  5902. IN PMMVAD Vad,
  5903. IN OUT PVOID StartingAddress,
  5904. IN OUT PVOID EndingAddress,
  5905. IN ULONG ProtectionMask,
  5906. OUT PMMVAD *CallerNewVad
  5907. );
  5908. VOID
  5909. MiRemoveAliasedVads (
  5910. IN PEPROCESS Process,
  5911. IN PMMVAD Vad
  5912. );
  5913. PVOID
  5914. MiDuplicateAliasVadList (
  5915. IN PMMVAD Vad
  5916. );
  5917. #endif
  5918. //
  5919. // The LDR_DATA_TABLE_ENTRY->LoadedImports is used as a list of imported DLLs.
  5920. //
  5921. // This field is zero if the module was loaded at boot time and the
  5922. // import information was never filled in.
  5923. //
  5924. // This field is -1 if no imports are defined by the module.
  5925. //
  5926. // This field contains a valid paged pool PLDR_DATA_TABLE_ENTRY pointer
  5927. // with a low-order (bit 0) tag of 1 if there is only 1 usable import needed
  5928. // by this driver.
  5929. //
  5930. // This field will contain a valid paged pool PLOAD_IMPORTS pointer in all
  5931. // other cases (ie: where at least 2 imports exist).
  5932. //
  5933. typedef struct _LOAD_IMPORTS {
  5934. SIZE_T Count;
  5935. PKLDR_DATA_TABLE_ENTRY Entry[1];
  5936. } LOAD_IMPORTS, *PLOAD_IMPORTS;
  5937. #define LOADED_AT_BOOT ((PLOAD_IMPORTS)0)
  5938. #define NO_IMPORTS_USED ((PLOAD_IMPORTS)-2)
  5939. #define SINGLE_ENTRY(ImportVoid) ((ULONG)((ULONG_PTR)(ImportVoid) & 0x1))
  5940. #define SINGLE_ENTRY_TO_POINTER(ImportVoid) ((PKLDR_DATA_TABLE_ENTRY)((ULONG_PTR)(ImportVoid) & ~0x1))
  5941. #define POINTER_TO_SINGLE_ENTRY(Pointer) ((PKLDR_DATA_TABLE_ENTRY)((ULONG_PTR)(Pointer) | 0x1))
  5942. //
  5943. // This tracks allocated group virtual addresses. The term SESSIONWIDE is used
  5944. // to denote data that is the same across all sessions (as opposed to
  5945. // per-session data which can vary from session to session).
  5946. //
  5947. // Since each driver loaded into a session space is linked and fixed up
  5948. // against the system image, it must remain at the same virtual address
  5949. // across the system regardless of the session.
  5950. //
  5951. // A list is maintained by the group allocator of which virtual
  5952. // addresses are in use and by which DLL.
  5953. //
  5954. // The reference count tracks the number of sessions that have loaded
  5955. // this image.
  5956. //
  5957. // Access to this structure is guarded by the MmSystemLoadLock.
  5958. //
  5959. typedef struct _SESSIONWIDE_DRIVER_ADDRESS {
  5960. LIST_ENTRY Link;
  5961. ULONG ReferenceCount;
  5962. PVOID Address;
  5963. ULONG_PTR Size;
  5964. ULONG_PTR WritablePages;
  5965. UNICODE_STRING FullDllName;
  5966. } SESSIONWIDE_DRIVER_ADDRESS, *PSESSIONWIDE_DRIVER_ADDRESS;
  5967. // #define _MI_DEBUG_RONLY 1 // Uncomment this for session readonly tracking
  5968. #if _MI_DEBUG_RONLY
  5969. VOID
  5970. MiAssertNotSessionData (
  5971. IN PMMPTE PointerPte
  5972. );
  5973. VOID
  5974. MiLogSessionDataStart (
  5975. IN PKLDR_DATA_TABLE_ENTRY DataTableEntry
  5976. );
  5977. #define MI_ASSERT_NOT_SESSION_DATA(PTE) MiAssertNotSessionData(PTE)
  5978. #define MI_LOG_SESSION_DATA_START(DataTableEntry) MiLogSessionDataStart(DataTableEntry)
  5979. #else
  5980. #define MI_ASSERT_NOT_SESSION_DATA(PTE)
  5981. #define MI_LOG_SESSION_DATA_START(DataTableEntry)
  5982. #endif
  5983. //
  5984. // This tracks driver-specified individual verifier thunks.
  5985. //
  5986. typedef struct _DRIVER_SPECIFIED_VERIFIER_THUNKS {
  5987. LIST_ENTRY ListEntry;
  5988. PKLDR_DATA_TABLE_ENTRY DataTableEntry;
  5989. ULONG NumberOfThunks;
  5990. } DRIVER_SPECIFIED_VERIFIER_THUNKS, *PDRIVER_SPECIFIED_VERIFIER_THUNKS;
  5991. // #define _MI_DEBUG_SUB 1 // Uncomment this for subsection logging
  5992. #if defined (_MI_DEBUG_SUB)
  5993. extern ULONG MiTrackSubs;
  5994. #define MI_SUB_BACKTRACE_LENGTH 8
  5995. typedef struct _MI_SUB_TRACES {
  5996. PETHREAD Thread;
  5997. PMSUBSECTION Subsection;
  5998. PCONTROL_AREA ControlArea;
  5999. ULONG_PTR CallerId;
  6000. PVOID StackTrace [MI_SUB_BACKTRACE_LENGTH];
  6001. MSUBSECTION SubsectionContents;
  6002. CONTROL_AREA ControlAreaContents;
  6003. } MI_SUB_TRACES, *PMI_SUB_TRACES;
  6004. extern LONG MiSubsectionIndex;
  6005. extern PMI_SUB_TRACES MiSubsectionTraces;
  6006. VOID
  6007. FORCEINLINE
  6008. MiSnapSubsection (
  6009. IN PMSUBSECTION Subsection,
  6010. IN ULONG CallerId
  6011. )
  6012. {
  6013. PMI_SUB_TRACES Information;
  6014. PCONTROL_AREA ControlArea;
  6015. ULONG Index;
  6016. ULONG Hash;
  6017. if (MiSubsectionTraces == NULL) {
  6018. return;
  6019. }
  6020. ControlArea = Subsection->ControlArea;
  6021. #if 0
  6022. if (ControlArea->u.Flags.Mft == 0) {
  6023. return;
  6024. }
  6025. #endif
  6026. Index = InterlockedIncrement(&MiSubsectionIndex);
  6027. Index &= (MiTrackSubs - 1);
  6028. Information = &MiSubsectionTraces[Index];
  6029. Information->Subsection = Subsection;
  6030. Information->ControlArea = ControlArea;
  6031. *(PMSUBSECTION)&Information->SubsectionContents = *Subsection;
  6032. *(PCONTROL_AREA)&Information->ControlAreaContents = *ControlArea;
  6033. Information->Thread = PsGetCurrentThread();
  6034. Information->CallerId = CallerId;
  6035. RtlCaptureStackBackTrace (0, MI_SUB_BACKTRACE_LENGTH, Information->StackTrace, &Hash);
  6036. }
  6037. #define MI_SNAP_SUB(_Sub, callerid) MiSnapSubsection(_Sub, callerid)
  6038. #else
  6039. #define MI_SNAP_SUB(_Sub, callerid)
  6040. #endif
  6041. #ifdef _MI_MESSAGE_SERVER
  6042. LOGICAL
  6043. MiQueueMessage (
  6044. IN PVOID Message
  6045. );
  6046. PVOID
  6047. MiRemoveMessage (
  6048. VOID
  6049. );
  6050. #define MI_INSTRUMENT_QUEUE(Message) MiQueueMessage (Message)
  6051. #define MI_INSTRUMENTR_QUEUE() MiRemoveMessage ()
  6052. #else
  6053. #define MI_INSTRUMENT_QUEUE(Message)
  6054. #define MI_INSTRUMENTR_QUEUE()
  6055. #endif
  6056. #endif // MI