Source code of Windows XP (NT5)
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

1630 lines
36 KiB

  1. /*++
  2. Copyright (c) 1989 Microsoft Corporation
  3. Module Name:
  4. mmsup.c
  5. Abstract:
  6. This module contains the various routines for miscellaneous support
  7. operations for memory management.
  8. Author:
  9. Lou Perazzoli (loup) 31-Aug-1989
  10. Landy Wang (landyw) 02-June-1997
  11. Revision History:
  12. --*/
  13. #include "mi.h"
  14. #ifdef ALLOC_PRAGMA
  15. #pragma alloc_text(PAGE, MmHibernateInformation)
  16. #pragma alloc_text(PAGE, MiMakeSystemAddressValid)
  17. #pragma alloc_text(PAGE, MiIsPteDecommittedPage)
  18. #endif
  19. #if _WIN64
  20. #if DBGXX
  21. VOID
  22. MiCheckPageTableTrim(
  23. IN PMMPTE PointerPte
  24. );
  25. #endif
  26. #endif
  27. ULONG
  28. FASTCALL
  29. MiIsPteDecommittedPage (
  30. IN PMMPTE PointerPte
  31. )
  32. /*++
  33. Routine Description:
  34. This function checks the contents of a PTE to determine if the
  35. PTE is explicitly decommitted.
  36. If the PTE is a prototype PTE and the protection is not in the
  37. prototype PTE, the value FALSE is returned.
  38. Arguments:
  39. PointerPte - Supplies a pointer to the PTE to examine.
  40. Return Value:
  41. TRUE if the PTE is in the explicit decommitted state.
  42. FALSE if the PTE is not in the explicit decommitted state.
  43. Environment:
  44. Kernel mode, APCs disabled, WorkingSetLock held.
  45. --*/
  46. {
  47. MMPTE PteContents;
  48. PteContents = *PointerPte;
  49. //
  50. // If the protection in the PTE is not decommitted, return FALSE.
  51. //
  52. if (PteContents.u.Soft.Protection != MM_DECOMMIT) {
  53. return FALSE;
  54. }
  55. //
  56. // Check to make sure the protection field is really being interpreted
  57. // correctly.
  58. //
  59. if (PteContents.u.Hard.Valid == 1) {
  60. //
  61. // The PTE is valid and therefore cannot be decommitted.
  62. //
  63. return FALSE;
  64. }
  65. if ((PteContents.u.Soft.Prototype == 1) &&
  66. (PteContents.u.Soft.PageFileHigh != MI_PTE_LOOKUP_NEEDED)) {
  67. //
  68. // The PTE's protection is not known as it is in
  69. // prototype PTE format. Return FALSE.
  70. //
  71. return FALSE;
  72. }
  73. //
  74. // It is a decommitted PTE.
  75. //
  76. return TRUE;
  77. }
  78. //
  79. // Data for is protection compatible.
  80. //
  81. ULONG MmCompatibleProtectionMask[8] = {
  82. PAGE_NOACCESS,
  83. PAGE_NOACCESS | PAGE_READONLY | PAGE_WRITECOPY,
  84. PAGE_NOACCESS | PAGE_EXECUTE,
  85. PAGE_NOACCESS | PAGE_READONLY | PAGE_WRITECOPY | PAGE_EXECUTE |
  86. PAGE_EXECUTE_READ,
  87. PAGE_NOACCESS | PAGE_READONLY | PAGE_WRITECOPY | PAGE_READWRITE,
  88. PAGE_NOACCESS | PAGE_READONLY | PAGE_WRITECOPY,
  89. PAGE_NOACCESS | PAGE_READONLY | PAGE_WRITECOPY | PAGE_READWRITE |
  90. PAGE_EXECUTE | PAGE_EXECUTE_READ | PAGE_EXECUTE_READWRITE |
  91. PAGE_EXECUTE_WRITECOPY,
  92. PAGE_NOACCESS | PAGE_READONLY | PAGE_WRITECOPY | PAGE_EXECUTE |
  93. PAGE_EXECUTE_READ | PAGE_EXECUTE_WRITECOPY
  94. };
  95. ULONG
  96. FASTCALL
  97. MiIsProtectionCompatible (
  98. IN ULONG OldProtect,
  99. IN ULONG NewProtect
  100. )
  101. /*++
  102. Routine Description:
  103. This function takes two user supplied page protections and checks
  104. to see if the new protection is compatible with the old protection.
  105. protection compatible protections
  106. NoAccess NoAccess
  107. ReadOnly NoAccess, ReadOnly, ReadWriteCopy
  108. ReadWriteCopy NoAccess, ReadOnly, ReadWriteCopy
  109. ReadWrite NoAccess, ReadOnly, ReadWriteCopy, ReadWrite
  110. Execute NoAccess, Execute
  111. ExecuteRead NoAccess, ReadOnly, ReadWriteCopy, Execute, ExecuteRead,
  112. ExecuteWriteCopy
  113. ExecuteWrite NoAccess, ReadOnly, ReadWriteCopy, Execute, ExecuteRead,
  114. ExecuteWriteCopy, ReadWrite, ExecuteWrite
  115. ExecuteWriteCopy NoAccess, ReadOnly, ReadWriteCopy, Execute, ExecuteRead,
  116. ExecuteWriteCopy
  117. Arguments:
  118. OldProtect - Supplies the protection to be compatible with.
  119. NewProtect - Supplies the protection to check out.
  120. Return Value:
  121. Returns TRUE if the protection is compatible, FALSE if not.
  122. Environment:
  123. Kernel Mode.
  124. --*/
  125. {
  126. ULONG Mask;
  127. ULONG ProtectMask;
  128. ULONG PteProtection;
  129. PteProtection = MiMakeProtectionMask (OldProtect);
  130. if (PteProtection == MM_INVALID_PROTECTION) {
  131. return FALSE;
  132. }
  133. Mask = PteProtection & 0x7;
  134. ProtectMask = MmCompatibleProtectionMask[Mask] | PAGE_GUARD | PAGE_NOCACHE;
  135. if ((ProtectMask | NewProtect) != ProtectMask) {
  136. return FALSE;
  137. }
  138. return TRUE;
  139. }
  140. ULONG
  141. FASTCALL
  142. MiIsPteProtectionCompatible (
  143. IN ULONG PteProtection,
  144. IN ULONG NewProtect
  145. )
  146. {
  147. ULONG Mask;
  148. ULONG ProtectMask;
  149. Mask = PteProtection & 0x7;
  150. ProtectMask = MmCompatibleProtectionMask[Mask] | PAGE_GUARD | PAGE_NOCACHE;
  151. if ((ProtectMask | NewProtect) != ProtectMask) {
  152. return FALSE;
  153. }
  154. return TRUE;
  155. }
  156. //
  157. // Protection data for MiMakeProtectionMask
  158. //
  159. CCHAR MmUserProtectionToMask1[16] = {
  160. 0,
  161. MM_NOACCESS,
  162. MM_READONLY,
  163. -1,
  164. MM_READWRITE,
  165. -1,
  166. -1,
  167. -1,
  168. MM_WRITECOPY,
  169. -1,
  170. -1,
  171. -1,
  172. -1,
  173. -1,
  174. -1,
  175. -1 };
  176. CCHAR MmUserProtectionToMask2[16] = {
  177. 0,
  178. MM_EXECUTE,
  179. MM_EXECUTE_READ,
  180. -1,
  181. MM_EXECUTE_READWRITE,
  182. -1,
  183. -1,
  184. -1,
  185. MM_EXECUTE_WRITECOPY,
  186. -1,
  187. -1,
  188. -1,
  189. -1,
  190. -1,
  191. -1,
  192. -1 };
  193. ULONG
  194. FASTCALL
  195. MiMakeProtectionMask (
  196. IN ULONG Protect
  197. )
  198. /*++
  199. Routine Description:
  200. This function takes a user supplied protection and converts it
  201. into a 5-bit protection code for the PTE.
  202. Arguments:
  203. Protect - Supplies the protection.
  204. Return Value:
  205. Returns the protection code for use in the PTE. Note that
  206. MM_INVALID_PROTECTION (-1) is returned for an invalid protection
  207. request. Since valid PTE protections fit in 5 bits and are
  208. zero-extended, it's easy for callers to distinguish this.
  209. Environment:
  210. Kernel Mode.
  211. --*/
  212. {
  213. ULONG Field1;
  214. ULONG Field2;
  215. ULONG ProtectCode;
  216. if (Protect >= (PAGE_NOCACHE * 2)) {
  217. return MM_INVALID_PROTECTION;
  218. }
  219. Field1 = Protect & 0xF;
  220. Field2 = (Protect >> 4) & 0xF;
  221. //
  222. // Make sure at least one field is set.
  223. //
  224. if (Field1 == 0) {
  225. if (Field2 == 0) {
  226. //
  227. // Both fields are zero, return failure.
  228. //
  229. return MM_INVALID_PROTECTION;
  230. }
  231. ProtectCode = MmUserProtectionToMask2[Field2];
  232. } else {
  233. if (Field2 != 0) {
  234. //
  235. // Both fields are non-zero, raise failure.
  236. //
  237. return MM_INVALID_PROTECTION;
  238. }
  239. ProtectCode = MmUserProtectionToMask1[Field1];
  240. }
  241. if (ProtectCode == -1) {
  242. return MM_INVALID_PROTECTION;
  243. }
  244. if (Protect & PAGE_GUARD) {
  245. if (ProtectCode == MM_NOACCESS) {
  246. //
  247. // Invalid protection, no access and no_cache.
  248. //
  249. return MM_INVALID_PROTECTION;
  250. }
  251. ProtectCode |= MM_GUARD_PAGE;
  252. }
  253. if (Protect & PAGE_NOCACHE) {
  254. if (ProtectCode == MM_NOACCESS) {
  255. //
  256. // Invalid protection, no access and no cache.
  257. //
  258. return MM_INVALID_PROTECTION;
  259. }
  260. ProtectCode |= MM_NOCACHE;
  261. }
  262. return ProtectCode;
  263. }
  264. ULONG
  265. MiDoesPdeExistAndMakeValid (
  266. IN PMMPTE PointerPde,
  267. IN PEPROCESS TargetProcess,
  268. IN LOGICAL PfnLockHeld,
  269. OUT PULONG Waited
  270. )
  271. /*++
  272. Routine Description:
  273. This routine examines the specified Page Directory Entry to determine
  274. if the page table page mapped by the PDE exists.
  275. If the page table page exists and is not currently in memory, the
  276. working set mutex and, if held, the PFN lock are released and the
  277. page table page is faulted into the working set. The mutexes are
  278. reacquired.
  279. If the PDE exists, the function returns TRUE.
  280. Arguments:
  281. PointerPde - Supplies a pointer to the PDE to examine and potentially
  282. bring into the working set.
  283. TargetProcess - Supplies a pointer to the current process.
  284. PfnLockHeld - Supplies the value TRUE if the PFN lock is held, FALSE
  285. otherwise.
  286. Waited - Supplies a pointer to a ULONG to increment if the mutex is released
  287. and reacquired. Note this value may be incremented more than once.
  288. Return Value:
  289. TRUE if the PDE exists, FALSE if the PDE is zero.
  290. Environment:
  291. Kernel mode, APCs disabled, WorkingSetLock held.
  292. --*/
  293. {
  294. PMMPTE PointerPte;
  295. KIRQL OldIrql;
  296. OldIrql = APC_LEVEL;
  297. if (PointerPde->u.Long == 0) {
  298. //
  299. // This page directory entry doesn't exist, return FALSE.
  300. //
  301. return FALSE;
  302. }
  303. if (PointerPde->u.Hard.Valid == 1) {
  304. //
  305. // Already valid.
  306. //
  307. return TRUE;
  308. }
  309. //
  310. // Page directory entry exists, it is either valid, in transition
  311. // or in the paging file. Fault it in.
  312. //
  313. if (PfnLockHeld) {
  314. UNLOCK_PFN (OldIrql);
  315. *Waited += 1;
  316. }
  317. PointerPte = MiGetVirtualAddressMappedByPte (PointerPde);
  318. *Waited += MiMakeSystemAddressValid (PointerPte, TargetProcess);
  319. if (PfnLockHeld) {
  320. LOCK_PFN (OldIrql);
  321. }
  322. return TRUE;
  323. }
  324. #if (_MI_PAGING_LEVELS >= 4)
  325. VOID
  326. MiMakePxeExistAndMakeValid (
  327. IN PMMPTE PointerPxe,
  328. IN PEPROCESS TargetProcess,
  329. IN LOGICAL PfnLockHeld
  330. )
  331. /*++
  332. Routine Description:
  333. This routine examines the extended Page Directory Parent Entry to
  334. determine if the page directory parent page mapped by the PXE exists.
  335. If the page directory parent page exists and is not currently in memory, the
  336. working set mutex and, if held, the PFN lock are released and the
  337. page directory parent page is faulted into the working set. The mutex and
  338. lock are then reacquired.
  339. If the PXE does not exist, a zero filled page directory parent is created
  340. and it is brought into the working set.
  341. Arguments:
  342. PointerPxe - Supplies a pointer to the PXE to examine and bring
  343. into the working set.
  344. TargetProcess - Supplies a pointer to the current process.
  345. PfnLockHeld - Supplies the value TRUE if the PFN lock is held, FALSE
  346. otherwise.
  347. Return Value:
  348. None.
  349. Environment:
  350. Kernel mode, APCs disabled, WorkingSetLock held.
  351. --*/
  352. {
  353. PMMPTE PointerPpe;
  354. KIRQL OldIrql;
  355. if (PointerPxe->u.Hard.Valid == 1) {
  356. //
  357. // Already valid.
  358. //
  359. return;
  360. }
  361. //
  362. // Deal with the page directory page first.
  363. //
  364. if (PfnLockHeld) {
  365. OldIrql = APC_LEVEL;
  366. UNLOCK_PFN (OldIrql);
  367. }
  368. //
  369. // Fault it in.
  370. //
  371. PointerPpe = MiGetVirtualAddressMappedByPte (PointerPxe);
  372. MiMakeSystemAddressValid (PointerPpe, TargetProcess);
  373. ASSERT (PointerPxe->u.Hard.Valid == 1);
  374. if (PfnLockHeld) {
  375. LOCK_PFN (OldIrql);
  376. }
  377. return;
  378. }
  379. #endif
  380. #if (_MI_PAGING_LEVELS >= 3)
  381. VOID
  382. MiMakePpeExistAndMakeValid (
  383. IN PMMPTE PointerPpe,
  384. IN PEPROCESS TargetProcess,
  385. IN LOGICAL PfnLockHeld
  386. )
  387. /*++
  388. Routine Description:
  389. This routine examines the specified Page Directory Parent Entry to
  390. determine if the page directory page mapped by the PPE exists.
  391. If the page directory page exists and is not currently in memory, the
  392. working set mutex and, if held, the PFN lock are released and the
  393. page directory page is faulted into the working set. The mutex and lock
  394. are then reacquired.
  395. If the PPE does not exist, a zero filled page directory is created
  396. and it is brought into the working set.
  397. Arguments:
  398. PointerPpe - Supplies a pointer to the PPE to examine and bring
  399. into the working set.
  400. TargetProcess - Supplies a pointer to the current process.
  401. PfnLockHeld - Supplies the value TRUE if the PFN lock is held, FALSE
  402. otherwise.
  403. Return Value:
  404. None.
  405. Environment:
  406. Kernel mode, APCs disabled, WorkingSetLock held.
  407. --*/
  408. {
  409. PMMPTE PointerPde;
  410. PMMPTE PointerPxe;
  411. KIRQL OldIrql;
  412. PointerPxe = MiGetPteAddress (PointerPpe);
  413. if ((PointerPxe->u.Hard.Valid == 1) &&
  414. (PointerPpe->u.Hard.Valid == 1)) {
  415. //
  416. // Already valid.
  417. //
  418. return;
  419. }
  420. //
  421. // Deal with the page directory and parent pages first.
  422. //
  423. if (PfnLockHeld) {
  424. OldIrql = APC_LEVEL;
  425. UNLOCK_PFN (OldIrql);
  426. }
  427. //
  428. // Fault it in.
  429. //
  430. PointerPde = MiGetVirtualAddressMappedByPte (PointerPpe);
  431. MiMakeSystemAddressValid (PointerPde, TargetProcess);
  432. ASSERT (PointerPxe->u.Hard.Valid == 1);
  433. ASSERT (PointerPpe->u.Hard.Valid == 1);
  434. if (PfnLockHeld) {
  435. LOCK_PFN (OldIrql);
  436. }
  437. return;
  438. }
  439. #endif
  440. VOID
  441. MiMakePdeExistAndMakeValid (
  442. IN PMMPTE PointerPde,
  443. IN PEPROCESS TargetProcess,
  444. IN LOGICAL PfnLockHeld
  445. )
  446. /*++
  447. Routine Description:
  448. This routine examines the specified Page Directory Parent Entry to
  449. determine if the page directory page mapped by the PPE exists. If it does,
  450. then it examines the specified Page Directory Entry to determine if
  451. the page table page mapped by the PDE exists.
  452. If the page table page exists and is not currently in memory, the
  453. working set mutex and, if held, the PFN lock are released and the
  454. page table page is faulted into the working set. The mutexes are
  455. reacquired.
  456. If the PDE does not exist, a zero filled PTE is created and it
  457. too is brought into the working set.
  458. Arguments:
  459. PointerPde - Supplies a pointer to the PDE to examine and bring
  460. into the working set.
  461. TargetProcess - Supplies a pointer to the current process.
  462. PfnLockHeld - Supplies the value TRUE if the PFN lock is held, FALSE
  463. otherwise.
  464. Return Value:
  465. None.
  466. Environment:
  467. Kernel mode, APCs disabled, WorkingSetLock held.
  468. --*/
  469. {
  470. PMMPTE PointerPte;
  471. PMMPTE PointerPpe;
  472. PMMPTE PointerPxe;
  473. KIRQL OldIrql;
  474. PointerPpe = MiGetPteAddress (PointerPde);
  475. PointerPxe = MiGetPdeAddress (PointerPde);
  476. if ((PointerPxe->u.Hard.Valid == 1) &&
  477. (PointerPpe->u.Hard.Valid == 1) &&
  478. (PointerPde->u.Hard.Valid == 1)) {
  479. //
  480. // Already valid.
  481. //
  482. return;
  483. }
  484. //
  485. // Page directory parent (or extended parent) entry not valid,
  486. // make it valid.
  487. //
  488. OldIrql = APC_LEVEL;
  489. PointerPte = MiGetVirtualAddressMappedByPte (PointerPde);
  490. do {
  491. if (PfnLockHeld) {
  492. UNLOCK_PFN (OldIrql);
  493. }
  494. //
  495. // Fault it in.
  496. //
  497. MiMakeSystemAddressValid (PointerPpe, TargetProcess);
  498. ASSERT (PointerPxe->u.Hard.Valid == 1);
  499. //
  500. // Now deal with the page directory page.
  501. //
  502. MiMakeSystemAddressValid (PointerPde, TargetProcess);
  503. ASSERT (PointerPxe->u.Hard.Valid == 1);
  504. ASSERT (PointerPpe->u.Hard.Valid == 1);
  505. //
  506. // Now deal with the page table page.
  507. //
  508. ASSERT (OldIrql == APC_LEVEL);
  509. //
  510. // Fault it in.
  511. //
  512. MiMakeSystemAddressValid (PointerPte, TargetProcess);
  513. ASSERT (PointerPxe->u.Hard.Valid == 1);
  514. ASSERT (PointerPpe->u.Hard.Valid == 1);
  515. ASSERT (PointerPde->u.Hard.Valid == 1);
  516. if (PfnLockHeld) {
  517. LOCK_PFN (OldIrql);
  518. }
  519. } while ((PointerPxe->u.Hard.Valid == 0) ||
  520. (PointerPpe->u.Hard.Valid == 0) ||
  521. (PointerPde->u.Hard.Valid == 0));
  522. return;
  523. }
  524. ULONG
  525. FASTCALL
  526. MiMakeSystemAddressValid (
  527. IN PVOID VirtualAddress,
  528. IN PEPROCESS CurrentProcess
  529. )
  530. /*++
  531. Routine Description:
  532. This routine checks to see if the virtual address is valid, and if
  533. not makes it valid.
  534. Arguments:
  535. VirtualAddress - Supplies the virtual address to make valid.
  536. CurrentProcess - Supplies a pointer to the current process.
  537. Return Value:
  538. Returns TRUE if lock released and wait performed, FALSE otherwise.
  539. Environment:
  540. Kernel mode, APCs disabled, WorkingSetLock held.
  541. --*/
  542. {
  543. NTSTATUS status;
  544. LOGICAL WsHeldSafe;
  545. ULONG Waited;
  546. Waited = FALSE;
  547. ASSERT (VirtualAddress > MM_HIGHEST_USER_ADDRESS);
  548. ASSERT ((VirtualAddress < MM_PAGED_POOL_START) ||
  549. (VirtualAddress > MmPagedPoolEnd));
  550. while (!MmIsAddressValid(VirtualAddress)) {
  551. //
  552. // The virtual address is not present. Release
  553. // the working set mutex and fault it in.
  554. //
  555. // The working set lock may have been acquired safely or unsafely
  556. // by our caller. Handle both cases here and below.
  557. //
  558. UNLOCK_WS_REGARDLESS(CurrentProcess, WsHeldSafe);
  559. status = MmAccessFault (FALSE, VirtualAddress, KernelMode, (PVOID)0);
  560. if (!NT_SUCCESS(status)) {
  561. KeBugCheckEx (KERNEL_DATA_INPAGE_ERROR,
  562. 1,
  563. (ULONG)status,
  564. (ULONG_PTR)CurrentProcess,
  565. (ULONG_PTR)VirtualAddress);
  566. }
  567. LOCK_WS_REGARDLESS(CurrentProcess, WsHeldSafe);
  568. Waited = TRUE;
  569. }
  570. return Waited;
  571. }
  572. ULONG
  573. FASTCALL
  574. MiMakeSystemAddressValidPfnWs (
  575. IN PVOID VirtualAddress,
  576. IN PEPROCESS CurrentProcess OPTIONAL
  577. )
  578. /*++
  579. Routine Description:
  580. This routine checks to see if the virtual address is valid, and if
  581. not makes it valid.
  582. Arguments:
  583. VirtualAddress - Supplies the virtual address to make valid.
  584. CurrentProcess - Supplies a pointer to the current process, if the
  585. working set lock is not held, this value is NULL.
  586. Return Value:
  587. Returns TRUE if lock released and wait performed, FALSE otherwise.
  588. Environment:
  589. Kernel mode, APCs disabled, PFN lock held, working set lock held
  590. if CurrentProcess != NULL.
  591. --*/
  592. {
  593. NTSTATUS status;
  594. ULONG Waited;
  595. KIRQL OldIrql;
  596. LOGICAL WsHeldSafe;
  597. Waited = FALSE;
  598. OldIrql = APC_LEVEL;
  599. //
  600. // Initializing WsHeldSafe is not needed for correctness, but without it
  601. // the compiler cannot compile this code W4 to check for use of
  602. // uninitialized variables.
  603. //
  604. WsHeldSafe = FALSE;
  605. ASSERT (VirtualAddress > MM_HIGHEST_USER_ADDRESS);
  606. while (!MmIsAddressValid(VirtualAddress)) {
  607. //
  608. // The virtual address is not present. Release
  609. // the working set mutex and fault it in.
  610. //
  611. UNLOCK_PFN (OldIrql);
  612. if (CurrentProcess != NULL) {
  613. //
  614. // The working set lock may have been acquired safely or unsafely
  615. // by our caller. Handle both cases here and below.
  616. //
  617. UNLOCK_WS_REGARDLESS(CurrentProcess, WsHeldSafe);
  618. }
  619. status = MmAccessFault (FALSE, VirtualAddress, KernelMode, (PVOID)0);
  620. if (!NT_SUCCESS(status)) {
  621. KeBugCheckEx (KERNEL_DATA_INPAGE_ERROR,
  622. 2,
  623. (ULONG)status,
  624. (ULONG_PTR)CurrentProcess,
  625. (ULONG_PTR)VirtualAddress);
  626. }
  627. if (CurrentProcess != NULL) {
  628. LOCK_WS_REGARDLESS(CurrentProcess, WsHeldSafe);
  629. }
  630. LOCK_PFN (OldIrql);
  631. Waited = TRUE;
  632. }
  633. return Waited;
  634. }
  635. ULONG
  636. FASTCALL
  637. MiMakeSystemAddressValidPfnSystemWs (
  638. IN PVOID VirtualAddress
  639. )
  640. /*++
  641. Routine Description:
  642. This routine checks to see if the virtual address is valid, and if
  643. not makes it valid.
  644. Arguments:
  645. VirtualAddress - Supplies the virtual address to make valid.
  646. Return Value:
  647. Returns TRUE if lock released and wait performed, FALSE otherwise.
  648. Environment:
  649. Kernel mode, APCs disabled, PFN lock held, system working set lock held.
  650. --*/
  651. {
  652. NTSTATUS status;
  653. ULONG Waited;
  654. KIRQL OldIrql;
  655. KIRQL OldIrqlWs;
  656. LOGICAL SessionSpace;
  657. Waited = FALSE;
  658. OldIrql = APC_LEVEL;
  659. OldIrqlWs = APC_LEVEL;
  660. ASSERT (VirtualAddress > MM_HIGHEST_USER_ADDRESS);
  661. SessionSpace = MI_IS_SESSION_IMAGE_ADDRESS (VirtualAddress);
  662. while (!MmIsAddressValid(VirtualAddress)) {
  663. //
  664. // The virtual address is not present. Release
  665. // the working set mutex and fault it in.
  666. //
  667. UNLOCK_PFN (OldIrql);
  668. if (SessionSpace == TRUE) {
  669. UNLOCK_SESSION_SPACE_WS (OldIrqlWs);
  670. }
  671. else {
  672. UNLOCK_SYSTEM_WS (OldIrqlWs);
  673. }
  674. status = MmAccessFault (FALSE, VirtualAddress, KernelMode, (PVOID)0);
  675. if (!NT_SUCCESS(status)) {
  676. KeBugCheckEx (KERNEL_DATA_INPAGE_ERROR,
  677. 2,
  678. (ULONG)status,
  679. (ULONG_PTR)0,
  680. (ULONG_PTR)VirtualAddress);
  681. }
  682. if (SessionSpace == TRUE) {
  683. LOCK_SESSION_SPACE_WS (OldIrqlWs, PsGetCurrentThread ());
  684. }
  685. else {
  686. LOCK_SYSTEM_WS (OldIrqlWs, PsGetCurrentThread ());
  687. }
  688. LOCK_PFN (OldIrql);
  689. Waited = TRUE;
  690. }
  691. return Waited;
  692. }
  693. ULONG
  694. FASTCALL
  695. MiMakeSystemAddressValidPfn (
  696. IN PVOID VirtualAddress
  697. )
  698. /*++
  699. Routine Description:
  700. This routine checks to see if the virtual address is valid, and if
  701. not makes it valid.
  702. Arguments:
  703. VirtualAddress - Supplies the virtual address to make valid.
  704. Return Value:
  705. Returns TRUE if lock released and wait performed, FALSE otherwise.
  706. Environment:
  707. Kernel mode, APCs disabled, only the PFN Lock held.
  708. --*/
  709. {
  710. NTSTATUS status;
  711. KIRQL OldIrql = APC_LEVEL;
  712. ULONG Waited = FALSE;
  713. ASSERT (VirtualAddress > MM_HIGHEST_USER_ADDRESS);
  714. while (!MmIsAddressValid(VirtualAddress)) {
  715. //
  716. // The virtual address is not present. Release
  717. // the working set mutex and fault it in.
  718. //
  719. UNLOCK_PFN (OldIrql);
  720. status = MmAccessFault (FALSE, VirtualAddress, KernelMode, (PVOID)0);
  721. if (!NT_SUCCESS(status)) {
  722. KeBugCheckEx (KERNEL_DATA_INPAGE_ERROR,
  723. 3,
  724. (ULONG)status,
  725. (ULONG_PTR)VirtualAddress,
  726. 0);
  727. }
  728. LOCK_PFN (OldIrql);
  729. Waited = TRUE;
  730. }
  731. return Waited;
  732. }
  733. ULONG
  734. FASTCALL
  735. MiLockPagedAddress (
  736. IN PVOID VirtualAddress,
  737. IN ULONG PfnLockHeld
  738. )
  739. /*++
  740. Routine Description:
  741. This routine checks to see if the virtual address is valid, and if
  742. not makes it valid.
  743. Arguments:
  744. VirtualAddress - Supplies the virtual address to make valid.
  745. CurrentProcess - Supplies a pointer to the current process.
  746. Return Value:
  747. Returns TRUE if lock released and wait performed, FALSE otherwise.
  748. Environment:
  749. Kernel mode.
  750. --*/
  751. {
  752. PMMPTE PointerPte;
  753. PMMPFN Pfn1;
  754. KIRQL OldIrql;
  755. ULONG Waited;
  756. //
  757. // Initializing OldIrql is not needed for correctness, but without it
  758. // the compiler cannot compile this code W4 to check for use of
  759. // uninitialized variables.
  760. //
  761. OldIrql = PASSIVE_LEVEL;
  762. Waited = FALSE;
  763. PointerPte = MiGetPteAddress(VirtualAddress);
  764. //
  765. // The address must be within paged pool.
  766. //
  767. if (PfnLockHeld == FALSE) {
  768. LOCK_PFN2 (OldIrql);
  769. }
  770. if (PointerPte->u.Hard.Valid == 0) {
  771. Waited = MiMakeSystemAddressValidPfn (
  772. MiGetVirtualAddressMappedByPte(PointerPte));
  773. }
  774. Pfn1 = MI_PFN_ELEMENT (PointerPte->u.Hard.PageFrameNumber);
  775. MI_ADD_LOCKED_PAGE_CHARGE(Pfn1, 6);
  776. Pfn1->u3.e2.ReferenceCount += 1;
  777. if (PfnLockHeld == FALSE) {
  778. UNLOCK_PFN2 (OldIrql);
  779. }
  780. return Waited;
  781. }
  782. VOID
  783. FASTCALL
  784. MiUnlockPagedAddress (
  785. IN PVOID VirtualAddress,
  786. IN ULONG PfnLockHeld
  787. )
  788. /*++
  789. Routine Description:
  790. This routine checks to see if the virtual address is valid, and if
  791. not makes it valid.
  792. Arguments:
  793. VirtualAddress - Supplies the virtual address to make valid.
  794. Return Value:
  795. None.
  796. Environment:
  797. Kernel mode. PFN LOCK MUST NOT BE HELD.
  798. --*/
  799. {
  800. PMMPFN Pfn1;
  801. PMMPTE PointerPte;
  802. KIRQL OldIrql;
  803. PFN_NUMBER PageFrameIndex;
  804. PointerPte = MiGetPteAddress(VirtualAddress);
  805. //
  806. // Initializing OldIrql is not needed for correctness, but without it
  807. // the compiler cannot compile this code W4 to check for use of
  808. // uninitialized variables.
  809. //
  810. OldIrql = PASSIVE_LEVEL;
  811. //
  812. // Address must be within paged pool.
  813. //
  814. if (PfnLockHeld == FALSE) {
  815. LOCK_PFN2 (OldIrql);
  816. }
  817. ASSERT (PointerPte->u.Hard.Valid == 1);
  818. PageFrameIndex = MI_GET_PAGE_FRAME_FROM_PTE (PointerPte);
  819. Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
  820. ASSERT (Pfn1->u3.e2.ReferenceCount > 1);
  821. MI_REMOVE_LOCKED_PAGE_CHARGE_AND_DECREF(Pfn1, 7);
  822. if (PfnLockHeld == FALSE) {
  823. UNLOCK_PFN2 (OldIrql);
  824. }
  825. return;
  826. }
  827. VOID
  828. FASTCALL
  829. MiZeroPhysicalPage (
  830. IN PFN_NUMBER PageFrameIndex,
  831. IN ULONG PageColor
  832. )
  833. /*++
  834. Routine Description:
  835. This procedure maps the specified physical page into hyper space
  836. and fills the page with zeros.
  837. Arguments:
  838. PageFrameIndex - Supplies the physical page number to fill with zeroes.
  839. Return Value:
  840. None.
  841. Environment:
  842. Kernel mode.
  843. --*/
  844. {
  845. KIRQL OldIrql;
  846. PVOID VirtualAddress;
  847. PEPROCESS Process;
  848. UNREFERENCED_PARAMETER (PageColor);
  849. Process = PsGetCurrentProcess ();
  850. VirtualAddress = MiMapPageInHyperSpace (Process, PageFrameIndex, &OldIrql);
  851. KeZeroPage (VirtualAddress);
  852. MiUnmapPageInHyperSpace (Process, VirtualAddress, OldIrql);
  853. return;
  854. }
  855. VOID
  856. FASTCALL
  857. MiRestoreTransitionPte (
  858. IN PFN_NUMBER PageFrameIndex
  859. )
  860. /*++
  861. Routine Description:
  862. This procedure restores the original contents into the PTE (which could
  863. be a prototype PTE) referred to by the PFN database for the specified
  864. physical page. It also updates all necessary data structures to
  865. reflect the fact that the referenced PTE is no longer in transition.
  866. The physical address of the referenced PTE is mapped into hyper space
  867. of the current process and the PTE is then updated.
  868. Arguments:
  869. PageFrameIndex - Supplies the physical page number which refers to a
  870. transition PTE.
  871. Return Value:
  872. none.
  873. Environment:
  874. Must be holding the PFN database lock with APCs disabled.
  875. --*/
  876. {
  877. PMMPFN Pfn1;
  878. PMMPFN Pfn2;
  879. PMMPTE PointerPte;
  880. PSUBSECTION Subsection;
  881. PCONTROL_AREA ControlArea;
  882. PEPROCESS Process;
  883. PFN_NUMBER PageTableFrameIndex;
  884. Process = NULL;
  885. Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
  886. ASSERT (Pfn1->u3.e1.PageLocation == StandbyPageList);
  887. ASSERT (Pfn1->u3.e1.CacheAttribute == MiCached);
  888. if (Pfn1->u3.e1.PrototypePte) {
  889. if (MmIsAddressValid (Pfn1->PteAddress)) {
  890. PointerPte = Pfn1->PteAddress;
  891. } else {
  892. //
  893. // The page containing the prototype PTE is not valid,
  894. // map the page into hyperspace and reference it that way.
  895. //
  896. Process = PsGetCurrentProcess ();
  897. PointerPte = MiMapPageInHyperSpaceAtDpc (Process, Pfn1->u4.PteFrame);
  898. PointerPte = (PMMPTE)((PCHAR)PointerPte +
  899. MiGetByteOffset(Pfn1->PteAddress));
  900. }
  901. ASSERT ((MI_GET_PAGE_FRAME_FROM_TRANSITION_PTE (PointerPte) == PageFrameIndex) &&
  902. (PointerPte->u.Hard.Valid == 0));
  903. //
  904. // This page is referenced by a prototype PTE. The
  905. // segment structures need to be updated when the page
  906. // is removed from the transition state.
  907. //
  908. if (Pfn1->OriginalPte.u.Soft.Prototype) {
  909. //
  910. // The prototype PTE is in subsection format, calculate the
  911. // address of the control area for the subsection and decrement
  912. // the number of PFN references to the control area.
  913. //
  914. // Calculate address of subsection for this prototype PTE.
  915. //
  916. Subsection = MiGetSubsectionAddress (&Pfn1->OriginalPte);
  917. ControlArea = Subsection->ControlArea;
  918. ControlArea->NumberOfPfnReferences -= 1;
  919. ASSERT ((LONG)ControlArea->NumberOfPfnReferences >= 0);
  920. MiCheckForControlAreaDeletion (ControlArea);
  921. }
  922. } else {
  923. //
  924. // The page points to a page or page table page which may not be
  925. // for the current process. Map the page into hyperspace and
  926. // reference it through hyperspace. If the page resides in
  927. // system space (but not session space), it does not need to be
  928. // mapped as all PTEs for system space must be resident. Session
  929. // space PTEs are only mapped per session so access to them must
  930. // also go through hyperspace.
  931. //
  932. PointerPte = Pfn1->PteAddress;
  933. if (PointerPte < MiGetPteAddress ((PVOID)MM_SYSTEM_SPACE_START) ||
  934. MI_IS_SESSION_PTE (PointerPte)) {
  935. Process = PsGetCurrentProcess ();
  936. PointerPte = MiMapPageInHyperSpaceAtDpc (Process, Pfn1->u4.PteFrame);
  937. PointerPte = (PMMPTE)((PCHAR)PointerPte +
  938. MiGetByteOffset(Pfn1->PteAddress));
  939. }
  940. ASSERT ((MI_GET_PAGE_FRAME_FROM_TRANSITION_PTE (PointerPte) == PageFrameIndex) &&
  941. (PointerPte->u.Hard.Valid == 0));
  942. MI_CAPTURE_USED_PAGETABLE_ENTRIES (Pfn1);
  943. #if _WIN64
  944. #if DBGXX
  945. MiCheckPageTableTrim(PointerPte);
  946. #endif
  947. #endif
  948. }
  949. ASSERT (Pfn1->OriginalPte.u.Hard.Valid == 0);
  950. ASSERT (!((Pfn1->OriginalPte.u.Soft.Prototype == 0) &&
  951. (Pfn1->OriginalPte.u.Soft.Transition == 1)));
  952. MI_WRITE_INVALID_PTE (PointerPte, Pfn1->OriginalPte);
  953. if (Process != NULL) {
  954. MiUnmapPageInHyperSpaceFromDpc (Process, PointerPte);
  955. }
  956. Pfn1->u3.e1.CacheAttribute = MiNotMapped;
  957. //
  958. // The PTE has been restored to its original contents and is
  959. // no longer in transition. Decrement the share count on
  960. // the page table page which contains the PTE.
  961. //
  962. PageTableFrameIndex = Pfn1->u4.PteFrame;
  963. Pfn2 = MI_PFN_ELEMENT (PageTableFrameIndex);
  964. MiDecrementShareCountInline (Pfn2, PageTableFrameIndex);
  965. return;
  966. }
  967. PSUBSECTION
  968. MiGetSubsectionAndProtoFromPte (
  969. IN PMMPTE PointerPte,
  970. OUT PMMPTE *ProtoPte
  971. )
  972. /*++
  973. Routine Description:
  974. This routine examines the contents of the supplied PTE (which must
  975. map a page within a section) and determines the address of the
  976. subsection in which the PTE is contained.
  977. Arguments:
  978. PointerPte - Supplies a pointer to the PTE.
  979. ProtoPte - Supplies a pointer to a PMMPTE which receives the
  980. address of the prototype PTE which is mapped by the supplied
  981. PointerPte.
  982. Return Value:
  983. Returns the pointer to the subsection for this PTE.
  984. Environment:
  985. Kernel mode - Must be holding the PFN database lock and
  986. working set mutex (acquired safely) with APCs disabled.
  987. --*/
  988. {
  989. PMMPTE PointerProto;
  990. PMMPFN Pfn1;
  991. if (PointerPte->u.Hard.Valid == 1) {
  992. Pfn1 = MI_PFN_ELEMENT (PointerPte->u.Hard.PageFrameNumber);
  993. *ProtoPte = Pfn1->PteAddress;
  994. return MiGetSubsectionAddress (&Pfn1->OriginalPte);
  995. }
  996. PointerProto = MiPteToProto (PointerPte);
  997. *ProtoPte = PointerProto;
  998. MiMakeSystemAddressValidPfn (PointerProto);
  999. if (PointerProto->u.Hard.Valid == 1) {
  1000. //
  1001. // Prototype Pte is valid.
  1002. //
  1003. Pfn1 = MI_PFN_ELEMENT (PointerProto->u.Hard.PageFrameNumber);
  1004. return MiGetSubsectionAddress (&Pfn1->OriginalPte);
  1005. }
  1006. if ((PointerProto->u.Soft.Transition == 1) &&
  1007. (PointerProto->u.Soft.Prototype == 0)) {
  1008. //
  1009. // Prototype Pte is in transition.
  1010. //
  1011. Pfn1 = MI_PFN_ELEMENT (PointerProto->u.Trans.PageFrameNumber);
  1012. return MiGetSubsectionAddress (&Pfn1->OriginalPte);
  1013. }
  1014. ASSERT (PointerProto->u.Soft.Prototype == 1);
  1015. return MiGetSubsectionAddress (PointerProto);
  1016. }
  1017. BOOLEAN
  1018. MmIsNonPagedSystemAddressValid (
  1019. IN PVOID VirtualAddress
  1020. )
  1021. /*++
  1022. Routine Description:
  1023. For a given virtual address this function returns TRUE if the address
  1024. is within the nonpagable portion of the system's address space,
  1025. FALSE otherwise.
  1026. Arguments:
  1027. VirtualAddress - Supplies the virtual address to check.
  1028. Return Value:
  1029. TRUE if the address is within the nonpagable portion of the system
  1030. address space, FALSE otherwise.
  1031. Environment:
  1032. Kernel mode.
  1033. --*/
  1034. {
  1035. //
  1036. // Return TRUE if address is within the nonpagable portion
  1037. // of the system. Check limits for paged pool and if not within
  1038. // those limits, return TRUE.
  1039. //
  1040. if ((VirtualAddress >= MmPagedPoolStart) &&
  1041. (VirtualAddress <= MmPagedPoolEnd)) {
  1042. return FALSE;
  1043. }
  1044. //
  1045. // Check special pool before checking session space because on NT64
  1046. // nonpaged session pool exists in session space (on NT32, nonpaged
  1047. // session requests are satisfied from systemwide nonpaged pool instead).
  1048. //
  1049. if (MmIsSpecialPoolAddress (VirtualAddress)) {
  1050. if (MiIsSpecialPoolAddressNonPaged (VirtualAddress)) {
  1051. return TRUE;
  1052. }
  1053. return FALSE;
  1054. }
  1055. if ((VirtualAddress >= (PVOID) MmSessionBase) &&
  1056. (VirtualAddress < (PVOID) MiSessionSpaceEnd)) {
  1057. return FALSE;
  1058. }
  1059. return TRUE;
  1060. }
  1061. VOID
  1062. MmHibernateInformation (
  1063. IN PVOID MemoryMap,
  1064. OUT PULONG_PTR HiberVa,
  1065. OUT PPHYSICAL_ADDRESS HiberPte
  1066. )
  1067. {
  1068. //
  1069. // Mark PTE page where the 16 dump PTEs reside as needing cloning.
  1070. //
  1071. PoSetHiberRange (MemoryMap, PO_MEM_CLONE, MmCrashDumpPte, 1, ' etP');
  1072. //
  1073. // Return the dump PTEs to the loader (as it needs to use them
  1074. // to map it's relocation code into the kernel space on the
  1075. // final bit of restoring memory).
  1076. //
  1077. *HiberVa = (ULONG_PTR) MiGetVirtualAddressMappedByPte(MmCrashDumpPte);
  1078. *HiberPte = MmGetPhysicalAddress(MmCrashDumpPte);
  1079. }
  1080. #if _WIN64
  1081. #if DBGXX
  1082. ULONG zok[16];
  1083. VOID
  1084. MiCheckPageTableTrim(
  1085. IN PMMPTE PointerPte
  1086. )
  1087. {
  1088. ULONG i;
  1089. PFN_NUMBER Frame;
  1090. PMMPFN Pfn;
  1091. PMMPTE FrameData;
  1092. PMMPTE p;
  1093. ULONG count;
  1094. Frame = MI_GET_PAGE_FRAME_FROM_PTE(PointerPte);
  1095. Pfn = MI_PFN_ELEMENT (Frame);
  1096. if (Pfn->UsedPageTableEntries) {
  1097. count = 0;
  1098. p = FrameData = (PMMPTE)KSEG_ADDRESS (Frame);
  1099. for (i = 0; i < PTE_PER_PAGE; i += 1, p += 1) {
  1100. if (p->u.Long != 0) {
  1101. count += 1;
  1102. }
  1103. }
  1104. DbgPrint ("MiCheckPageTableTrim: %I64X %I64X %I64X\n",
  1105. PointerPte, Pfn, Pfn->UsedPageTableEntries);
  1106. if (count != Pfn->UsedPageTableEntries) {
  1107. DbgPrint ("MiCheckPageTableTrim1: %I64X %I64X %I64X %I64X\n",
  1108. PointerPte, Pfn, Pfn->UsedPageTableEntries, count);
  1109. DbgBreakPoint();
  1110. }
  1111. zok[0] += 1;
  1112. }
  1113. else {
  1114. zok[1] += 1;
  1115. }
  1116. }
  1117. VOID
  1118. MiCheckPageTableInPage(
  1119. IN PMMPFN Pfn,
  1120. IN PMMINPAGE_SUPPORT Support
  1121. )
  1122. {
  1123. ULONG i;
  1124. PFN_NUMBER Frame;
  1125. PMMPTE FrameData;
  1126. PMMPTE p;
  1127. ULONG count;
  1128. if (Support->UsedPageTableEntries) {
  1129. Frame = (PFN_NUMBER)((PMMPFN)Pfn - (PMMPFN)MmPfnDatabase);
  1130. count = 0;
  1131. p = FrameData = (PMMPTE)KSEG_ADDRESS (Frame);
  1132. for (i = 0; i < PTE_PER_PAGE; i += 1, p += 1) {
  1133. if (p->u.Long != 0) {
  1134. count += 1;
  1135. }
  1136. }
  1137. DbgPrint ("MiCheckPageTableIn: %I64X %I64X %I64X\n",
  1138. FrameData, Pfn, Support->UsedPageTableEntries);
  1139. if (count != Support->UsedPageTableEntries) {
  1140. DbgPrint ("MiCheckPageTableIn1: %I64X %I64X %I64X %I64X\n",
  1141. FrameData, Pfn, Support->UsedPageTableEntries, count);
  1142. DbgBreakPoint();
  1143. }
  1144. zok[2] += 1;
  1145. }
  1146. else {
  1147. zok[3] += 1;
  1148. }
  1149. }
  1150. #endif
  1151. #endif