windows-xp/Source/XPSP1/NT/base/ntos/ke/i386/largepag.c

#include "ki.h"

PVOID
Ki386AllocateContiguousMemory(
    IN OUT PIDENTITY_MAP IdentityMap,
    IN     ULONG Pages,
    IN     BOOLEAN Low4Meg
    );

BOOLEAN
Ki386IdentityMapMakeValid(
    IN OUT PIDENTITY_MAP IdentityMap,
    IN     PHARDWARE_PTE PageTableEntry,
    OUT    PVOID *Page OPTIONAL
    );

BOOLEAN
Ki386MapAddress(
    IN OUT PIDENTITY_MAP IdentityMap,
    IN     ULONG Va,
    IN     PHYSICAL_ADDRESS PhysicalAddress
    );

PVOID
Ki386ConvertPte(
    IN OUT PHARDWARE_PTE Pte
    );

PHYSICAL_ADDRESS
Ki386BuildIdentityBuffer(
    IN OUT PIDENTITY_MAP IdentityMap,
    IN     PVOID StartVa,
    IN     ULONG Length,
    OUT    PULONG PagesToMap
    );

#ifdef ALLOC_PRAGMA

#pragma alloc_text(INIT,Ki386AllocateContiguousMemory)
#pragma alloc_text(INIT,Ki386BuildIdentityBuffer)
#pragma alloc_text(INIT,Ki386ClearIdentityMap)
#pragma alloc_text(INIT,Ki386ConvertPte)
#pragma alloc_text(INIT,Ki386CreateIdentityMap)
#pragma alloc_text(INIT,Ki386EnableTargetLargePage)
#pragma alloc_text(INIT,Ki386IdentityMapMakeValid)
#pragma alloc_text(INIT,Ki386MapAddress)

#endif

#define PTES_PER_PAGE (PAGE_SIZE / sizeof(HARDWARE_PTE))

BOOLEAN
Ki386CreateIdentityMap(
    IN OUT PIDENTITY_MAP IdentityMap,
    IN     PVOID StartVa,
    IN     PVOID EndVa
    )
{
/*++

    This function creates an identity mapping for a region of memory.

    If the region of memory passed in includes memory that lies above
    4G, then a new buffer is allocated below 4G.

Arguments:

    IdentityMap - Pointer to the structure which will be filled with the newly
                  created top-level directory address.  It also provides
                  storage for the pointers used in alloating and freeing the
                  memory.

    StartVa - Pointer to the first byte of the region of memory that is to be
              memory mapped.

    EndVa - Pointer to the byte immediately after the last byte of the region
            that is to be memory mapped.

Return Value:

    TRUE if the function succeeds, FALSE otherwise.

    Note - Ki386ClearIdentityMap() should be called even on FALSE return to
    free any memory allocated.

--*/

    ULONG pageDirectoryIndex;
    ULONG pagesToMap;
    PCHAR currentVa;
    ULONG length;
    BOOLEAN result;
    PHARDWARE_PTE pageDirectory;
    PHARDWARE_PTE pageDirectoryEntry;
    PHYSICAL_ADDRESS identityAddress;

#if defined(_X86PAE_)

    ULONG pageDirectoryPointerTableIndex;
    PHARDWARE_PTE pageDirectoryPointerTable;
    PHARDWARE_PTE pageDirectoryPointerTableEntry;

#endif

    //
    // Initialize the IdentityMap structure to a known state.
    //

    RtlZeroMemory( IdentityMap, sizeof(IDENTITY_MAP) );
    length = (PCHAR)EndVa - (PCHAR)StartVa;

    //
    // Get the physical address of the input buffer (or suitable copy).
    //

    identityAddress = Ki386BuildIdentityBuffer( IdentityMap,
                                                StartVa,
                                                length,
                                                &pagesToMap );
    if( identityAddress.QuadPart == 0) {

        //
        // The input buffer was not contiguous or not below 4G, and a
        // suitable buffer could not be allocated.
        //

        return FALSE;
    }

    IdentityMap->IdentityAddr = identityAddress.LowPart;

    //
    // Set up the mappings.
    //

    currentVa = StartVa;
    do {

        //
        // Map in the virtual address
        //

        result = Ki386MapAddress( IdentityMap,
                                  (ULONG)currentVa,
                                  identityAddress );
        if (result == FALSE) {
            return FALSE;
        }

        //
        // Map in the identity (physical) address
        //

        result = Ki386MapAddress( IdentityMap,
                                  identityAddress.LowPart,
                                  identityAddress );
        if (result == FALSE) {
            return FALSE;
        }

        //
        // Advance both the Va and identityAddress pointers in anticipation
        // of mapping in another page.
        //

        currentVa += PAGE_SIZE;
        identityAddress.QuadPart += PAGE_SIZE;
        pagesToMap -= 1;

    } while (pagesToMap > 0);

    //
    // Now go through the page directory pointer table and page directories,
    // converting virtual page frames to physical ones.
    //

#if defined(_X86PAE_)

    //
    // This PAE-only outer loop walks the page directory pointer table entries
    // and processes each valid page directory referenced.
    //

    pageDirectoryPointerTable = IdentityMap->TopLevelDirectory;
    for (pageDirectoryPointerTableIndex = 0;
         pageDirectoryPointerTableIndex < (1 << PPI_BITS);
         pageDirectoryPointerTableIndex++) {

        pageDirectoryPointerTableEntry =
            &pageDirectoryPointerTable[ pageDirectoryPointerTableIndex ];

        if (pageDirectoryPointerTableEntry->Valid == 0) {
            continue;
        }

        pageDirectory =
            (PHARDWARE_PTE)Ki386ConvertPte( pageDirectoryPointerTableEntry );

#else
        pageDirectory = IdentityMap->TopLevelDirectory;
#endif

        for (pageDirectoryIndex = 0;
             pageDirectoryIndex < PTES_PER_PAGE;
             pageDirectoryIndex++) {

            pageDirectoryEntry = &pageDirectory[ pageDirectoryIndex ];
            if (pageDirectoryEntry->Valid == 0) {
                continue;
            }

            Ki386ConvertPte( pageDirectoryEntry );
        }

#if defined(_X86PAE_)
    }
#endif

    identityAddress = MmGetPhysicalAddress( IdentityMap->TopLevelDirectory );
    IdentityMap->IdentityCR3 = identityAddress.LowPart;

    return TRUE;
}

PVOID
Ki386AllocateContiguousMemory(
    IN OUT PIDENTITY_MAP IdentityMap,
    IN     ULONG Pages,
    IN     BOOLEAN Low4Meg
    )
/*++

    This function allocates page-aligned, physically contiguous memory.
    The allocation is recorded in the IdentityMap structure, so that it
    can be freed on cleanup.

Arguments:

    IdentityMap - Context pointer for this identity mapping.

    Pages - Number of pages to allocate

    Low4Meg - Indicates whether the allocation must be below 4M.

Return Value:

    Pointer to the new page on success, NULL otherwise.

--*/
{
    ULONG pageListIndex;
    PVOID page;
    ULONG allocationSize;
    PHYSICAL_ADDRESS highestAddress;

    if (Low4Meg != FALSE) {

        //
        // The caller has specified that a page must reside physically
        // below 4 MB.
        //

        highestAddress.LowPart = 0xFFFFFFFF;
        highestAddress.HighPart = 0;

    } else {

        //
        // Memory can reside anywhere
        //

        highestAddress.LowPart = 0xFFFFFFFF;
        highestAddress.HighPart = 0xFFFFFFFF;
    }

    allocationSize = Pages * PAGE_SIZE;
    page = MmAllocateContiguousMemory( allocationSize, highestAddress );
    if (page != NULL) {

        //
        // Record that this page was allocated so that it can be freed when
        // the IdentityMap structure is cleared.
        //

        pageListIndex = IdentityMap->PagesAllocated;
        IdentityMap->PageList[ pageListIndex ] = page;
        IdentityMap->PagesAllocated++;

        //
        // Initialize it.
        //

        RtlZeroMemory( page, allocationSize );
    }

    return page;
}

BOOLEAN
Ki386IdentityMapMakeValid(
    IN OUT PIDENTITY_MAP IdentityMap,
    IN     PHARDWARE_PTE PageTableEntry,
    OUT    PVOID *Page OPTIONAL
    )
/*++

    If the page table has the valid bit set, this function merely returns
    the address referenced by the page table entry.

    If the page table does not have the valid bit set, then another page
    is allocated and inserted into the page table entry and the entry is
    marked valid.

    NOTE: At this point, PTE frames are virtual.  After the entire mapping
          is built, we go through and convert all virtual frames to physical
          ones.

Arguments:

    IdentityMap - Context pointer for this identity mapping.

    PageTableEntry - Pointer to the page table entry.

    Page - Virtual address now referenced by the PTE, whether it was
           valid before or not.

Return Value:

    TRUE on success, FALSE otherwise.

--*/
{
    PVOID page;

    if (PageTableEntry->Valid != 0) {

        //
        // If it already is present, there is nothing to do except record
        // the virtual page number that is already there.
        //

        page = (PVOID)((ULONG)(PageTableEntry->PageFrameNumber << PAGE_SHIFT));

    } else {

        //
        // The page table entry is not valid.  Allocate a new page table.
        //

        page = Ki386AllocateContiguousMemory( IdentityMap, 1, FALSE );
        if (page == NULL) {
            return FALSE;
        }

        //
        // Insert it into the page table entry and mark it valid.
        //
        // NOTE: Virtual page numbers are inserted into the page table
        //       structure as it is being built.  When it is finished, we walk
        //       the tables and convert all of the virtual page numbers to
        //       physical page numbers.
        //

        PageTableEntry->PageFrameNumber = ((ULONG)page) >> PAGE_SHIFT;
        PageTableEntry->Valid = 1;
    }

    if (ARGUMENT_PRESENT( Page )) {
        *Page = page;
    }

    return TRUE;
}

BOOLEAN
Ki386MapAddress(
    IN OUT PIDENTITY_MAP IdentityMap,
    IN     ULONG Va,
    IN     PHYSICAL_ADDRESS PhysicalAddress
    )

/*++

    Creates a new virtual->physical mapping in the identity map.

Arguments:

    IdentityMap - Context pointer for this identity mapping.

    Va - Virtual address to map.

    PhysicalAddress - Physical address to map.

Return Value:

    TRUE on success, FALSE otherwise.

--*/
{
    PHARDWARE_PTE pageTable;
    PHARDWARE_PTE pageTableEntry;
    PHARDWARE_PTE pageDirectory;
    PHARDWARE_PTE pageDirectoryEntry;
    PVOID table;
    ULONG index;
    BOOLEAN result;

#if defined(_X86PAE_)
    PHARDWARE_PTE pageDirectoryPointerTable;
    PHARDWARE_PTE pageDirectoryPointerTableEntry;
#endif

    if (IdentityMap->TopLevelDirectory == NULL) {

        //
        // Allocate a top-level directory structure, either a page directory
        // or a page directory pointer table.
        //

        table = Ki386AllocateContiguousMemory( IdentityMap, 1, TRUE );
        if (table == FALSE) {
            return FALSE;
        }

        IdentityMap->TopLevelDirectory = table;
    }

#if defined(_X86PAE_)

    index = KiGetPpeIndex( Va );
    pageDirectoryPointerTable = IdentityMap->TopLevelDirectory;
    pageDirectoryPointerTableEntry = &pageDirectoryPointerTable[ index ];

    result = Ki386IdentityMapMakeValid( IdentityMap,
                                        pageDirectoryPointerTableEntry,
                                        &pageDirectory );
    if (result == FALSE) {
        return FALSE;
    }

#else

    pageDirectory = IdentityMap->TopLevelDirectory;

#endif

    //
    // Get a pointer to the appropriate page directory entry.  If it is
    // not valid, allocate a new page table and mark the page directory
    // entry valid and writeable.
    //

    index = KiGetPdeIndex( Va );
    pageDirectoryEntry = &pageDirectory[ index ];
    result = Ki386IdentityMapMakeValid( IdentityMap,
                                        pageDirectoryEntry,
                                        &pageTable );
    if (result == FALSE) {
        return FALSE;
    }
    pageDirectoryEntry->Write = 1;

    //
    // Get a pointer to the appropriate page table entry and fill it in.
    //

    index = KiGetPteIndex( Va );
    pageTableEntry = &pageTable[ index ];

#if defined(_X86PAE_)
    pageTableEntry->PageFrameNumber = PhysicalAddress.QuadPart >> PAGE_SHIFT;
#else
    pageTableEntry->PageFrameNumber = PhysicalAddress.LowPart >> PAGE_SHIFT;
#endif
    pageTableEntry->Valid = 1;

    return TRUE;
}

PVOID
Ki386ConvertPte(
    IN OUT PHARDWARE_PTE Pte
    )
/*++

    Converts the virtual frame number in a PTE to a physical frame number.

Arguments:

    Pte - Pointer to the page table entry to convert.

Return Value:

    None.

--*/
{
    PVOID va;
    PHYSICAL_ADDRESS physicalAddress;

    va = (PVOID)(Pte->PageFrameNumber << PAGE_SHIFT);
    physicalAddress = MmGetPhysicalAddress( va );

#if defined(_X86PAE_)
    Pte->PageFrameNumber = physicalAddress.QuadPart >> PAGE_SHIFT;
#else
    Pte->PageFrameNumber = physicalAddress.LowPart >> PAGE_SHIFT;
#endif

    return va;
}

PHYSICAL_ADDRESS
Ki386BuildIdentityBuffer(
    IN OUT PIDENTITY_MAP IdentityMap,
    IN     PVOID StartVa,
    IN     ULONG Length,
    OUT    PULONG PagesToMap
    )
{

/*++

    This function checks to see if the physical memory backing a virtual
    buffer is physically contiguous and lies completely below 4G.

    If these requirements are met, then the physical address of StartVa is
    returned.

    If not, then a physically contiguous buffer is allocated, the contents
    of the region is copied in, and its address is returned.

Arguments:

    IdentityMap - Pointer to the identity map building structure.

    StartVa - Virtual address of the start of the region for which a
              physically contiguous copy is desired.

    Length - Length of the region for which a physically contiguous copy
             is desired.

--*/

    ULONG pagesToMap;
    ULONG pagesRemaining;
    PCHAR nextVirtualAddress;
    PHYSICAL_ADDRESS nextPhysicalAddress;
    PHYSICAL_ADDRESS physicalAddress;
    PHYSICAL_ADDRESS firstPhysicalAddress;
    ULONG pageOffset;
    PCHAR identityBuffer;

    //
    // Count the number of pages in the buffer, and record the physical
    // address of the start of the buffer.
    //

    pagesToMap = ADDRESS_AND_SIZE_TO_SPAN_PAGES( StartVa, Length );
    nextVirtualAddress = StartVa;
    firstPhysicalAddress = MmGetPhysicalAddress( StartVa );
    nextPhysicalAddress = firstPhysicalAddress;

    //
    // Examine each page in the region.
    //

    pagesRemaining = pagesToMap;
    while (TRUE) {

        physicalAddress = MmGetPhysicalAddress( nextVirtualAddress );
        if (physicalAddress.QuadPart != nextPhysicalAddress.QuadPart) {

            //
            // The buffer is not physically contiguous.
            //

            break;
        }

        if (physicalAddress.HighPart != 0) {

            //
            // The buffer does not lie entirely below 4G
            //

            break;
        }

        pagesRemaining -= 1;
        if (pagesRemaining == 0) {

            //
            // All of the pages in the buffer have been examined, and have
            // been found to meet the critera.  Return the physical address
            // of the start of the buffer.
            //

            *PagesToMap = pagesToMap;
            return firstPhysicalAddress;
        }

        nextVirtualAddress += PAGE_SIZE;
        nextPhysicalAddress.QuadPart += PAGE_SIZE;
    }

    //
    // The buffer does not meet the criteria and so its contents must be
    // copied to a buffer that does.
    //

    identityBuffer = Ki386AllocateContiguousMemory( IdentityMap,
                                                    pagesToMap,
                                                    TRUE );
    if (identityBuffer == 0) {

        //
        // A contiguous region of the appropriate size could not be located
        // below 4G physical.
        //

        physicalAddress.QuadPart = 0;

    } else {

        //
        // Got an appropriate physical buffer, now copy in the data
        //

        pageOffset = (ULONG)StartVa & (PAGE_SIZE-1);
        identityBuffer += pageOffset;

        RtlCopyMemory( identityBuffer, StartVa, Length );
        physicalAddress = MmGetPhysicalAddress( identityBuffer );

        *PagesToMap = pagesToMap;
    }

    return physicalAddress;
}



VOID
Ki386ClearIdentityMap(
    IN PIDENTITY_MAP IdentityMap
    )
{
/*++

    This function just frees the page directory and page tables created in
    Ki386CreateIdentityMap().

--*/

    ULONG index;
    PVOID page;

    //
    // IdentityMap->PageList is an array of addresses of pages allocated with
    // MmAllocateContiguousMemory().  Walk the array, freeing each page.
    //

    for (index = 0; index < IdentityMap->PagesAllocated; index++) {

        page = IdentityMap->PageList[ index ];
        MmFreeContiguousMemory( page );
    }
}

VOID
Ki386EnableTargetLargePage(
    IN PIDENTITY_MAP IdentityMap
    )
{
/*++

    This function just passes info on to the assembly routine
    Ki386EnableLargePage().

--*/

    Ki386EnableCurrentLargePage(IdentityMap->IdentityAddr,
                                IdentityMap->IdentityCR3);
}