windows-xp/Source/XPSP1/NT/base/ntos/ke/i386/mtrr.c

/*++

Copyright (c) 1991-2000  Microsoft Corporation

Module Name:

    mtrr.c

Abstract:

    This module implements interfaces that support manipulation of
    memory type range registers.

    These entry points only exist on x86 machines.

Author:

    Ken Reneris (kenr)  11-Oct-95

Environment:

    Kernel mode only.

Revision History:

--*/

#include "ki.h"
#include "mtrr.h"

#define STATIC

#define IDBG    0

#if DBG
#define DBGMSG(a)   DbgPrint(a)
#else
#define DBGMSG(a)
#endif

//
// Internal declarations
//

//
// Range in generic terms
//

typedef struct _ONE_RANGE {
    ULONGLONG           Base;
    ULONGLONG           Limit;
    UCHAR               Type;
} ONE_RANGE, *PONE_RANGE;

#define GROW_RANGE_TABLE    4

//
// Range in specific mtrr terms
//

typedef struct _MTRR_RANGE {
    MTRR_VARIABLE_BASE  Base;
    MTRR_VARIABLE_MASK  Mask;
} MTRR_RANGE, *PMTRR_RANGE;

//
// System static information concerning cached range types
//

typedef struct _RANGE_INFO {

    //
    // Global MTRR info
    //

    MTRR_DEFAULT        Default;            // h/w mtrr default
    MTRR_CAPABILITIES   Capabilities;       // h/w mtrr Capabilities
    UCHAR               DefaultCachedType;  // default type for MmCached

    //
    // Variable MTRR information
    //

    BOOLEAN             RangesValid;        // Ranges initialized and valid.
    BOOLEAN             MtrrWorkaround;     // Work Around needed/not.
    UCHAR               NoRange;            // No ranges currently in Ranges
    UCHAR               MaxRange;           // Max size of Ranges
    PONE_RANGE          Ranges;             // Current ranges as set into h/w

} RANGE_INFO, *PRANGE_INFO;


//
// Structure used while processing range database
//

typedef struct _NEW_RANGE {
    //
    // Current Status
    //

    NTSTATUS            Status;

    //
    // Generic info on new range
    //

    ULONGLONG           Base;
    ULONGLONG           Limit;
    UCHAR               Type;

    //
    // MTRR image to be set into h/w
    //

    PMTRR_RANGE         MTRR;

    //
    // RangeDatabase before edits were started
    //

    UCHAR               NoRange;
    PONE_RANGE          Ranges;

    //
    // IPI context to coordinate concurrent processor update
    //

    ULONG               NoMTRR;

    PROCESSOR_LOCKSTEP  Synchronize;
    ULONG               Processor;
} NEW_RANGE, *PNEW_RANGE;

//
// Prototypes
//

VOID
KiInitializeMTRR (
    IN BOOLEAN LastProcessor
    );

BOOLEAN
KiRemoveRange (
    IN PNEW_RANGE   NewRange,
    IN ULONGLONG    Base,
    IN ULONGLONG    Limit,
    IN PBOOLEAN     RemoveThisType
    );

VOID
KiAddRange (
    IN PNEW_RANGE   NewRange,
    IN ULONGLONG    Base,
    IN ULONGLONG    Limit,
    IN UCHAR        Type
    );

VOID
KiStartEffectiveRangeChange (
    IN PNEW_RANGE   NewRange
    );

VOID
KiCompleteEffectiveRangeChange (
    IN PNEW_RANGE   NewRange
    );

STATIC ULONG
KiRangeWeight (
    IN PONE_RANGE   Range
    );

STATIC ULONG
KiFindFirstSetLeftBit (
    IN ULONGLONG    Set
    );

STATIC ULONG
KiFindFirstSetRightBit (
    IN ULONGLONG    Set
    );

VOID
KiLoadMTRRTarget (
    IN PKIPI_CONTEXT SignalDone,
    IN PVOID Context,
    IN PVOID Parameter2,
    IN PVOID Parameter3
    );

NTSTATUS
KiLoadMTRR (
    IN PNEW_RANGE Context
    );

ULONGLONG
KiMaskToLength (
    IN ULONGLONG    Mask
    );

ULONGLONG
KiLengthToMask (
    IN ULONGLONG    Length
    );

#if IDBG
VOID
KiDumpMTRR (
    PUCHAR      DebugString,
    PMTRR_RANGE MTRR
    );
#endif

//
// --- AMD - Prototypes for AMD K6 MTRR Support functions. ---
//

NTSTATUS
KiAmdK6MtrrSetMemoryType (
    IN ULONG BaseAddress,
    IN ULONG NumberOfBytes,
    IN MEMORY_CACHING_TYPE CacheType
    );

VOID
KiAmdK6MtrrWRMSR (
    VOID
    );

// --- AMD - End ---

#ifdef ALLOC_PRAGMA
#pragma alloc_text(INIT,KiInitializeMTRR)
#pragma alloc_text(PAGELK,KiRemoveRange)
#pragma alloc_text(PAGELK,KiAddRange)
#pragma alloc_text(PAGELK,KiStartEffectiveRangeChange)
#pragma alloc_text(PAGELK,KiCompleteEffectiveRangeChange)
#pragma alloc_text(PAGELK,KiRangeWeight)
#pragma alloc_text(PAGELK,KiFindFirstSetLeftBit)
#pragma alloc_text(PAGELK,KiFindFirstSetRightBit)
#pragma alloc_text(PAGELK,KiLoadMTRR)
#pragma alloc_text(PAGELK,KiLoadMTRRTarget)
#pragma alloc_text(PAGELK,KiLockStepExecution)
#pragma alloc_text(PAGELK,KiLengthToMask)
#pragma alloc_text(PAGELK,KiMaskToLength)

#if IDBG
#pragma alloc_text(PAGELK,KiDumpMTRR)
#endif

#endif

//
// KiRangeLock - Used to synchronize accesses to KiRangeInfo
//

KSPIN_LOCK          KiRangeLock;

//
// KiRangeInfo - Range type mapping information.  Details specific h/w support
//               and contains the current range database of how physical
//               addresses have been set

RANGE_INFO          KiRangeInfo;

VOID
KiInitializeMTRR (
    IN BOOLEAN LastProcessor
    )
/*++

Routine Description:

    Called to incrementally initialize the physical range
    database feature.   First processor's MTRR set is read into the
    physical range database.

Arguments:

    LastProcessor - If set this is the last processor to execute this routine
    such that when this processor finishes, the initialization is complete.

Return Value:

    None - if there was a problem the function
    KeSetPhysicalCacheTypeRange type is disabled.

--*/
{
    BOOLEAN             Status;
    ULONG               Index, Size;
    MTRR_DEFAULT        Default;
    MTRR_CAPABILITIES   Capabilities;
    NEW_RANGE           NewRange;
    MTRR_VARIABLE_BASE  MtrrBase;
    MTRR_VARIABLE_MASK  MtrrMask;
    ULONGLONG           Base, Mask, Length;
    BOOLEAN             RemoveThisType[MTRR_TYPE_MAX];
    NTSTATUS            NtStatus;
    PKPRCB              Prcb;

    Status = TRUE;
    RtlZeroMemory (&NewRange, sizeof (NewRange));
    NewRange.Status = STATUS_UNSUCCESSFUL;

    //
    // If this is the first processor, initialize some fields
    //

    if (KeGetPcr()->Number == 0) {
        KeInitializeSpinLock (&KiRangeLock);

        KiRangeInfo.Capabilities.u.QuadPart = RDMSR(MTRR_MSR_CAPABILITIES);
        KiRangeInfo.Default.u.QuadPart = RDMSR(MTRR_MSR_DEFAULT);
        KiRangeInfo.DefaultCachedType = MTRR_TYPE_MAX;

        //
        // If h/w mtrr support is not enabled, disable OS support
        //

        if (!KiRangeInfo.Default.u.hw.MtrrEnabled ||
            KiRangeInfo.Capabilities.u.hw.VarCnt == 0 ||
            KiRangeInfo.Default.u.hw.Type != MTRR_TYPE_UC) {

            DBGMSG("MTRR feature disabled.\n");
            Status = FALSE;

        } else {

            //
            // If USWC type is supported by hardware, but the MTRR
            // feature is not set in KeFeatureBits, it is because
            // the HAL indicated USWC should not be used on this
            // machine.  (Possibly due to shared memory clusters).
            //

            if (KiRangeInfo.Capabilities.u.hw.UswcSupported &&
                ((KeFeatureBits & KF_MTRR) == 0)) {

                DBGMSG("KiInitializeMTRR: MTRR use globally disabled on this machine.\n");
                KiRangeInfo.Capabilities.u.hw.UswcSupported = 0;
            }

            //
            // Allocate initial range type database
            //

            KiRangeInfo.NoRange = 0;
            KiRangeInfo.MaxRange = (UCHAR) KiRangeInfo.Capabilities.u.hw.VarCnt + GROW_RANGE_TABLE;

            //
            // Don't allocate a new range on reinitialization from
            // hibernate.
            //

            if (KiRangeInfo.Ranges == NULL) {
                KiRangeInfo.Ranges = ExAllocatePoolWithTag (NonPagedPool,
                                        sizeof(ONE_RANGE) * KiRangeInfo.MaxRange,
                                        '  eK');
            }
            if (KiRangeInfo.Ranges != NULL) {
                RtlZeroMemory (KiRangeInfo.Ranges,
                               sizeof(ONE_RANGE) * KiRangeInfo.MaxRange);
            }
        }
    }

    //
    // Workaround for cpu signatures 611, 612, 616 and 617
    // - if the request for setting a variable MTRR specifies
    // an address which is not 4M aligned or length is not
    // a multiple of 4M then possible problem for INVLPG inst.
    // Detect if workaround is required
    //

    Prcb = KeGetCurrentPrcb();
    if (Prcb->CpuType == 6  &&
        (Prcb->CpuStep == 0x0101 || Prcb->CpuStep == 0x0102 ||
         Prcb->CpuStep == 0x0106 || Prcb->CpuStep == 0x0107 )) {

        if (strcmp(Prcb->VendorString, "GenuineIntel") == 0) {

            //
            // Only do this if it's an Intel part, other
            // manufacturers may have the same stepping
            // numbers but no bug.
            //

            KiRangeInfo.MtrrWorkaround = TRUE;
        }
    }

    //
    // If MTRR support disabled on first processor or if
    // buffer not allocated then fall through
    //

    if (!KiRangeInfo.Ranges){
        Status = FALSE;
    } else {

        //
        // Verify MTRR support is symmetric
        //

        Capabilities.u.QuadPart = RDMSR(MTRR_MSR_CAPABILITIES);

        if ((Capabilities.u.hw.UswcSupported) &&
            ((KeFeatureBits & KF_MTRR) == 0)) {
            DBGMSG ("KiInitializeMTRR: setting UswcSupported FALSE\n");
            Capabilities.u.hw.UswcSupported = 0;
        }

        Default.u.QuadPart = RDMSR(MTRR_MSR_DEFAULT);

        if (Default.u.QuadPart != KiRangeInfo.Default.u.QuadPart ||
            Capabilities.u.QuadPart != KiRangeInfo.Capabilities.u.QuadPart) {
            DBGMSG ("KiInitializeMTRR: asymmetric mtrr support\n");
            Status = FALSE;
        }
    }

    NewRange.Status = STATUS_SUCCESS;

    //
    // MTRR registers should be identically set on each processor.
    // Ranges should be added to the range database only for one
    // processor.
    //

    if (Status && (KeGetPcr()->Number == 0)) {
#if IDBG
        KiDumpMTRR ("Processor MTRR:", NULL);
#endif

        //
        // Read current MTRR settings for various cached range types
        // and add them to the range database
        //

        for (Index=0; Index < Capabilities.u.hw.VarCnt; Index++) {

            MtrrBase.u.QuadPart = RDMSR(MTRR_MSR_VARIABLE_BASE+Index*2);
            MtrrMask.u.QuadPart = RDMSR(MTRR_MSR_VARIABLE_MASK+Index*2);

            Mask = MtrrMask.u.QuadPart & MTRR_MASK_MASK;
            Base = MtrrBase.u.QuadPart & MTRR_MASK_BASE;

            //
            // Note - the variable MTRR Mask does NOT contain the length
            // spanned by the variable MTRR. Thus just checking the Valid
            // Bit should be sufficient for identifying a valid MTRR.
            //

            if (MtrrMask.u.hw.Valid) {

                Length = KiMaskToLength(Mask);

                //
                // Check for non-contiguous MTRR mask.
                //

                if ((Mask + Length) & MASK_OVERFLOW_MASK) {
                    DBGMSG ("KiInitializeMTRR: Found non-contiguous MTRR mask!\n");
                    Status = FALSE;
                }

                //
                // Add this MTRR to the range database
                //

                Base &= Mask;
                KiAddRange (
                    &NewRange,
                    Base,
                    Base + Length - 1,
                    (UCHAR) MtrrBase.u.hw.Type
                    );

                //
                // Check for default cache type
                //

                if (MtrrBase.u.hw.Type == MTRR_TYPE_WB) {
                    KiRangeInfo.DefaultCachedType = MTRR_TYPE_WB;
                }

                if (KiRangeInfo.DefaultCachedType == MTRR_TYPE_MAX  &&
                    MtrrBase.u.hw.Type == MTRR_TYPE_WT) {
                    KiRangeInfo.DefaultCachedType = MTRR_TYPE_WT;
                }
            }
        }

        //
        // If a default type for "cached" was not found, assume write-back
        //

        if (KiRangeInfo.DefaultCachedType == MTRR_TYPE_MAX) {
            DBGMSG ("KiInitializeMTRR: assume write-back\n");
            KiRangeInfo.DefaultCachedType = MTRR_TYPE_WB;
        }
    }

    //
    // Done
    //

    if (!NT_SUCCESS(NewRange.Status)) {
        Status = FALSE;
    }

    if (!Status) {
        DBGMSG ("KiInitializeMTRR: OS support for MTRRs disabled\n");
        if (KiRangeInfo.Ranges != NULL) {
            ExFreePool (KiRangeInfo.Ranges);
            KiRangeInfo.Ranges = NULL;
        }
    } else {

        // if last processor indicate initialization complete
        if (LastProcessor) {
            KiRangeInfo.RangesValid = TRUE;
        }
    }
}

VOID
KeRestoreMtrr (
    VOID
    )
/*++

Routine Description:

    This function reloads the MTRR registers to be the current
    known values.   This is used on a system wakeup to ensure the
    registers are sane.

    N.B. The caller must have the PAGELK code locked

Arguments:

    none

Return Value:

    none

--*/
{
    NEW_RANGE           NewRange;
    KIRQL               OldIrql;

    if (KiRangeInfo.RangesValid) {
        RtlZeroMemory (&NewRange, sizeof (NewRange));
        KeAcquireSpinLock (&KiRangeLock, &OldIrql);
        KiStartEffectiveRangeChange (&NewRange);
        ASSERT (NT_SUCCESS(NewRange.Status));
        KiCompleteEffectiveRangeChange (&NewRange);
        KeReleaseSpinLock (&KiRangeLock, OldIrql);
        return;
    }

    //
    // If the processor is a AMD K6 with MTRR support then perform
    // processor specific implentaiton.
    //

    if (KeFeatureBits & KF_AMDK6MTRR) {
        KeAcquireSpinLock (&KiRangeLock, &OldIrql);
        KiLoadMTRR(NULL);
        KeReleaseSpinLock (&KiRangeLock, OldIrql);
    }
}


NTSTATUS
KeSetPhysicalCacheTypeRange (
    IN PHYSICAL_ADDRESS PhysicalAddress,
    IN ULONG NumberOfBytes,
    IN MEMORY_CACHING_TYPE CacheType
    )
/*++

Routine Description:

    This function sets a physical range to a particular cache type.
    If the system does not support setting cache policies based on
    physical ranges, no action is taken.

Arguments:

    PhysicalAddress - The starting address of the range being set

    NumberOfBytes   - The length, in bytes, of the range being set

    CacheType       - The caching type for which the physical range is
                      to be set to.

                     NonCached:
                        Setting ranges to be NonCached is done for
                        book keeping reasons.  A return of SUCCESS when
                        setting a range NonCached does not mean it has
                        been physically set to as NonCached.  The caller
                        must use a cache-disabled virtual pointer for
                        any NonCached range.

                     Cached:
                        A successful return indicates that the physical
                        range has been set to cached.   This mode requires
                        the caller to be at irql < dispatch_level.

                     FrameBuffer:
                        A successful return indicates that the physical
                        range has been set to be framebuffer cached.
                        This mode requires the caller to be at irql <
                        dispatch_level.

                     USWCCached:
                        This type is to be satisfied only via PAT and
                        fails for the MTRR interface.

Return Value:

    STATUS_SUCCESS - if success, the cache attributes of the physical range
                     have been set.

    STATUS_NOT_SUPPORTED - either feature not supported or not yet initialized,
                           or MmWriteCombined type not supported and is
                           requested, or input range does not match restrictions
                           imposed by workarounds for current processor stepping
                           or is below 1M (in the fixed MTRR range), or not yet
                           initialized.

    STATUS_UNSUCCESSFUL - Unable to satisfy request due to
                        - Unable to map software image into limited # of
                          hardware MTRRs.
                        - irql was not < DISPATCH_LEVEL.
                        - Failure due to other internal error (out of memory).

  STATUS_INVALID_PARAMETER - Incorrect input memory type.

--*/
{
    KIRQL               OldIrql;
    NEW_RANGE           NewRange;
    BOOLEAN             RemoveThisType[MTRR_TYPE_MAX];
    BOOLEAN             EffectRangeChange, AddToRangeDatabase;

    //
    // If caller has requested the MmUSWCCached memory type then fail
    // - MmUSWCCached is supported via PAT and not otherwise
    //

    if (CacheType == MmUSWCCached) {
        return STATUS_NOT_SUPPORTED;
    }

    //
    // Addresses above 4GB, below 1MB or not page aligned and
    // page length are not supported.
    //

    if ((PhysicalAddress.HighPart != 0)               ||
        (PhysicalAddress.LowPart < (1 * 1024 * 1024)) ||
        (PhysicalAddress.LowPart & 0xfff)             ||
        (NumberOfBytes & 0xfff)                          ) {
        return STATUS_NOT_SUPPORTED;
    }

    ASSERT (NumberOfBytes != 0);

    //
    // If the processor is a AMD K6 with MTRR support then perform
    // processor specific implentaiton.
    //

    if (KeFeatureBits & KF_AMDK6MTRR) {

        if ((CacheType != MmWriteCombined) && (CacheType != MmNonCached)) {
            return STATUS_NOT_SUPPORTED;
        }

        return KiAmdK6MtrrSetMemoryType(PhysicalAddress.LowPart,
                                        NumberOfBytes,
                                        CacheType);
    }

    //
    // If processor doesn't have the memory type range feature
    // return not supported.
    //

    if (!KiRangeInfo.RangesValid) {
        return STATUS_NOT_SUPPORTED;
    }

    //
    // Workaround for cpu signatures 611, 612, 616 and 617
    // - if the request for setting a variable MTRR specifies
    // an address which is not 4M aligned or length is not
    // a multiple of 4M then return status not supported
    //

    if ((KiRangeInfo.MtrrWorkaround) &&
        ((PhysicalAddress.LowPart & 0x3fffff) ||
         (NumberOfBytes & 0x3fffff))) {

            return STATUS_NOT_SUPPORTED;
    }

    RtlZeroMemory (&NewRange, sizeof (NewRange));
    NewRange.Base  = PhysicalAddress.QuadPart;
    NewRange.Limit = NewRange.Base + NumberOfBytes - 1;

    //
    // Determine what the new mtrr range type is.   If setting NonCached then
    // the database need not be updated to reflect the virtual change.  This
    // is because non-cached virtual pointers are mapped as cache disabled.
    //

    EffectRangeChange = TRUE;
    AddToRangeDatabase = TRUE;
    switch (CacheType) {
        case MmNonCached:
            NewRange.Type = MTRR_TYPE_UC;

            //
            // NonCached ranges do not need to be reflected into the h/w state
            // as all non-cached ranges are mapped with cache-disabled pointers.
            // This also means that cache-disabled ranges do not need to
            // be put into mtrrs, or held in the range, regardless of the default
            // range type.
            //

            EffectRangeChange = FALSE;
            AddToRangeDatabase = FALSE;
            break;

        case MmCached:
            NewRange.Type = KiRangeInfo.DefaultCachedType;
            break;

        case MmWriteCombined:
            NewRange.Type = MTRR_TYPE_USWC;

            //
            // If USWC type isn't supported, then request can not be honored
            //

            if (!KiRangeInfo.Capabilities.u.hw.UswcSupported) {
                DBGMSG ("KeSetPhysicalCacheTypeRange: USWC not supported\n");
                return STATUS_NOT_SUPPORTED;
            }
            break;

        default:
            DBGMSG ("KeSetPhysicalCacheTypeRange: no such cache type\n");
            return STATUS_INVALID_PARAMETER;
            break;
    }

    NewRange.Status = STATUS_SUCCESS;

    //
    // The default type is UC thus the range is still mapped using
    // a Cache Disabled VirtualPointer and hence it need not be added.
    //

    //
    // If h/w needs updated, lock down the code required to effect the change
    //

    if (EffectRangeChange) {
        if (KeGetCurrentIrql() >= DISPATCH_LEVEL) {

            //
            // Code can not be locked down.   Supplying a new range type requires
            // that the caller calls at irql < dispatch_level.
            //

            DBGMSG ("KeSetPhysicalCacheTypeRange failed due to calling IRQL == DISPATCH_LEVEL\n");
            return STATUS_UNSUCCESSFUL;
        }

        MmLockPagableSectionByHandle(ExPageLockHandle);
    }

    //
    // Serialize the range type database
    //

    KeAcquireSpinLock (&KiRangeLock, &OldIrql);

    //
    // If h/w is going to need updated, then start an effective range change
    //

    if (EffectRangeChange) {
        KiStartEffectiveRangeChange (&NewRange);
    }

    if (NT_SUCCESS (NewRange.Status)) {

        //
        // If the new range is NonCached, then don't remove standard memory
        // caching types
        //

        memset (RemoveThisType, TRUE, MTRR_TYPE_MAX);
        if (NewRange.Type != MTRR_TYPE_UC) {
            //
            // If the requested type is uncached then the physical
            // memory region is mapped using a cache disabled virtual pointer.
            // The effective memory type for that region will be the lowest
            // common denominator of the MTRR type and the cache type in the
            // PTE.  Therefore for a request of type UC, the effective type
            // will be UC irrespective of the MTRR settings in that range.
            // Hence it is not necessary to remove the existing MTRR settings
            // (if any) for that range.
            //

            //
            // Clip/remove any ranges in the target area
            //

            KiRemoveRange (&NewRange, NewRange.Base, NewRange.Limit, RemoveThisType);
        }

        //
        // If needed, add new range type
        //

        if (AddToRangeDatabase) {
            ASSERT (EffectRangeChange == TRUE);
            KiAddRange (&NewRange, NewRange.Base, NewRange.Limit, NewRange.Type);
        }

        //
        // If this is an effect range change, then complete it
        //

        if (EffectRangeChange) {
            KiCompleteEffectiveRangeChange (&NewRange);
        }
    }

    KeReleaseSpinLock (&KiRangeLock, OldIrql);
    if (EffectRangeChange) {
        MmUnlockPagableImageSection(ExPageLockHandle);
    }

    return NewRange.Status;
}

BOOLEAN
KiRemoveRange (
    IN PNEW_RANGE   NewRange,
    IN ULONGLONG    Base,
    IN ULONGLONG    Limit,
    IN PBOOLEAN     RemoveThisType
    )
/*++

Routine Description:

    This function removes any range overlapping with the passed range, of
    type supplied in RemoveThisType from the global range database.

Arguments:

    NewRange        - Context information

    Base            - Base & Limit signify the first & last address of a range
    Limit           - which is to be removed from the range database

    RemoveThisType  - A TRUE flag for each type which can not overlap the
                      target range


Return Value:

    TRUE  - if the range database was altered such that it may no longer
            be sorted.

--*/
{
    ULONG       i;
    PONE_RANGE  Range;
    BOOLEAN     DatabaseNeedsSorted;


    DatabaseNeedsSorted = FALSE;

    //
    // Check each range
    //

    for (i=0, Range=KiRangeInfo.Ranges; i < KiRangeInfo.NoRange; i++, Range++) {

        //
        // If this range type doesn't need to be altered, skip it
        //

        if (!RemoveThisType[Range->Type]) {
            continue;
        }

        //
        // Check range to see if it overlaps with range being removed
        //

        if (Range->Base < Base) {

            if (Range->Limit >= Base  &&  Range->Limit <= Limit) {

                //
                // Truncate range to not overlap with area being removed
                //

                Range->Limit = Base - 1;
            }

            if (Range->Limit > Limit) {

                //
                // Target area is contained totally within this area.
                // Split into two ranges
                //

                //
                // Add range at end
                //

                DatabaseNeedsSorted = TRUE;
                KiAddRange (
                    NewRange,
                    Limit+1,
                    Range->Limit,
                    Range->Type
                    );

                //
                // Turn current range into range at beginning
                //

                Range->Limit = Base - 1;
            }

        } else {

            // Range->Base >= Base

            if (Range->Base <= Limit) {
                if (Range->Limit <= Limit) {
                    //
                    // This range is totally within the target area.  Remove it.
                    //

                    DatabaseNeedsSorted = TRUE;
                    KiRangeInfo.NoRange -= 1;
                    Range->Base  = KiRangeInfo.Ranges[KiRangeInfo.NoRange].Base;
                    Range->Limit = KiRangeInfo.Ranges[KiRangeInfo.NoRange].Limit;
                    Range->Type = KiRangeInfo.Ranges[KiRangeInfo.NoRange].Type;

                    //
                    // recheck at current location
                    //

                    i -= 1;
                    Range -= 1;

                } else {

                    //
                    // Bump beginning past area being removed
                    //

                    Range->Base = Limit + 1;
                }
            }
        }
    }

    if (!NT_SUCCESS (NewRange->Status)) {
        DBGMSG ("KiRemoveRange: failure\n");
    }

    return DatabaseNeedsSorted;
}


VOID
KiAddRange (
    IN PNEW_RANGE   NewRange,
    IN ULONGLONG    Base,
    IN ULONGLONG    Limit,
    IN UCHAR        Type
    )
/*++

Routine Description:

    This function adds the passed range to the global range database.

Arguments:

    NewRange        - Context information

    Base            - Base & Limit signify the first & last address of a range
    Limit           - which is to be added to the range database

    Type            - Type of caching required for this range

Return Value:

    None - Context is updated with an error if the table has overflowed

--*/
{
    PONE_RANGE      Range, OldRange;
    ULONG           size;

    if (KiRangeInfo.NoRange >= KiRangeInfo.MaxRange) {

        //
        // Table is out of space, get a bigger one
        //

        OldRange = KiRangeInfo.Ranges;
        size = sizeof(ONE_RANGE) * (KiRangeInfo.MaxRange + GROW_RANGE_TABLE);
        Range  = ExAllocatePoolWithTag (NonPagedPool, size, '  eK');

        if (!Range) {
            NewRange->Status = STATUS_UNSUCCESSFUL;
            return ;
        }

        //
        // Grow table
        //

        RtlZeroMemory (Range, size);
        RtlCopyMemory (Range, OldRange, sizeof(ONE_RANGE) * KiRangeInfo.MaxRange);
        KiRangeInfo.Ranges = Range;
        KiRangeInfo.MaxRange += GROW_RANGE_TABLE;
        ExFreePool (OldRange);
    }

    //
    // Add new entry to table
    //

    KiRangeInfo.Ranges[KiRangeInfo.NoRange].Base = Base;
    KiRangeInfo.Ranges[KiRangeInfo.NoRange].Limit = Limit;
    KiRangeInfo.Ranges[KiRangeInfo.NoRange].Type = Type;
    KiRangeInfo.NoRange += 1;
}


VOID
KiStartEffectiveRangeChange (
    IN PNEW_RANGE   NewRange
    )
/*++

Routine Description:

    This functions sets up the context information required to
    track & later effect a range change in hardware

Arguments:

    NewRange        - Context information

Return Value:

    None

--*/
{
    ULONG   size;

    //
    // Allocate working space for MTRR image
    //

    size = sizeof(MTRR_RANGE) * ((ULONG) KiRangeInfo.Capabilities.u.hw.VarCnt + 1);
    NewRange->MTRR = ExAllocatePoolWithTag (NonPagedPool, size, '  eK');
    if (!NewRange->MTRR) {
        NewRange->Status = STATUS_UNSUCCESSFUL;
        return ;
    }

    RtlZeroMemory (NewRange->MTRR, size);

    //
    // Save current range information in case of an error
    //

    size = sizeof(ONE_RANGE) * KiRangeInfo.NoRange;
    NewRange->NoRange = KiRangeInfo.NoRange;
    NewRange->Ranges = ExAllocatePoolWithTag (NonPagedPool, size, '  eK');
    if (!NewRange->Ranges) {
        NewRange->Status = STATUS_UNSUCCESSFUL;
        return ;
    }

    RtlCopyMemory (NewRange->Ranges, KiRangeInfo.Ranges, size);
}


VOID
KiCompleteEffectiveRangeChange (
    IN PNEW_RANGE   NewRange
    )
/*++

Routine Description:

    This functions commits the range database to hardware, or backs
    out the current changes to it.

Arguments:

    NewRange        - Context information

Return Value:

    None

--*/
{
    BOOLEAN         Restart;
    ULONG           Index, Index2, RemIndex2, NoMTRR;
    ULONGLONG       BestLength, WhichMtrr;
    ULONGLONG       CurrLength;
    ULONGLONG       l, Base, Length, MLength;
    PONE_RANGE      Range;
    ONE_RANGE       OneRange;
    PMTRR_RANGE     MTRR;
    BOOLEAN         RoundDown;
    BOOLEAN         RemoveThisType[MTRR_TYPE_MAX];
    PKPRCB          Prcb;
    KIRQL           OldIrql, OldIrql2;
    KAFFINITY       TargetProcessors;


    ASSERT (KeGetCurrentIrql() == DISPATCH_LEVEL);
    Prcb = KeGetCurrentPrcb();

    //
    // Round all ranges, according to type, to match what h/w can support
    //

    for (Index=0; Index < KiRangeInfo.NoRange; Index++) {
        Range = &KiRangeInfo.Ranges[Index];

        //
        // Determine rounding for this range type
        //

        RoundDown = TRUE;
        if (Range->Type == MTRR_TYPE_UC) {
            RoundDown = FALSE;
        }

        //
        // Apply rounding
        //

        if (RoundDown) {
            Range->Base  = (Range->Base  + MTRR_PAGE_SIZE - 1) & MTRR_PAGE_MASK;
            Range->Limit = ((Range->Limit+1) & MTRR_PAGE_MASK)-1;
        } else {
            Range->Base  = (Range->Base  & MTRR_PAGE_MASK);
            Range->Limit = ((Range->Limit + MTRR_PAGE_SIZE) & MTRR_PAGE_MASK)-1;
        }
    }

    do {
        Restart = FALSE;

        //
        // Sort the ranges by base address
        //

        for (Index=0; Index < KiRangeInfo.NoRange; Index++) {
            Range = &KiRangeInfo.Ranges[Index];

            for (Index2=Index+1; Index2 < KiRangeInfo.NoRange; Index2++) {

                if (KiRangeInfo.Ranges[Index2].Base < Range->Base) {

                    //
                    // Swap KiRangeInfo.Ranges[Index] with KiRangeInfo.Ranges[Index2]
                    //

                    OneRange = *Range;
                    *Range = KiRangeInfo.Ranges[Index2];
                    KiRangeInfo.Ranges[Index2] = OneRange;
                }
            }
        }

        //
        // At this point the range database is sorted on
        // base address. Scan range database combining adjacent and
        // overlapping ranges of the same type
        //

        for (Index=0; Index < (ULONG) KiRangeInfo.NoRange-1; Index++) {
            Range = &KiRangeInfo.Ranges[Index];

            //
            // Scan the range database. If ranges are adjacent/overlap and are of
            // the same type, combine them.
            //

            for (Index2 = Index+1; Index2 < (ULONG) KiRangeInfo.NoRange; Index2++) {

                l = Range[0].Limit + 1;
                if (l < Range[0].Limit) {
                    l = Range[0].Limit;
                }

                if (l >= KiRangeInfo.Ranges[Index2].Base  &&
                    Range[0].Type == KiRangeInfo.Ranges[Index2].Type) {

                    //
                    // Increase Range[0] limit to cover Range[Index2]
                    //

                    if (KiRangeInfo.Ranges[Index2].Limit > Range[0].Limit) {
                        Range[0].Limit = KiRangeInfo.Ranges[Index2].Limit;
                    }

                    //
                    // Remove KiRangeInfo.Ranges[Index2]
                    //

                    if (Index2 < (ULONG) KiRangeInfo.NoRange - 1 ) {

                        //
                        // Copy everything from Index2 till end
                        // of range list. # Entries to copy is
                        // (KiRangeInfo.NoRange -1) - (Index2+1) + 1
                        //

                        RtlCopyMemory(
                            &(KiRangeInfo.Ranges[Index2]),
                            &(KiRangeInfo.Ranges[Index2+1]),
                            sizeof(ONE_RANGE) * (KiRangeInfo.NoRange-Index2-1)
                            );
                    }

                    KiRangeInfo.NoRange -= 1;

                    //
                    // Recheck current location
                    //

                    Index2 -= 1;
                }
            }
        }

        //
        // At this point the range database is sorted on base
        // address and adjacent/overlapping ranges of the same
        // type are combined. Check for overlapping ranges -
        // If legal then allow else truncate the less "weighty" range
        //

        for (Index = 0; Index < (ULONG) KiRangeInfo.NoRange-1  &&  !Restart; Index++) {

            Range = &KiRangeInfo.Ranges[Index];

            l = Range[0].Limit + 1;
            if (l < Range[0].Limit) {
                l = Range[0].Limit;
            }

            //
            // If ranges overlap and are not of same type, and if the
            // overlap is not legal then carve them to the best cache type
            // available.
            //

            for (Index2 = Index+1; Index2 < (ULONG) KiRangeInfo.NoRange && !Restart; Index2++) {

                if (l > KiRangeInfo.Ranges[Index2].Base) {

                    if (Range[0].Type == MTRR_TYPE_UC ||
                        KiRangeInfo.Ranges[Index2].Type == MTRR_TYPE_UC) {

                        //
                        // Overlap of a UC type with a range of any other type is
                        // legal
                        //

                    } else if ((Range[0].Type == MTRR_TYPE_WT &&
                                KiRangeInfo.Ranges[Index2].Type == MTRR_TYPE_WB) ||
                               (Range[0].Type == MTRR_TYPE_WB &&
                                KiRangeInfo.Ranges[Index2].Type == MTRR_TYPE_WT) ) {
                        //
                        // Overlap of WT and WB range is legal. The overlap range will
                        // be WT.
                        //

                    } else {

                        //
                        // This is an illegal overlap and we need to carve the ranges
                        // to remove the overlap.
                        //
                        // Pick range which has the cache type which should be used for
                        // the overlapped area
                        //

                        if (KiRangeWeight(&Range[0]) > KiRangeWeight(&(KiRangeInfo.Ranges[Index2]))){
                            RemIndex2 = Index2;
                        } else {
                            RemIndex2 = Index;
                        }

                        //
                        // Remove ranges of type which do not belong in the overlapped area
                        //

                        RtlZeroMemory (RemoveThisType, MTRR_TYPE_MAX);
                        RemoveThisType[KiRangeInfo.Ranges[RemIndex2].Type] = TRUE;

                        //
                        // Remove just the overlapped portion of the range.
                        //

                        Restart = KiRemoveRange (
                           NewRange,
                           KiRangeInfo.Ranges[Index2].Base,
                           (Range[0].Limit < KiRangeInfo.Ranges[Index2].Limit ?
                                    Range[0].Limit : KiRangeInfo.Ranges[Index2].Limit),
                           RemoveThisType
                           );
                    }
                }
            }
        }

    } while (Restart);

    //
    // The range database is now rounded to fit in the h/w and sorted.
    // Attempt to build MTRR settings which exactly describe the ranges
    //

    MTRR = NewRange->MTRR;
    NoMTRR = 0;
    for (Index=0;NT_SUCCESS(NewRange->Status)&& Index<KiRangeInfo.NoRange;Index++) {
        Range = &KiRangeInfo.Ranges[Index];

        //
        // Build MTRRs to fit this range
        //

        Base   = Range->Base;
        Length = Range->Limit - Base + 1;

        while (Length) {

            //
            // Compute MTRR length for current range base & length
            //

            if (Base == 0) {
                MLength = Length;
            } else {
                MLength = (ULONGLONG) 1 << KiFindFirstSetRightBit(Base);
            }
            if (MLength > Length) {
                MLength = Length;
            }

            l = (ULONGLONG) 1 << KiFindFirstSetLeftBit (MLength);
            if (MLength > l) {
                MLength = l;
            }

            //
            // Store it in the next MTRR
            //

            MTRR[NoMTRR].Base.u.QuadPart = Base;
            MTRR[NoMTRR].Base.u.hw.Type  = Range->Type;
            MTRR[NoMTRR].Mask.u.QuadPart = KiLengthToMask(MLength);
            MTRR[NoMTRR].Mask.u.hw.Valid = 1;
            NoMTRR += 1;

            //
            // Adjust off amount of data covered by that last MTRR
            //

            Base += MLength;
            Length -= MLength;

            //
            // If there are too many MTRRs, and currently setting a
            // Non-USWC range try to remove a USWC MTRR.
            // (ie, convert some MmWriteCombined to MmNonCached).
            //

            if (NoMTRR > (ULONG) KiRangeInfo.Capabilities.u.hw.VarCnt) {

                if (Range->Type != MTRR_TYPE_USWC) {

                    //
                    // Find smallest USWC type and drop it
                    //
                    // This is okay only if the default type is UC.
                    // Default type should always be UC unless BIOS changes
                    // it. Still ASSERT!
                    //

                    ASSERT(KiRangeInfo.Default.u.hw.Type == MTRR_TYPE_UC);

                    BestLength = (ULONGLONG) 1 << (MTRR_MAX_RANGE_SHIFT + 1);

                    for (Index2=0; Index2 < KiRangeInfo.Capabilities.u.hw.VarCnt; Index2++) {

                        if (MTRR[Index2].Base.u.hw.Type == MTRR_TYPE_USWC) {

                            CurrLength = KiMaskToLength(MTRR[Index2].Mask.u.QuadPart &
                                                 MTRR_MASK_MASK);

                            if (CurrLength < BestLength) {
                                WhichMtrr = Index2;
                                BestLength = CurrLength;
                            }
                        }
                    }

                    if (BestLength == ((ULONGLONG) 1 << (MTRR_MAX_RANGE_SHIFT + 1))) {
                        //
                        // Range was not found which could be dropped.  Abort process
                        //

                        NewRange->Status = STATUS_UNSUCCESSFUL;
                        Length = 0;

                    } else {
                        //
                        // Remove WhichMtrr
                        //

                        NoMTRR -= 1;
                        MTRR[WhichMtrr] = MTRR[NoMTRR];
                    }

                } else {

                    NewRange->Status = STATUS_UNSUCCESSFUL;
                    Length =0;
                }
            }
        }
    }

    //
    // Done building new MTRRs
    //

    if (NT_SUCCESS(NewRange->Status)) {

        //
        // Update the MTRRs on all processors
        //

#if IDBG
        KiDumpMTRR ("Loading the following MTRR:", NewRange->MTRR);
#endif

        NewRange->Synchronize.TargetCount = 0;
        NewRange->Synchronize.TargetPhase = &Prcb->ReverseStall;
        NewRange->Synchronize.Processor = Prcb->Number;

        //
        // Previously enabled MTRRs with index > NoMTRR
        // which could conflict with existing setting should be disabled
        // This is taken care of by setting NewRange->NoMTRR to total
        // number of variable MTRRs.
        //

        NewRange->NoMTRR = (ULONG) KiRangeInfo.Capabilities.u.hw.VarCnt;

        //
        // Synchronize with other IPI functions which may stall
        //

        KiLockContextSwap(&OldIrql);

#if !defined(NT_UP)
        //
        // Collect all the (other) processors
        //

        TargetProcessors = KeActiveProcessors & ~Prcb->SetMember;
        if (TargetProcessors != 0) {

            KiIpiSendSynchronousPacket (
                Prcb,
                TargetProcessors,
                KiLoadMTRRTarget,
                (PVOID) NewRange,
                NULL,
                NULL
                );

            //
            // Wait for all processors to be collected
            //

            KiIpiStallOnPacketTargets(TargetProcessors);

            //
            // All processors are now waiting.  Raise to high level to
            // ensure this processor doesn't enter the debugger due to
            // some interrupt service routine.
            //

            KeRaiseIrql (HIGH_LEVEL, &OldIrql2);

            //
            // There's no reason for any debug events now, so signal
            // the other processors that they can all disable interrupts
            // and begin the MTRR update
            //

            Prcb->ReverseStall += 1;
        }
#endif

        //
        // Update MTRRs
        //

        KiLoadMTRR (NewRange);

        //
        // Release ContextSwap lock
        //

        KiUnlockContextSwap(OldIrql);


#if IDBG
        KiDumpMTRR ("Processor MTRR:", NewRange->MTRR);
#endif

    } else {

        //
        // There was an error, put original range database back
        //

        DBGMSG ("KiCompleteEffectiveRangeChange: mtrr update did not occur\n");

        if (NewRange->Ranges) {
            KiRangeInfo.NoRange = NewRange->NoRange;

            RtlCopyMemory (
                KiRangeInfo.Ranges,
                NewRange->Ranges,
                sizeof (ONE_RANGE) * KiRangeInfo.NoRange
                );
        }
    }

    //
    // Cleanup
    //

    ExFreePool (NewRange->Ranges);
    ExFreePool (NewRange->MTRR);
}


STATIC ULONG
KiRangeWeight (
    IN PONE_RANGE   Range
    )
/*++

Routine Description:

    This functions returns a weighting of the passed in range's cache
    type.   When two or more regions collide within the same h/w region
    the types are weighted and that cache type of the higher weight
    is used for the collision area.

Arguments:

    Range   - Range to obtain weighting for

Return Value:

    The weight of the particular cache type

--*/
{
    ULONG   Weight;

    switch (Range->Type) {
        case MTRR_TYPE_UC:      Weight = 5;     break;
        case MTRR_TYPE_USWC:    Weight = 4;     break;
        case MTRR_TYPE_WP:      Weight = 3;     break;
        case MTRR_TYPE_WT:      Weight = 2;     break;
        case MTRR_TYPE_WB:      Weight = 1;     break;
        default:                Weight = 0;     break;
    }

    return Weight;
}


STATIC ULONGLONG
KiMaskToLength (
    IN ULONGLONG    Mask
    )
/*++

Routine Description:

    This function returns the length specified by a particular
    mtrr variable register mask.

--*/
{
    if (Mask == 0) {
        // Zero Mask signifies a length of      2**36
        return(((ULONGLONG) 1 << MTRR_MAX_RANGE_SHIFT));
    } else {
        return(((ULONGLONG) 1 << KiFindFirstSetRightBit(Mask)));
    }
}

STATIC ULONGLONG
KiLengthToMask (
    IN ULONGLONG    Length
    )
/*++

Routine Description:

    This function constructs the mask corresponding to the input length
    to be set in a variable MTRR register. The length is assumed to be
    a multiple of 4K.

--*/
{
    ULONGLONG FullMask = 0xffffff;

    if (Length == ((ULONGLONG) 1 << MTRR_MAX_RANGE_SHIFT)) {
        return(0);
    } else {
        return(((FullMask << KiFindFirstSetRightBit(Length)) &
            MTRR_RESVBIT_MASK));
    }
}

STATIC ULONG
KiFindFirstSetRightBit (
    IN ULONGLONG    Set
    )
/*++

Routine Description:

    This function returns a bit position of the least significant
    bit set in the passed ULONGLONG parameter. Passed parameter
    must be non-zero.

--*/
{
    ULONG   bitno;

    ASSERT(Set != 0);
    for (bitno=0; !(Set & 0xFF); bitno += 8, Set >>= 8) ;
    return KiFindFirstSetRight[Set & 0xFF] + bitno;
}

STATIC ULONG
KiFindFirstSetLeftBit (
    IN ULONGLONG    Set
    )
/*++

Routine Description:

    This function returns a bit position of the most significant
    bit set in the passed ULONGLONG parameter. Passed parameter
    must be non-zero.

--*/
{
    ULONG   bitno;

    ASSERT(Set != 0);
    for (bitno=56;!(Set & 0xFF00000000000000); bitno -= 8, Set <<= 8) ;
    return KiFindFirstSetLeft[Set >> 56] + bitno;
}

#if IDBG
VOID
KiDumpMTRR (
    PUCHAR          DebugString,
    PMTRR_RANGE     MTRR
    )
/*++

Routine Description:

    This function dumps the MTRR information to the debugger

--*/
{
    static PUCHAR Type[] = {
    //  0       1       2       3       4       5       6
        "UC  ", "USWC", "????", "????", "WT  ", "WP  ", "WB  " };
    MTRR_VARIABLE_BASE  Base;
    MTRR_VARIABLE_MASK  Mask;
    ULONG       Index;
    ULONG       i;
    PUCHAR      p;

    DbgPrint ("%s\n", DebugString);
    for (Index=0; Index < (ULONG) KiRangeInfo.Capabilities.u.hw.VarCnt; Index++) {
        if (MTRR) {
            Base = MTRR[Index].Base;
            Mask = MTRR[Index].Mask;
        } else {
            Base.u.QuadPart = RDMSR(MTRR_MSR_VARIABLE_BASE+2*Index);
            Mask.u.QuadPart = RDMSR(MTRR_MSR_VARIABLE_MASK+2*Index);
        }

        DbgPrint ("  %d. ", Index);
        if (Mask.u.hw.Valid) {
            p = "????";
            if (Base.u.hw.Type < 7) {
                p = Type[Base.u.hw.Type];
            }

            DbgPrint ("%s  %08x:%08x  %08x:%08x",
                p,
                (ULONG) (Base.u.QuadPart >> 32),
                ((ULONG) (Base.u.QuadPart & MTRR_MASK_BASE)),
                (ULONG) (Mask.u.QuadPart >> 32),
                ((ULONG) (Mask.u.QuadPart & MTRR_MASK_MASK))
                );

        }
        DbgPrint ("\n");
    }
}
#endif


VOID
KiLoadMTRRTarget (
    IN PKIPI_CONTEXT SignalDone,
    IN PVOID NewRange,
    IN PVOID Parameter2,
    IN PVOID Parameter3
    )
{
    PNEW_RANGE Context;

    Context = (PNEW_RANGE) NewRange;

    //
    // Wait for all processors to be ready
    //

    KiIpiSignalPacketDoneAndStall(SignalDone,
                                  Context->Synchronize.TargetPhase);

    //
    // Update MTRRs
    //

    KiLoadMTRR (Context);
}



#define MOV_EAX_CR4   _emit { 0Fh, 20h, E0h }
#define MOV_CR4_EAX   _emit { 0Fh, 22h, E0h }

NTSTATUS
KiLoadMTRR (
    IN PNEW_RANGE Context
    )
/*++

Routine Description:

    This function loads the memory type range registers into all processors

Arguments:

    Context     - Context which include the MTRRs to load

Return Value:

    All processors are set into the new state

--*/
{
    MTRR_DEFAULT        Default;
    BOOLEAN             Enable;
    ULONG               HldCr0, HldCr4;
    ULONG               Index;

    //
    // Disable interrupts
    //

    Enable = KeDisableInterrupts();

    //
    // Synchronize all processors
    //

    if (!(KeFeatureBits & KF_AMDK6MTRR)) {
        KiLockStepExecution (&Context->Synchronize);
    }

    _asm {
        ;
        ; Get current CR0
        ;

        mov     eax, cr0
        mov     HldCr0, eax

        ;
        ; Disable caching & line fill
        ;

        and     eax, not CR0_NW
        or      eax, CR0_CD
        mov     cr0, eax

        ;
        ; Flush caches
        ;

        ;
        ; wbinvd
        ;

        _emit 0Fh
        _emit 09h

        ;
        ; Get current cr4
        ;

        _emit  0Fh
        _emit  20h
        _emit  0E0h             ; mov eax, cr4
        mov     HldCr4, eax

        ;
        ; Disable global page
        ;

        and     eax, not CR4_PGE
        _emit  0Fh
        _emit  22h
        _emit  0E0h             ; mov cr4, eax

        ;
        ; Flush TLB
        ;

        mov     eax, cr3
        mov     cr3, eax
    }

    if (KeFeatureBits & KF_AMDK6MTRR) {

        //
        // Write the MTRRs
        //

        KiAmdK6MtrrWRMSR();

    } else {

        //
        // Disable MTRRs
        //

        Default.u.QuadPart = RDMSR(MTRR_MSR_DEFAULT);
        Default.u.hw.MtrrEnabled = 0;
        WRMSR (MTRR_MSR_DEFAULT, Default.u.QuadPart);

        //
        // Load new MTRRs
        //

        for (Index=0; Index < Context->NoMTRR; Index++) {
            WRMSR (MTRR_MSR_VARIABLE_BASE+2*Index, Context->MTRR[Index].Base.u.QuadPart);
            WRMSR (MTRR_MSR_VARIABLE_MASK+2*Index, Context->MTRR[Index].Mask.u.QuadPart);
        }
    }
    _asm {

        ;
        ; Flush caches (this should be a "nop", but it was in the Intel reference algorithm)
        ; This is required because of aggressive prefetch of both instr + data
        ;

        ;
        ; wbinvd
        ;

        _emit 0Fh
        _emit 09h

        ;
        ; Flush TLBs (same comment as above)
        ; Same explanation as above
        ;

        mov     eax, cr3
        mov     cr3, eax
    }

    if (!(KeFeatureBits & KF_AMDK6MTRR)) {

        //
        // Enable MTRRs
        //

        Default.u.hw.MtrrEnabled = 1;
        WRMSR (MTRR_MSR_DEFAULT, Default.u.QuadPart);
    }

    _asm {
        ;
        ; Restore CR4 (global page enable)
        ;

        mov     eax, HldCr4
        _emit  0Fh
        _emit  22h
        _emit  0E0h             ; mov cr4, eax

        ;
        ; Restore CR0 (cache enable)
        ;

        mov     eax, HldCr0
        mov     cr0, eax
    }

    //
    // Wait for all processors to reach the same place,
    // restore interrupts and return.
    //

    if (!(KeFeatureBits & KF_AMDK6MTRR)) {
        KiLockStepExecution (&Context->Synchronize);
    }

    KeEnableInterrupts (Enable);
    return STATUS_SUCCESS;
}


VOID
KiLockStepExecution (
    IN PPROCESSOR_LOCKSTEP  Context
    )
{

#if !defined(NT_UP)

    ULONG               CurrentPhase;
    volatile ULONG      *TargetPhase;
    PKPRCB              Prcb;

    TargetPhase = Context->TargetPhase;
    Prcb = KeGetCurrentPrcb();

    if (Prcb->Number == (CCHAR) Context->Processor) {

        //
        // Wait for all processors to signal
        //

        while (Context->TargetCount != (ULONG) KeNumberProcessors - 1) {
            KeYieldProcessor ();
        }

        //
        // Reset count for next time
        //

        Context->TargetCount = 0;

        //
        // Let waiting processor go to next synchronization point
        //

        InterlockedIncrement ((PULONG) TargetPhase);


    } else {

        //
        // Get current phase
        //

        CurrentPhase = *TargetPhase;

        //
        // Signal that we have completed the current phase
        //

        InterlockedIncrement ((PULONG) &Context->TargetCount);

        //
        // Wait for new phase to begin
        //

        while (*TargetPhase == CurrentPhase) {
            KeYieldProcessor ();
        }
    }

#endif

}