mirror of https://github.com/tongzx/nt5src
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1082 lines
22 KiB
1082 lines
22 KiB
|
|
/*++
|
|
|
|
Module Name:
|
|
|
|
flush.c
|
|
|
|
Abstract:
|
|
|
|
This module implements IA64 machine dependent kernel functions to flush
|
|
the data and instruction caches and to flush I/O buffers.
|
|
|
|
Author:
|
|
|
|
07-Mar-1996
|
|
|
|
Bernard Lint
|
|
M. Jayakumar ([email protected])
|
|
|
|
|
|
Environment:
|
|
|
|
Kernel mode only.
|
|
|
|
Revision History:
|
|
|
|
--*/
|
|
|
|
#include "ki.h"
|
|
#include "kxia64.h"
|
|
|
|
//
|
|
// PROBE_VISIBILITY_PAL_SUPPORT flag is one time write (RESET) only and multiple time read
|
|
// only flag. It is used to check to see if the processor needs PAL_SUPPORT for VISIBILITY // in prefetches. Once the check is made, this flag optimizes such that further checks are // eliminated.
|
|
//
|
|
|
|
ULONG ProbePalVisibilitySupport=1;
|
|
ULONG NeedPalVisibilitySupport=1;
|
|
extern KSPIN_LOCK KiCacheFlushLock;
|
|
//
|
|
// Define forward referenced prototyes.
|
|
//
|
|
|
|
VOID
|
|
KiSweepDcacheTarget (
|
|
IN PULONG SignalDone,
|
|
IN PVOID Parameter1,
|
|
IN PVOID Parameter2,
|
|
IN PVOID Parameter3
|
|
);
|
|
|
|
VOID
|
|
KiSweepIcacheTarget (
|
|
IN PULONG SignalDone,
|
|
IN PVOID Parameter1,
|
|
IN PVOID Parameter2,
|
|
IN PVOID Parameter3
|
|
);
|
|
|
|
VOID
|
|
KiFlushIoBuffersTarget (
|
|
IN PKIPI_CONTEXT SignalDone,
|
|
IN PVOID Mdl,
|
|
IN PVOID ReadOperation,
|
|
IN PVOID DmaOperation
|
|
);
|
|
|
|
VOID
|
|
KiSyncCacheTarget(
|
|
IN PKIPI_CONTEXT SignalDone,
|
|
IN PVOID Parameter1,
|
|
IN PVOID Parameter2,
|
|
IN PVOID Parameter3
|
|
);
|
|
|
|
ULONG_PTR
|
|
KiSyncMC_DrainTarget(
|
|
);
|
|
|
|
|
|
ULONG_PTR
|
|
KiSyncMC_Drain(
|
|
IN BOOLEAN AllProcessors,
|
|
IN PVOID BaseAddress,
|
|
IN ULONG Length
|
|
);
|
|
|
|
ULONG_PTR
|
|
KiSyncPrefetchVisibleTarget(
|
|
);
|
|
|
|
ULONG_PTR
|
|
KiSyncPrefetchVisible (
|
|
IN BOOLEAN AllProcessors,
|
|
IN PVOID BaseAddress,
|
|
IN ULONG Length
|
|
);
|
|
|
|
|
|
|
|
|
|
VOID
|
|
KiSyncCacheTarget (
|
|
IN PKIPI_CONTEXT SignalDone,
|
|
IN PVOID Parameter1,
|
|
IN PVOID Parameter2,
|
|
IN PVOID Parameter3
|
|
)
|
|
|
|
/*++
|
|
Routine Description:
|
|
|
|
This function synchronizes the I-fetch pipeline. Typically this routine will be
|
|
executed by every processor in the system in response to an IPI after the cache
|
|
is flushed. Each processor executing RFI while leaving the IPI produces the
|
|
serialization effect that is required after isync to make sure that further
|
|
instruction prefetches wait till the ISYNC completes.
|
|
|
|
Arguements:
|
|
|
|
SignalDone Supplies a pointer to a variable that is cleared when the
|
|
requested operation has been performed.
|
|
|
|
Parameter1 - Parameter3 - Not used.
|
|
|
|
Return Value:
|
|
|
|
Nothing.
|
|
--*/
|
|
{
|
|
|
|
#if !defined(NT_UP)
|
|
|
|
__synci();
|
|
KiIpiSignalPacketDone(SignalDone);
|
|
|
|
#endif
|
|
return;
|
|
|
|
}
|
|
|
|
VOID
|
|
KeSweepIcache (
|
|
IN BOOLEAN AllProcessors
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This function flushes the instruction cache on all processors that are
|
|
currently running threads which are children of the current process or
|
|
flushes the instruction cache on all processors in the host configuration.
|
|
|
|
N.B. Although PowerPC maintains cache coherency across processors, we
|
|
use the flash invalidate function (h/w) for I-Cache sweeps which doesn't
|
|
maintain coherency so we still do the MP I-Cache flush in s/w. plj.
|
|
|
|
Arguments:
|
|
|
|
AllProcessors - Supplies a boolean value that determines which instruction
|
|
caches are flushed.
|
|
|
|
Return Value:
|
|
|
|
None.
|
|
|
|
--*/
|
|
|
|
{
|
|
|
|
KIRQL OldIrql;
|
|
KAFFINITY TargetProcessors;
|
|
|
|
ASSERT(KeGetCurrentIrql() <= DISPATCH_LEVEL);
|
|
|
|
#if !defined(NT_UP)
|
|
//
|
|
// Acquire cache flush spinlock
|
|
// Cache flush is not MP safe yet
|
|
//
|
|
KeAcquireSpinLock(&KiCacheFlushLock, &OldIrql);
|
|
|
|
#endif
|
|
|
|
HalSweepIcache();
|
|
|
|
#if !defined(NT_UP)
|
|
|
|
//
|
|
// Compute the set of target processors and send the sweep parameters
|
|
// to the target processors, if any, for execution.
|
|
//
|
|
|
|
TargetProcessors = KeActiveProcessors & PCR->NotMember;
|
|
if (TargetProcessors != 0) {
|
|
KiIpiSendPacket(TargetProcessors,
|
|
KiSweepIcacheTarget,
|
|
NULL,
|
|
NULL,
|
|
NULL);
|
|
}
|
|
|
|
|
|
//
|
|
// Wait until all target processors have finished sweeping their
|
|
// instruction caches.
|
|
//
|
|
|
|
|
|
if (TargetProcessors != 0) {
|
|
KiIpiStallOnPacketTargets(TargetProcessors);
|
|
}
|
|
|
|
//
|
|
// Lower IRQL to its previous level and return.
|
|
//
|
|
|
|
KeReleaseSpinLock(&KiCacheFlushLock, OldIrql);
|
|
|
|
#endif
|
|
|
|
return;
|
|
}
|
|
|
|
VOID
|
|
KiSweepIcacheTarget (
|
|
IN PULONG SignalDone,
|
|
IN PVOID Parameter1,
|
|
IN PVOID Parameter2,
|
|
IN PVOID Parameter3
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This is the target function for sweeping the instruction cache on
|
|
target processors.
|
|
|
|
Arguments:
|
|
|
|
SignalDone Supplies a pointer to a variable that is cleared when the
|
|
requested operation has been performed.
|
|
|
|
Parameter1 - Parameter3 - Not used.
|
|
|
|
Return Value:
|
|
|
|
None.
|
|
|
|
--*/
|
|
|
|
{
|
|
|
|
//
|
|
// Sweep the instruction cache on the current processor and clear
|
|
// the sweep instruction cache packet address to signal the source
|
|
// to continue.
|
|
//
|
|
|
|
#if !defined(NT_UP)
|
|
|
|
HalSweepIcache();
|
|
|
|
KiIpiSignalPacketDone(SignalDone);
|
|
|
|
#endif
|
|
|
|
return;
|
|
}
|
|
|
|
VOID
|
|
KeSweepDcache (
|
|
IN BOOLEAN AllProcessors
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This function flushes the data cache on all processors that are currently
|
|
running threads which are children of the current process or flushes the
|
|
data cache on all processors in the host configuration.
|
|
|
|
N.B. PowerPC maintains cache coherency across processors however
|
|
in this routine, the range of addresses being flushed is unknown
|
|
so we must still broadcast the request to the other processors.
|
|
|
|
Arguments:
|
|
|
|
AllProcessors - Supplies a boolean value that determines which data
|
|
caches are flushed.
|
|
|
|
Return Value:
|
|
|
|
None.
|
|
|
|
--*/
|
|
|
|
{
|
|
|
|
KIRQL OldIrql;
|
|
KAFFINITY TargetProcessors;
|
|
|
|
ASSERT(KeGetCurrentIrql() <= DISPATCH_LEVEL);
|
|
|
|
#if !defined(NT_UP)
|
|
//
|
|
// Acquire cache flush spinlock
|
|
// Cache flush is not MP safe yet
|
|
//
|
|
KeAcquireSpinLock(&KiCacheFlushLock, &OldIrql);
|
|
|
|
#endif
|
|
|
|
HalSweepDcache();
|
|
|
|
#if !defined(NT_UP)
|
|
|
|
//
|
|
// Compute the set of target processors and send the sweep parameters
|
|
// to the target processors, if any, for execution.
|
|
//
|
|
|
|
TargetProcessors = KeActiveProcessors & PCR->NotMember;
|
|
if (TargetProcessors != 0) {
|
|
KiIpiSendPacket(TargetProcessors,
|
|
KiSweepDcacheTarget,
|
|
NULL,
|
|
NULL,
|
|
NULL);
|
|
}
|
|
|
|
|
|
//
|
|
// Wait until all target processors have finished sweeping their
|
|
// data caches.
|
|
//
|
|
|
|
|
|
if (TargetProcessors != 0) {
|
|
KiIpiStallOnPacketTargets(TargetProcessors);
|
|
}
|
|
|
|
//
|
|
// Lower IRQL to its previous level and return.
|
|
//
|
|
|
|
KeReleaseSpinLock(&KiCacheFlushLock, OldIrql);
|
|
|
|
#endif
|
|
|
|
return;
|
|
}
|
|
|
|
VOID
|
|
KiSweepDcacheTarget (
|
|
IN PULONG SignalDone,
|
|
IN PVOID Parameter1,
|
|
IN PVOID Parameter2,
|
|
IN PVOID Parameter3
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This is the target function for sweeping the data cache on target
|
|
processors.
|
|
|
|
Arguments:
|
|
|
|
SignalDone Supplies a pointer to a variable that is cleared when the
|
|
requested operation has been performed.
|
|
|
|
Parameter1 - Parameter3 - Not used.
|
|
|
|
Return Value:
|
|
|
|
None.
|
|
|
|
--*/
|
|
|
|
{
|
|
|
|
//
|
|
// Sweep the data cache on the current processor and clear the sweep
|
|
// data cache packet address to signal the source to continue.
|
|
//
|
|
|
|
#if !defined(NT_UP)
|
|
|
|
HalSweepDcache();
|
|
KiIpiSignalPacketDone(SignalDone);
|
|
|
|
#endif
|
|
|
|
return;
|
|
}
|
|
|
|
|
|
|
|
ULONG_PTR
|
|
KiSyncMC_DrainTarget(
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This is the target function for issuing PAL_MC_DRAIN to drain
|
|
prefetches, demand references and pending fc cache line evictions on the
|
|
target CPU it executes.
|
|
|
|
Argument:
|
|
|
|
None
|
|
|
|
|
|
Return Value:
|
|
|
|
Returns the status from the function HalCallPal
|
|
|
|
--*/
|
|
|
|
{
|
|
ULONG_PTR Status;
|
|
|
|
//
|
|
// Call HalCallPal to drain.
|
|
//
|
|
|
|
Status = HalCallPal(PAL_MC_DRAIN,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0);
|
|
|
|
ASSERT(Status == PAL_STATUS_SUCCESS);
|
|
|
|
return Status;
|
|
|
|
}
|
|
|
|
|
|
VOID
|
|
KeSweepCacheRange (
|
|
IN BOOLEAN AllProcessors,
|
|
IN PVOID BaseAddress,
|
|
IN ULONG Length
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This function is used to flush a range of virtual addresses from both the
|
|
instruction and data cache on all processors in the system.
|
|
|
|
Irrespective of the length of the range, it should not call SweepIcache
|
|
or SweepDcache. This is because SweepDcache will only sweep D cache and
|
|
not the I cache and Vice versa. Since the caller of KeSweepCacheRange assumes
|
|
both the caches are being swept, one cannot call SweepIcache or SweepDcache
|
|
in trying to optimize.
|
|
|
|
|
|
Arguments:
|
|
|
|
AllProcessors - Not used
|
|
|
|
BaseAddress - Supplies a pointer to the base of the range that is flushed.
|
|
|
|
Length - Supplies the length of the range that is flushed if the base
|
|
address is specified.
|
|
|
|
Return Value:
|
|
|
|
None.
|
|
|
|
|
|
--*/
|
|
|
|
{
|
|
KIRQL OldIrql;
|
|
KAFFINITY TargetProcessors;
|
|
|
|
//
|
|
// We will not raise IRQL to synchronization level so that we can allow
|
|
// a context switch in between Flush Cache. FC need not run in the same processor
|
|
// throughout. It can be context switched. So no binding is done to any processor.
|
|
//
|
|
//
|
|
|
|
HalSweepCacheRange(BaseAddress,Length);
|
|
|
|
ASSERT(KeGetCurrentIrql() <= KiSynchIrql);
|
|
|
|
//
|
|
// Raise IRQL to synchronization level to prevent a context switch.
|
|
//
|
|
|
|
#if !defined(NT_UP)
|
|
|
|
OldIrql = KeRaiseIrqlToSynchLevel();
|
|
|
|
//
|
|
// Compute the set of target processors and send the sync parameters
|
|
// to the target processors, if any, for execution.
|
|
//
|
|
|
|
TargetProcessors = KeActiveProcessors & PCR->NotMember;
|
|
if (TargetProcessors != 0) {
|
|
KiIpiSendPacket(TargetProcessors,
|
|
KiSyncCacheTarget,
|
|
NULL,
|
|
NULL,
|
|
NULL);
|
|
}
|
|
|
|
#endif
|
|
|
|
//
|
|
// Synchronize the Instruction Prefetch pipe in the local processor.
|
|
//
|
|
|
|
__synci();
|
|
__isrlz();
|
|
|
|
//
|
|
// Wait until all target processors have finished sweeping the their
|
|
// data cache.
|
|
//
|
|
|
|
#if !defined(NT_UP)
|
|
|
|
if (TargetProcessors != 0) {
|
|
KiIpiStallOnPacketTargets(TargetProcessors);
|
|
}
|
|
|
|
//
|
|
// Lower IRQL to its previous level and return.
|
|
//
|
|
|
|
KeLowerIrql(OldIrql);
|
|
|
|
#endif
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
VOID
|
|
KeSweepIcacheRange (
|
|
IN BOOLEAN AllProcessors,
|
|
IN PVOID BaseAddress,
|
|
IN SIZE_T Length
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This function is used to flush a range of virtual addresses from the
|
|
primary instruction cache on all processors in the host configuration.
|
|
|
|
If the length of the range is greater than the size of the
|
|
instruction cache, then one can call HalSweepIcache which calls
|
|
SAL to flush the entire cache. Since SAL does not take care of MP
|
|
flushing, HalSweepIcache has to use IPI mechanism to execute SAL
|
|
flush from each processor. We need to weight the overhead of all these
|
|
versus using HalSweepIcacheRange and avoiding IPI mechanism since
|
|
HalSweepIcacheRange uses fc instruction and fc instruction takes care of MP.
|
|
|
|
Arguments:
|
|
|
|
AllProcessors - Not used
|
|
|
|
BaseAddress - Supplies a pointer to the base of the range that is flushed.
|
|
|
|
Length - Supplies the length of the range that is flushed if the base
|
|
address is specified.
|
|
|
|
Return Value:
|
|
|
|
None.
|
|
|
|
Note: For performance reason, we may update KeSweepIcacheRange to do the following:
|
|
if the range asked to sweep is very large, we may call KeSweepIcache to flush
|
|
the full cache.
|
|
|
|
|
|
|
|
--*/
|
|
|
|
{
|
|
KIRQL OldIrql;
|
|
KAFFINITY TargetProcessors;
|
|
|
|
//
|
|
// We will not raise IRQL to synchronization level so that we can allow
|
|
// a context switch in between Flush Cache. FC need not run in the same processor
|
|
// throughout. It can be context switched. So no binding is done to any processor.
|
|
//
|
|
//
|
|
|
|
HalSweepIcacheRange(BaseAddress,Length);
|
|
|
|
ASSERT(KeGetCurrentIrql() <= KiSynchIrql);
|
|
|
|
//
|
|
// Raise IRQL to synchronization level to prevent a context switch.
|
|
//
|
|
|
|
#if !defined(NT_UP)
|
|
|
|
OldIrql = KeRaiseIrqlToSynchLevel();
|
|
|
|
//
|
|
// Compute the set of target processors and send the sync parameters
|
|
// to the target processors, if any, for execution.
|
|
//
|
|
|
|
TargetProcessors = KeActiveProcessors & PCR->NotMember;
|
|
if (TargetProcessors != 0) {
|
|
KiIpiSendPacket(TargetProcessors,
|
|
KiSyncCacheTarget,
|
|
NULL,
|
|
NULL,
|
|
NULL);
|
|
}
|
|
|
|
#endif
|
|
|
|
//
|
|
// Synchronize the Instruction Prefetch pipe in the local processor.
|
|
//
|
|
|
|
__synci();
|
|
__isrlz();
|
|
|
|
//
|
|
// Wait until all target processors have finished sweeping the their
|
|
// data cache.
|
|
//
|
|
|
|
#if !defined(NT_UP)
|
|
|
|
if (TargetProcessors != 0) {
|
|
KiIpiStallOnPacketTargets(TargetProcessors);
|
|
}
|
|
|
|
//
|
|
// Lower IRQL to its previous level and return.
|
|
//
|
|
|
|
KeLowerIrql(OldIrql);
|
|
|
|
#endif
|
|
|
|
return;
|
|
|
|
|
|
}
|
|
|
|
VOID
|
|
KeSweepCurrentIcacheRange (
|
|
IN PVOID BaseAddress,
|
|
IN SIZE_T Length
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This function is used to flush a range of virtual addresses from the
|
|
primary instruction cache on the current processor.
|
|
|
|
This is used by the kernel debugger for flushing the i-cache after
|
|
modifying memory in case the instruction stream is changed.
|
|
|
|
To avoid calling SAL during phase 0 we use "fc" instead of the SAL cache
|
|
flush call.
|
|
|
|
Arguments:
|
|
|
|
BaseAddress - Supplies a pointer to the base of the range that is flushed.
|
|
|
|
Length - Supplies the length of the range that is flushed if the base
|
|
address is specified.
|
|
|
|
Return Value:
|
|
|
|
None.
|
|
|
|
--*/
|
|
|
|
{
|
|
KIRQL OldIrql;
|
|
|
|
KeRaiseIrql(HIGH_LEVEL, &OldIrql);
|
|
|
|
HalSweepIcacheRange(BaseAddress,Length);
|
|
|
|
//
|
|
// Synchronize the Instruction Prefetch pipe in the local processor.
|
|
//
|
|
|
|
__synci();
|
|
__isrlz();
|
|
|
|
KeLowerIrql(OldIrql);
|
|
|
|
return;
|
|
}
|
|
|
|
|
|
|
|
VOID
|
|
KeSweepDcacheRange (
|
|
IN BOOLEAN AllProcessors,
|
|
IN PVOID BaseAddress,
|
|
IN ULONG Length
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This function is used to flush a range of virtual addresses from the
|
|
primary data cache on all processors in the host configuration.
|
|
|
|
If the length of the range is greater than the size of the
|
|
data cache, then one can call HalSweepDcache which calls
|
|
SAL to flush the entire cache. Since SAL does not take care of MP
|
|
flushing, HalSweepDcache has to use IPI mechanism to execute SAL
|
|
flush from each processor. We need to weight the overhead of all these
|
|
versus using HalSweepDcacheRange and avoiding IPI mechanism since
|
|
HalSweepDcacheRange uses fc instruction and fc instruction takes care of MP.
|
|
|
|
Arguments:
|
|
|
|
AllProcessors - Not used
|
|
|
|
BaseAddress - Supplies a pointer to the base of the range that is flushed.
|
|
|
|
Length - Supplies the length of the range that is flushed if the base
|
|
address is specified.
|
|
|
|
Return Value:
|
|
|
|
None.
|
|
|
|
Note: For performance reason, we may update KeSweepDcacheRange to do the following:
|
|
if the range asked to sweep is very large, we may call KeSweepDcache to flush
|
|
the full cache.
|
|
|
|
|
|
|
|
--*/
|
|
|
|
{
|
|
KIRQL OldIrql;
|
|
KAFFINITY TargetProcessors;
|
|
|
|
//
|
|
// We will not raise IRQL to synchronization level so that we can allow
|
|
// a context switch in between Flush Cache. FC need not run in the same processor
|
|
// throughout. It can be context switched. So no binding is done to any processor.
|
|
//
|
|
//
|
|
|
|
HalSweepDcacheRange(BaseAddress,Length);
|
|
|
|
ASSERT(KeGetCurrentIrql() <= KiSynchIrql);
|
|
|
|
//
|
|
// Raise IRQL to synchronization level to prevent a context switch.
|
|
//
|
|
|
|
#if !defined(NT_UP)
|
|
|
|
OldIrql = KeRaiseIrqlToSynchLevel();
|
|
|
|
//
|
|
// Compute the set of target processors and send the sync parameters
|
|
// to the target processors, if any, for execution.
|
|
//
|
|
|
|
TargetProcessors = KeActiveProcessors & PCR->NotMember;
|
|
if (TargetProcessors != 0) {
|
|
KiIpiSendPacket(TargetProcessors,
|
|
KiSyncCacheTarget,
|
|
NULL,
|
|
NULL,
|
|
NULL);
|
|
}
|
|
|
|
#endif
|
|
|
|
//
|
|
// Synchronize the Instruction Prefetch pipe in the local processor.
|
|
//
|
|
|
|
__synci();
|
|
__isrlz();
|
|
|
|
//
|
|
// Wait until all target processors have finished sweeping the their
|
|
// data cache.
|
|
//
|
|
|
|
#if !defined(NT_UP)
|
|
|
|
if (TargetProcessors != 0) {
|
|
KiIpiStallOnPacketTargets(TargetProcessors);
|
|
}
|
|
|
|
//
|
|
// Lower IRQL to its previous level and return.
|
|
//
|
|
|
|
KeLowerIrql(OldIrql);
|
|
|
|
#endif
|
|
|
|
return;
|
|
|
|
|
|
}
|
|
|
|
ULONG_PTR
|
|
KiSyncMC_Drain (
|
|
IN BOOLEAN AllProcessors,
|
|
IN PVOID BaseAddress,
|
|
IN ULONG Length
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
KiSyncMC_Drain issues PAL_MC_DRAIN to drain either prefetches, demand references
|
|
or pending fc cache line evictions to all the processors in the system.
|
|
DrainTypePointer points to the variable, DrainType, which determines the type of
|
|
drain to be performed. This is typically used when changing the memory attribute
|
|
from WB to UC.
|
|
|
|
Arguments:
|
|
|
|
AllProcessors - All processors in the system.
|
|
|
|
BaseAddress - Supplies a pointer to the base of the range that is to be drained.
|
|
|
|
Length - Supplies the length of the range that is drained for the base
|
|
address specified.
|
|
|
|
Return Value:
|
|
|
|
Note: This is used when changing attributes of WB pages to UC pages.
|
|
|
|
|
|
--*/
|
|
|
|
{
|
|
ULONG_PTR Status;
|
|
//
|
|
// KiIpiGenericCall returns ULONG_PTR as the function value of the specified function
|
|
//
|
|
|
|
Status = (KiIpiGenericCall (
|
|
(PKIPI_BROADCAST_WORKER)KiSyncMC_DrainTarget,
|
|
(ULONG_PTR)NULL)
|
|
);
|
|
|
|
ASSERT(Status == PAL_STATUS_SUCCESS);
|
|
|
|
return Status;
|
|
|
|
|
|
}
|
|
|
|
ULONG_PTR
|
|
KiSyncPrefetchVisibleTarget(
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This is the target function for issuing PAL_PREFETCH VISIBILITY
|
|
on the target CPU it executes.
|
|
|
|
Argument:
|
|
|
|
Not used.
|
|
|
|
|
|
Return Value:
|
|
|
|
Returns the status from the function HalCallPal
|
|
|
|
--*/
|
|
|
|
{
|
|
ULONG_PTR Status;
|
|
|
|
//
|
|
// Call HalCallPal to drain.
|
|
//
|
|
|
|
Status = HalCallPal(PAL_PREFETCH_VISIBILITY,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0);
|
|
|
|
|
|
ASSERT(Status != PAL_STATUS_ERROR);
|
|
|
|
return Status;
|
|
|
|
}
|
|
|
|
|
|
|
|
ULONG_PTR
|
|
KiSyncPrefetchVisible (
|
|
IN BOOLEAN AllProcessors,
|
|
IN PVOID BaseAddress,
|
|
IN ULONG Length
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
KiSyncPrefetchVisible issues PAL_PREFETCH_VISIBILITY to cause the processor to make
|
|
all pending prefetches visible to subsequent fc instructions; or does nothing, on
|
|
processor implementations which does not require PAL support for disabling prefetch
|
|
in the architectural sequence. On processors that require PAL support for this
|
|
sequence, the actions performed by this procedure may include any or all
|
|
of the following (or none, as long as the processor guarantees that
|
|
prefetches that were issued prior to this call are not resident in the
|
|
processor's caches after the architected sequence is complete.
|
|
This is typically used when changing the memory attribute from WB to UC.
|
|
|
|
Arguments:
|
|
|
|
AllProcessors - All processors in the system.
|
|
|
|
BaseAddress - Supplies a pointer to the base of the range that is to be drained.
|
|
|
|
Length - Supplies the length of the range that is drained for the base
|
|
address specified.
|
|
|
|
Return Value:
|
|
|
|
Status of the PAL CALL
|
|
0 Success
|
|
1 Call not needed
|
|
-3 Error returned
|
|
|
|
Note: This is used when changing attributes of WB pages to UC pages.
|
|
|
|
|
|
--*/
|
|
|
|
{
|
|
ULONG_PTR Status;
|
|
|
|
switch (ProbePalVisibilitySupport) {
|
|
case 0:
|
|
if (NeedPalVisibilitySupport == 0)
|
|
Status = PAL_STATUS_SUPPORT_NOT_NEEDED;
|
|
else {
|
|
Status = (KiIpiGenericCall (
|
|
(PKIPI_BROADCAST_WORKER)KiSyncPrefetchVisibleTarget,
|
|
(ULONG_PTR)NULL)
|
|
);
|
|
}
|
|
break;
|
|
|
|
case 1:
|
|
Status = KiSyncPrefetchVisibleTarget();
|
|
|
|
ASSERT(Status != PAL_STATUS_ERROR);
|
|
|
|
ProbePalVisibilitySupport = 0;
|
|
|
|
if (Status == PAL_STATUS_SUPPORT_NOT_NEEDED) {
|
|
NeedPalVisibilitySupport = 0;
|
|
Status = PAL_STATUS_SUPPORT_NOT_NEEDED;
|
|
} else {
|
|
Status = (KiIpiGenericCall (
|
|
(PKIPI_BROADCAST_WORKER)KiSyncPrefetchVisibleTarget,
|
|
(ULONG_PTR)NULL)
|
|
);
|
|
}
|
|
break;
|
|
default:
|
|
Status = PAL_STATUS_ERROR;
|
|
break;
|
|
}
|
|
|
|
ASSERT(Status != PAL_STATUS_ERROR);
|
|
|
|
return Status;
|
|
|
|
}
|
|
|
|
|
|
|
|
VOID
|
|
KeSweepCacheRangeWithDrain (
|
|
IN BOOLEAN AllProcessors,
|
|
IN PVOID BaseAddress,
|
|
IN ULONG Length
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This function is used to drain prefetches,demand references followed by flushing
|
|
the cache followed by draining pending fc cache line evictions to a specified range
|
|
address in all processors in the system.
|
|
|
|
|
|
Arguments:
|
|
|
|
AllProcessors - All processors in the system.
|
|
|
|
BaseAddress - Supplies a pointer to the base of the range that is flushed and drained.
|
|
|
|
Length - Supplies the length of the range that is flushed and drained for the base
|
|
address is specified.
|
|
|
|
Return Value:
|
|
|
|
None.
|
|
|
|
Note: This is used when changing attributes of WB pages to UC pages.
|
|
|
|
--*/
|
|
|
|
{
|
|
ULONG_PTR Status;
|
|
|
|
Status = KiSyncPrefetchVisible(
|
|
AllProcessors,
|
|
BaseAddress,
|
|
Length
|
|
);
|
|
|
|
ASSERT(Status != PAL_STATUS_ERROR);
|
|
|
|
KeSweepCacheRange (
|
|
AllProcessors,
|
|
BaseAddress,
|
|
Length
|
|
);
|
|
|
|
Status = KiSyncMC_Drain (
|
|
AllProcessors,
|
|
BaseAddress,
|
|
Length
|
|
);
|
|
|
|
ASSERT(Status == PAL_STATUS_SUCCESS);
|
|
|
|
return;
|
|
|
|
|
|
}
|
|
|