You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
565 lines
12 KiB
565 lines
12 KiB
/*++
|
|
|
|
Copyright (c) 2000 Microsoft Corporation
|
|
|
|
Module Name:
|
|
|
|
mpipic.c
|
|
|
|
Abstract:
|
|
|
|
This module provides the HAL support for interprocessor interrupts and
|
|
processor initialization for MPS systems.
|
|
|
|
Author:
|
|
|
|
Forrest Foltz (forrestf) 27-Oct-2000
|
|
|
|
Environment:
|
|
|
|
Kernel mode only.
|
|
|
|
Revision History:
|
|
|
|
--*/
|
|
|
|
#if defined(APIC_HAL)
|
|
|
|
#if !defined(_MPIPIC_C_)
|
|
#define _MPIPIC_C_
|
|
|
|
#pragma optimize("Oxt2b2",on)
|
|
|
|
#include "halp.h"
|
|
#include "apic.inc"
|
|
#include "pcmp_nt.inc"
|
|
|
|
#if !defined(LOCAL_APIC)
|
|
|
|
//
|
|
// Temporary defines. These definitions are found in common header files in
|
|
// the private hal branch.
|
|
//
|
|
|
|
#define LOCAL_APIC(x) (*((ULONG volatile *)&pLocalApic[(x)/sizeof(ULONG)]))
|
|
|
|
/*++
|
|
|
|
HalpStallWhileApicBusy (
|
|
VOID
|
|
)
|
|
|
|
Routine Description:
|
|
|
|
This routine waits until the local apic has completed sending
|
|
an IPI.
|
|
|
|
Parameters:
|
|
|
|
None.
|
|
|
|
Return Value:
|
|
|
|
None.
|
|
|
|
--*/
|
|
|
|
#define HalpStallWhileApicBusy() \
|
|
while (((LOCAL_APIC(LU_INT_CMD_LOW) & DELIVERY_PENDING) != 0)){}
|
|
|
|
#endif // LOCAL_APIC
|
|
|
|
//
|
|
// HalpIpiTargetLookup[] and HalpIpiTargetMask[] are tables used by
|
|
// HalpSendNodeIpi() and are initialized by HalpBuildIpiDestinationMap().
|
|
//
|
|
// They assist in performing the translation between a (32- or 64- bit)
|
|
// KAFFINITY into a 64-bit Node Target Set.
|
|
//
|
|
// Each element of HalpIpiTargetLookup[] contains the logical sum of
|
|
// the 8 (or 4) Node Target Sets for a particular byte value. Each
|
|
// element of HalpIpiTargetMask[] contains the mask of all possible
|
|
// APIC targets for a particular byte position with KAFFINITY.
|
|
//
|
|
// For example: Suppose one wished to determine the set of APIC targets
|
|
// for affinity 0x00000000b7000000.
|
|
//
|
|
// First, find the value of HalpIpiTargetLookup[0xb7]. This represents the set
|
|
// of APIC targets for the affinity 0xb7b7b7b7b7b7b7b7.
|
|
//
|
|
// Next, mask the value with HalpIpiTargetMask[3]. The 3 represents the byte
|
|
// number within the KAFFINITY.
|
|
//
|
|
// The result of the operation will yield the set of APIC targets that
|
|
// correspond to an affinity of 0x00000000b7000000.
|
|
//
|
|
|
|
ULONG64 HalpIpiTargetLookup[256];
|
|
ULONG64 HalpIpiTargetMask[sizeof(KAFFINITY)];
|
|
|
|
//
|
|
// Local function prototypes and types. There are up to three versions of
|
|
// the send IPI code, depending on whether the apic topology is flat, cluster
|
|
// with 8 or fewer nodes, or cluster with more than 8 nodes.
|
|
//
|
|
|
|
VOID
|
|
FASTCALL
|
|
HalpSendFlatIpi (
|
|
IN KAFFINITY Affinity,
|
|
IN ULONG Command
|
|
);
|
|
|
|
VOID
|
|
FASTCALL
|
|
HalpSendNodeIpi32 (
|
|
IN KAFFINITY Affinity,
|
|
IN ULONG Command
|
|
);
|
|
|
|
VOID
|
|
FASTCALL
|
|
HalpSendNodeIpi64 (
|
|
IN KAFFINITY Affinity,
|
|
IN ULONG Command
|
|
);
|
|
|
|
VOID (FASTCALL *HalpIpiRoutine) (
|
|
IN KAFFINITY Affinity,
|
|
IN ULONG Command
|
|
);
|
|
|
|
//
|
|
// External data
|
|
//
|
|
|
|
extern INTERRUPT_DEST HalpIntDestMap[MAX_PROCESSORS];
|
|
|
|
|
|
|
|
//
|
|
// Implementation
|
|
//
|
|
|
|
__forceinline
|
|
VOID
|
|
HalpSendIpiWorker (
|
|
IN UCHAR TargetSet,
|
|
IN ULONG Command
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine is called to send an IPI command to a set of processors
|
|
on a single node.
|
|
|
|
Parameters:
|
|
|
|
TargetSet - Specifies the processor identifiers within the node.
|
|
|
|
Command - Specifies the IPI command to send.
|
|
|
|
Return Value:
|
|
|
|
None.
|
|
|
|
--*/
|
|
|
|
{
|
|
ULONG destination;
|
|
|
|
//
|
|
// Only high byte of the destination is used. Wait until the Apic is
|
|
// not busy before sending. Continue without waiting, there will be
|
|
// another wait after all IPIs have been submitted.
|
|
//
|
|
|
|
destination = (ULONG)TargetSet << DESTINATION_SHIFT;
|
|
|
|
HalpStallWhileApicBusy();
|
|
LOCAL_APIC(LU_INT_CMD_HIGH) = destination;
|
|
LOCAL_APIC(LU_INT_CMD_LOW) = Command;
|
|
}
|
|
|
|
|
|
VOID
|
|
FASTCALL
|
|
HalpSendFlatIpi (
|
|
IN KAFFINITY Affinity,
|
|
IN ULONG Command
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine is called to send an IPI command to a set of processors. This
|
|
routine is invoked when we have a maximum of 8 processors and the APICs have
|
|
been set up in "flat" mode.
|
|
|
|
Parameters:
|
|
|
|
TargetSet - Specifies the processor identifiers.
|
|
|
|
Command - Specifies the IPI command to send.
|
|
|
|
Return Value:
|
|
|
|
None.
|
|
|
|
--*/
|
|
|
|
{
|
|
HalpSendIpiWorker((UCHAR)Affinity,Command);
|
|
HalpStallWhileApicBusy();
|
|
}
|
|
|
|
VOID
|
|
FASTCALL
|
|
HalpSendIpi (
|
|
IN KAFFINITY Affinity,
|
|
IN ULONG Command
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine disables interrupts, dispatches to the correct IPI send
|
|
routine, and restores interrupts.
|
|
|
|
Parameters:
|
|
|
|
Affinity - Specifies the set of processors to receive the IPI.
|
|
|
|
Command - Specifies the IPI command to send.
|
|
|
|
Return Value:
|
|
|
|
None.
|
|
|
|
--*/
|
|
|
|
{
|
|
ULONG flags;
|
|
|
|
//
|
|
// Disable interrupts and call the appropriate routine.
|
|
//
|
|
// BUGBUG the compiler generates terrible code for this,
|
|
// most likely because of the inline _asm{} block generated
|
|
// by HalpDisableInterrupts().
|
|
//
|
|
// Ideally we could talk the x86 compiler team into giving
|
|
// us an intrinsic like the AMD64 compiler's __getcallerseflags()
|
|
//
|
|
|
|
flags = HalpDisableInterrupts();
|
|
HalpIpiRoutine(Affinity,Command);
|
|
HalpRestoreInterrupts(flags);
|
|
}
|
|
|
|
#define APIC_IPI (DELIVER_FIXED | LOGICAL_DESTINATION | ICR_USE_DEST_FIELD | APIC_IPI_VECTOR)
|
|
#define APIC_BROADCAST_EXCL \
|
|
(DELIVER_FIXED | LOGICAL_DESTINATION | ICR_ALL_EXCL_SELF | APIC_IPI_VECTOR)
|
|
|
|
#define APIC_BROADCAST_INCL \
|
|
(DELIVER_FIXED | LOGICAL_DESTINATION | ICR_ALL_INCL_SELF | APIC_IPI_VECTOR)
|
|
|
|
VOID
|
|
HalRequestIpi (
|
|
IN KAFFINITY Affinity
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
Requests an interprocessor interrupt
|
|
|
|
Arguments:
|
|
|
|
Affinity - Supplies the set of processors to be interrupted
|
|
|
|
Return Value:
|
|
|
|
None.
|
|
|
|
--*/
|
|
|
|
{
|
|
|
|
ULONG flags;
|
|
KAFFINITY Self;
|
|
|
|
//
|
|
// If the target set of processors is the complete set of processors,
|
|
// then use the broadcast capability of the APIC. Otherwise, send the
|
|
// IPI to the individual processors.
|
|
//
|
|
|
|
Self = KeGetCurrentPrcb()->SetMember;
|
|
if ((Affinity | Self) == HalpActiveProcessors) {
|
|
flags = HalpDisableInterrupts();
|
|
HalpStallWhileApicBusy();
|
|
if ((Affinity & Self) != 0) {
|
|
LOCAL_APIC(LU_INT_CMD_LOW) = APIC_BROADCAST_INCL;
|
|
|
|
} else {
|
|
LOCAL_APIC(LU_INT_CMD_LOW) = APIC_BROADCAST_EXCL;
|
|
}
|
|
|
|
HalpStallWhileApicBusy();
|
|
HalpRestoreInterrupts(flags);
|
|
|
|
} else {
|
|
HalpSendIpi(Affinity, APIC_IPI);
|
|
}
|
|
|
|
return;
|
|
}
|
|
|
|
VOID
|
|
HalpBuildIpiDestinationMap (
|
|
VOID
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description
|
|
|
|
This routine is called whenever a new processor comes on line, just
|
|
after its APIC is initialized. It (re)builds the lookup tables that
|
|
are used by HalpSendNodeIpi{32|64}.
|
|
|
|
This code isn't particularly fast, and doesn't need to be - it is
|
|
executed once per processor during boot.
|
|
|
|
Arguments:
|
|
|
|
None:
|
|
|
|
Return Value:
|
|
|
|
None
|
|
|
|
--*/
|
|
{
|
|
|
|
ULONG byteNumber;
|
|
ULONG index;
|
|
ULONG mask;
|
|
ULONG processor;
|
|
ULONG64 targetMap;
|
|
ULONG64 targetMapSum;
|
|
|
|
if (HalpMaxProcsPerCluster == 0) {
|
|
|
|
//
|
|
// Running in flat mode. IPIs are sent by the flat mode routine.
|
|
//
|
|
|
|
HalpIpiRoutine = HalpSendFlatIpi;
|
|
return;
|
|
}
|
|
|
|
//
|
|
// Build HalpIpiTargetLookup[] and HalpIpiTargetMask[] according to
|
|
// the contents of HalpIntDestMap[]. If an additional processor is
|
|
// added, this routine can be safely called again assuming the topology
|
|
// of the existing processors has not changed.
|
|
//
|
|
|
|
for (byteNumber = 0; byteNumber < sizeof(KAFFINITY); byteNumber++) {
|
|
|
|
targetMapSum = 0;
|
|
for (index = 0; index < 256; index++) {
|
|
|
|
processor = byteNumber * 8;
|
|
mask = index;
|
|
while (mask != 0) {
|
|
|
|
if ((mask & 0x1) != 0) {
|
|
targetMap = HalpIntDestMap[processor].Cluster.Hw.DestId;
|
|
targetMap <<=
|
|
(HalpIntDestMap[processor].Cluster.Hw.ClusterId * 4);
|
|
|
|
HalpIpiTargetLookup[index] |= targetMap;
|
|
targetMapSum |= targetMap;
|
|
}
|
|
|
|
processor += 1;
|
|
mask >>= 1;
|
|
}
|
|
}
|
|
|
|
HalpIpiTargetMask[byteNumber] = targetMapSum;
|
|
}
|
|
|
|
#if defined(_AMD64_)
|
|
|
|
HalpIpiRoutine = HalpSendNodeIpi64;
|
|
|
|
#else
|
|
|
|
//
|
|
// Determine which of the two IPI cluster send routines to invoke
|
|
// depending on the maximum node ID.
|
|
//
|
|
|
|
HalpIpiRoutine = HalpSendNodeIpi32;
|
|
for (processor = 0; processor < MAX_PROCESSORS; processor += 1) {
|
|
if (HalpIntDestMap[processor].Cluster.Hw.ClusterId > 7) {
|
|
HalpIpiRoutine = HalpSendNodeIpi64;
|
|
break;
|
|
}
|
|
}
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
#if !defined(_AMD64_)
|
|
|
|
//
|
|
// Here, two versions of HalpSendNodeIpi are compiled. The first,
|
|
// HalpSendNodeIpi32(), is used when we have a maximum of 8 APIC
|
|
// nodes. On a 32-bit processor, it is significantly faster because
|
|
// it uses 32-bit lookup, mask and shift operations.
|
|
//
|
|
|
|
#define HalpSendNodeIpi HalpSendNodeIpi32
|
|
#define TARGET_MASK ULONG
|
|
#include "mpipic.c"
|
|
|
|
#undef HalpSendNodeIpi
|
|
#undef TARGET_MASK
|
|
|
|
#endif
|
|
|
|
//
|
|
// Here the 64-bit version of HalpSendNodeIpi64 is created. On a
|
|
// 32-bit processor, this is used when we have more than 8 APIC
|
|
// nodes. It is the only multi-node routine on a 64-bit platform.
|
|
//
|
|
|
|
#define HalpSendNodeIpi HalpSendNodeIpi64
|
|
#define TARGET_MASK ULONG64
|
|
#include "mpipic.c"
|
|
|
|
#pragma optimize("",on)
|
|
|
|
#else // _MPIPIC_C_
|
|
|
|
//
|
|
// This portion of the module is included at least once (see above) in
|
|
// order to build HalpSendNodeIpi32() and/or HalpSendNodeIpi64().
|
|
//
|
|
|
|
VOID
|
|
FASTCALL
|
|
HalpSendNodeIpi (
|
|
IN KAFFINITY Affinity,
|
|
IN ULONG Command
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine sends one or more IPIs to APIC nodes. This code generates
|
|
two forms of this routine - HalpSendNodeIpi32() and HalpSendNodeIpi64() -
|
|
based on whether we have more than 8 APIC nodes or not.
|
|
|
|
Parameters:
|
|
|
|
Affinity - Specifies the set of processors to receive the IPI.
|
|
|
|
Command - Specifies the IPI command to send.
|
|
|
|
Return Value:
|
|
|
|
None.
|
|
|
|
--*/
|
|
|
|
{
|
|
KAFFINITY affinity;
|
|
UCHAR clusterIndex;
|
|
ULONG byteNumber;
|
|
TARGET_MASK targetMap;
|
|
TARGET_MASK targetMapSum;
|
|
ULONG64 *targetMask;
|
|
UCHAR logicalId;
|
|
ULONG mapIndex;
|
|
|
|
//
|
|
// Affinity has some number of target processors indicated. Each
|
|
// target processor is a member of a cluster of processors, or "node".
|
|
//
|
|
// Build targetMap by processing Affinity one byte at a time. This
|
|
// loop executes a maximum of sizeof(KAFFINITY) times.
|
|
//
|
|
|
|
affinity = Affinity;
|
|
targetMask = HalpIpiTargetMask;
|
|
targetMapSum = 0;
|
|
do {
|
|
|
|
mapIndex = (UCHAR)affinity;
|
|
if (mapIndex != 0) {
|
|
targetMap = (TARGET_MASK)HalpIpiTargetLookup[mapIndex];
|
|
targetMap &= (TARGET_MASK)*targetMask;
|
|
targetMapSum |= targetMap;
|
|
}
|
|
|
|
targetMask += 1;
|
|
affinity >>= 8;
|
|
|
|
} while (affinity != 0);
|
|
|
|
//
|
|
// targetMap is an array of 4-bit node-relative target masks.
|
|
// Process the array, sending an IPI to each non-zero element.
|
|
//
|
|
// This loop executes a maximum of sizeof(TARGET_MASK) times.
|
|
//
|
|
|
|
clusterIndex = 0;
|
|
do {
|
|
|
|
//
|
|
// Determine whether any APICs in this node
|
|
// are targeted, and send an IPI to the node
|
|
// if so.
|
|
//
|
|
|
|
logicalId = (UCHAR)targetMapSum & 0x0F;
|
|
if (logicalId != 0) {
|
|
logicalId |= clusterIndex;
|
|
HalpSendIpiWorker(logicalId,Command);
|
|
}
|
|
|
|
//
|
|
// Shift the APIC targets for the next node into place, increment
|
|
// the cluster ID, and continue processing if there are still
|
|
// APIC targets remaining.
|
|
//
|
|
|
|
targetMapSum >>= 4;
|
|
clusterIndex += 0x10;
|
|
|
|
} while (targetMapSum != 0);
|
|
|
|
//
|
|
// Wait for the APIC to process the final IPI.
|
|
//
|
|
|
|
HalpStallWhileApicBusy();
|
|
}
|
|
|
|
#endif // _MPIPIC_C_
|
|
|
|
#endif // APIC_HAL
|