You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
563 lines
13 KiB
563 lines
13 KiB
/*++
|
|
|
|
Copyright (c) 1990 Microsoft Corporation
|
|
|
|
Module Name:
|
|
|
|
allproc.c
|
|
|
|
Abstract:
|
|
|
|
This module allocates and intializes kernel resources required
|
|
to start a new processor, and passes a complete processor state
|
|
structure to the HAL to obtain a new processor.
|
|
|
|
Author:
|
|
|
|
Bernard Lint 31-Jul-96
|
|
|
|
Environment:
|
|
|
|
Kernel mode only.
|
|
|
|
Revision History:
|
|
|
|
Based on MIPS original (David N. Cutler 29-Apr-1993)
|
|
|
|
--*/
|
|
|
|
|
|
#include "ki.h"
|
|
#include "pool.h"
|
|
|
|
#if defined(KE_MULTINODE)
|
|
|
|
NTSTATUS
|
|
KiNotNumaQueryProcessorNode(
|
|
IN ULONG ProcessorNumber,
|
|
OUT PUSHORT Identifier,
|
|
OUT PUCHAR Node
|
|
);
|
|
|
|
#ifdef ALLOC_PRAGMA
|
|
#pragma alloc_text(INIT, KiNotNumaQueryProcessorNode)
|
|
#endif
|
|
|
|
#endif
|
|
|
|
#ifdef ALLOC_PRAGMA
|
|
|
|
#pragma alloc_text(INIT, KeStartAllProcessors)
|
|
#pragma alloc_text(INIT, KiAllProcessorsStarted)
|
|
|
|
#endif
|
|
|
|
//
|
|
// Define macro to round up to 128-byte boundary to account for cache
|
|
// line size increase and define block size.
|
|
//
|
|
|
|
#define ROUND_UP(x) ((sizeof(x) + 127) & (~127))
|
|
#define BLOCK_SIZE (PAGE_SIZE + ROUND_UP(KPRCB) + ROUND_UP(KNODE) + ROUND_UP(ETHREAD))
|
|
|
|
//
|
|
// PER PROCESSOR MEMORY ALLOCATION NOTES:
|
|
//
|
|
//
|
|
// Kernel/Panic stacks are allocated using MmCreateKernelStack. IA64
|
|
// stacks grows down, RSE (backing store) grows up. Stack allocations
|
|
// include associated backing store.
|
|
//
|
|
// Additional datastructures are layed out in a single memory
|
|
// allocation as follows:
|
|
//
|
|
// * PCR page = Base
|
|
// * KPRCB = Base + PAGE_SIZE
|
|
// * KNODE = Base + PAGE_SIZE + ROUND_UP(KPRCB)
|
|
// * ETHREAD = Base + PAGE_SIZE + ROUND_UP(KPRCB) + ROUND_UP(KNODE)
|
|
//
|
|
|
|
#if !defined(NT_UP)
|
|
|
|
//
|
|
// Define barrier wait static data.
|
|
//
|
|
|
|
ULONG KiBarrierWait = 0;
|
|
|
|
#endif
|
|
|
|
#if defined(KE_MULTINODE)
|
|
|
|
PHALNUMAQUERYPROCESSORNODE KiQueryProcessorNode = KiNotNumaQueryProcessorNode;
|
|
|
|
//
|
|
// Statically preallocate enough KNODE structures to allow MM
|
|
// to allocate pages by node during system initialization. As
|
|
// processors are brought online, real KNODE structures are
|
|
// allocated in the appropriate memory for the node.
|
|
//
|
|
// This statically allocated set will be deallocated once the
|
|
// system is initialized.
|
|
//
|
|
|
|
#ifdef ALLOC_DATA_PRAGMA
|
|
#pragma data_seg("INITDATA")
|
|
#endif
|
|
|
|
KNODE KiNodeInit[MAXIMUM_PROCESSORS];
|
|
|
|
#endif
|
|
|
|
extern ULONG_PTR KiUserSharedDataPage;
|
|
extern ULONG_PTR KiKernelPcrPage;
|
|
|
|
//
|
|
// Define forward referenced prototypes.
|
|
//
|
|
|
|
VOID
|
|
KiCalibratePerformanceCounter(
|
|
VOID
|
|
);
|
|
|
|
VOID
|
|
KiCalibratePerformanceCounterTarget (
|
|
IN PULONG SignalDone,
|
|
IN PVOID Count,
|
|
IN PVOID Parameter2,
|
|
IN PVOID Parameter3
|
|
);
|
|
|
|
VOID
|
|
KiOSRendezvous (
|
|
VOID
|
|
);
|
|
|
|
VOID
|
|
KeStartAllProcessors(
|
|
VOID
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This function is called during phase 1 initialize on the master boot
|
|
processor to start all of the other registered processors.
|
|
|
|
Arguments:
|
|
|
|
None.
|
|
|
|
Return Value:
|
|
|
|
None.
|
|
|
|
--*/
|
|
|
|
{
|
|
|
|
#if !defined(NT_UP)
|
|
|
|
ULONG_PTR MemoryBlock;
|
|
PUCHAR KernelStack;
|
|
PUCHAR PanicStack;
|
|
PVOID PoolTagTable;
|
|
ULONG_PTR PcrAddress;
|
|
ULONG Number;
|
|
ULONG Count;
|
|
PHYSICAL_ADDRESS PcrPage;
|
|
BOOLEAN Started;
|
|
KPROCESSOR_STATE ProcessorState;
|
|
UCHAR NodeNumber = 0;
|
|
USHORT ProcessorId;
|
|
PKNODE Node;
|
|
NTSTATUS Status;
|
|
PKPCR NewPcr;
|
|
|
|
#if defined(KE_MULTINODE)
|
|
|
|
//
|
|
// In the unlikely event that processor 0 is not on node
|
|
// 0, fix it.
|
|
//
|
|
|
|
|
|
if (KeNumberNodes > 1) {
|
|
Status = KiQueryProcessorNode(0,
|
|
&ProcessorId,
|
|
&NodeNumber);
|
|
|
|
if (NT_SUCCESS(Status)) {
|
|
|
|
//
|
|
// Adjust the data structures to reflect that P0 is not on Node 0.
|
|
//
|
|
|
|
if (NodeNumber != 0) {
|
|
|
|
ASSERT(KeNodeBlock[0] == &KiNode0);
|
|
KeNodeBlock[0]->ProcessorMask &= ~1I64;
|
|
KiNodeInit[0] = *KeNodeBlock[0];
|
|
KeNodeBlock[0] = &KiNodeInit[0];
|
|
|
|
KiNode0 = *KeNodeBlock[NodeNumber];
|
|
KeNodeBlock[NodeNumber] = &KiNode0;
|
|
KeNodeBlock[NodeNumber]->ProcessorMask |= 1;
|
|
}
|
|
KeGetCurrentPrcb()->ProcessorId = ProcessorId;
|
|
}
|
|
}
|
|
|
|
#endif
|
|
|
|
//
|
|
// If the registered number of processors is greater than the maximum
|
|
// number of processors supported, then only allow the maximum number
|
|
// of supported processors.
|
|
//
|
|
|
|
if (KeRegisteredProcessors > MAXIMUM_PROCESSORS) {
|
|
KeRegisteredProcessors = MAXIMUM_PROCESSORS;
|
|
}
|
|
|
|
//
|
|
// Set barrier that will prevent any other processor from entering the
|
|
// idle loop until all processors have been started.
|
|
//
|
|
|
|
KiBarrierWait = 1;
|
|
|
|
//
|
|
// Initialize the processor state that will be used to start each of
|
|
// processors. Each processor starts in the system initialization code
|
|
// with address of the loader parameter block as an argument.
|
|
//
|
|
|
|
Number = 0;
|
|
Count = 1;
|
|
RtlZeroMemory(&ProcessorState, sizeof(KPROCESSOR_STATE));
|
|
ProcessorState.ContextFrame.StIIP = ((PPLABEL_DESCRIPTOR)(ULONG_PTR)KiOSRendezvous)->EntryPoint;
|
|
while (Count < KeRegisteredProcessors) {
|
|
|
|
Number++;
|
|
|
|
if (Number >= MAXIMUM_PROCESSORS) {
|
|
break;
|
|
}
|
|
|
|
#if defined(KE_MULTINODE)
|
|
|
|
Status = KiQueryProcessorNode(Number,
|
|
&ProcessorId,
|
|
&NodeNumber);
|
|
if (!NT_SUCCESS(Status)) {
|
|
|
|
//
|
|
// No such processor, advance to next.
|
|
//
|
|
|
|
continue;
|
|
}
|
|
|
|
Node = KeNodeBlock[NodeNumber];
|
|
|
|
#endif
|
|
|
|
//
|
|
// Allocate idle thread kernel stack and panic stacks
|
|
//
|
|
|
|
KernelStack = MmCreateKernelStack(FALSE, NodeNumber);
|
|
PanicStack = MmCreateKernelStack(FALSE, NodeNumber);
|
|
|
|
//
|
|
// Allocate block containing PCR page, processor block, KNODE and an
|
|
// executive thread object. If any allocation fails, stop
|
|
// starting processors.
|
|
//
|
|
|
|
MemoryBlock = (ULONG_PTR)MmAllocateIndependentPages(BLOCK_SIZE, NodeNumber);
|
|
//
|
|
// Allocate a pool tag table for the new processor.
|
|
//
|
|
|
|
PoolTagTable = ExCreatePoolTagTable (Number, NodeNumber);
|
|
|
|
if ((KernelStack == NULL) ||
|
|
(PanicStack == NULL) ||
|
|
(MemoryBlock == 0) ||
|
|
(PoolTagTable == NULL)) {
|
|
|
|
if (PoolTagTable) {
|
|
ExDeletePoolTagTable(Number);
|
|
}
|
|
|
|
if (MemoryBlock) {
|
|
MmFreeIndependentPages((PUCHAR) MemoryBlock, BLOCK_SIZE);
|
|
}
|
|
|
|
if (PanicStack) {
|
|
MmDeleteKernelStack(PanicStack, FALSE);
|
|
}
|
|
|
|
if (KernelStack) {
|
|
MmDeleteKernelStack(KernelStack, FALSE);
|
|
}
|
|
|
|
break;
|
|
}
|
|
|
|
RtlZeroMemory((PVOID)MemoryBlock, BLOCK_SIZE);
|
|
|
|
//
|
|
// Set address of idle thread kernel stack in loader parameter block.
|
|
//
|
|
|
|
KeLoaderBlock->KernelStack = (ULONG_PTR) KernelStack;
|
|
|
|
//
|
|
// Set address of panic stack in loader parameter block.
|
|
//
|
|
|
|
KeLoaderBlock->u.Ia64.PanicStack = (ULONG_PTR) PanicStack;
|
|
|
|
//
|
|
// Set the address of the processor block and executive thread in the
|
|
// loader parameter block.
|
|
//
|
|
|
|
KeLoaderBlock->Prcb = MemoryBlock + PAGE_SIZE;
|
|
KeLoaderBlock->Thread = KeLoaderBlock->Prcb + ROUND_UP(KPRCB) +
|
|
ROUND_UP(KNODE);
|
|
((PKPRCB)KeLoaderBlock->Prcb)->Number = (UCHAR)Number;
|
|
|
|
#if defined(KE_MULTINODE)
|
|
|
|
//
|
|
// If this is the first processor on this node, use the
|
|
// space allocated for KNODE as the KNODE.
|
|
//
|
|
|
|
if (KeNodeBlock[NodeNumber] == &KiNodeInit[NodeNumber]) {
|
|
Node = (PKNODE)(KeLoaderBlock->Prcb + ROUND_UP(KPRCB));
|
|
*Node = KiNodeInit[NodeNumber];
|
|
KeNodeBlock[NodeNumber] = Node;
|
|
}
|
|
|
|
((PKPRCB)KeLoaderBlock->Prcb)->ParentNode = Node;
|
|
((PKPRCB)KeLoaderBlock->Prcb)->ProcessorId = ProcessorId;
|
|
|
|
#else
|
|
|
|
((PKPRCB)KeLoaderBlock->Prcb)->ParentNode = KeNodeBlock[0];
|
|
|
|
#endif
|
|
|
|
|
|
//
|
|
// Set the page frame of the PCR page in the loader parameter block.
|
|
//
|
|
|
|
PcrAddress = MemoryBlock;
|
|
PcrPage = MmGetPhysicalAddress((PVOID)PcrAddress);
|
|
KeLoaderBlock->u.Ia64.PcrPage = PcrPage.QuadPart >> PAGE_SHIFT;
|
|
KeLoaderBlock->u.Ia64.PcrPage2 = KiUserSharedDataPage;
|
|
KiKernelPcrPage = KeLoaderBlock->u.Ia64.PcrPage;
|
|
|
|
//
|
|
// Initialize the NT page table base addresses in PCR
|
|
//
|
|
|
|
NewPcr = (PKPCR) PcrAddress;
|
|
NewPcr->PteUbase = PCR->PteUbase;
|
|
NewPcr->PteKbase = PCR->PteKbase;
|
|
NewPcr->PteSbase = PCR->PteSbase;
|
|
NewPcr->PdeUbase = PCR->PdeUbase;
|
|
NewPcr->PdeKbase = PCR->PdeKbase;
|
|
NewPcr->PdeSbase = PCR->PdeSbase;
|
|
NewPcr->PdeUtbase = PCR->PdeUtbase;
|
|
NewPcr->PdeKtbase = PCR->PdeKtbase;
|
|
NewPcr->PdeStbase = PCR->PdeStbase;
|
|
|
|
//
|
|
// Attempt to start the next processor. If attempt is successful,
|
|
// then wait for the processor to get initialized. Otherwise,
|
|
// deallocate the processor resources and terminate the loop.
|
|
//
|
|
|
|
Started = HalStartNextProcessor(KeLoaderBlock, &ProcessorState);
|
|
|
|
if (Started) {
|
|
|
|
//
|
|
// Wait for processor to initialize in kernel,
|
|
// then loop for another
|
|
//
|
|
|
|
while (*((volatile ULONG_PTR *) &KeLoaderBlock->Prcb) != 0) {
|
|
KeYieldProcessor();
|
|
}
|
|
|
|
#if defined(KE_MULTINODE)
|
|
|
|
Node->ProcessorMask |= 1I64 << Number;
|
|
|
|
#endif
|
|
|
|
} else {
|
|
|
|
ExDeletePoolTagTable(Number);
|
|
MmFreeIndependentPages((PUCHAR) MemoryBlock, BLOCK_SIZE);
|
|
MmDeleteKernelStack(PanicStack, FALSE);
|
|
MmDeleteKernelStack(KernelStack, FALSE);
|
|
break;
|
|
}
|
|
|
|
Count += 1;
|
|
}
|
|
|
|
//
|
|
// All processors have been stated.
|
|
//
|
|
|
|
KiAllProcessorsStarted();
|
|
|
|
//
|
|
// Allow all processor that were started to enter the idle loop and
|
|
// begin execution.
|
|
//
|
|
|
|
KiBarrierWait = 0;
|
|
|
|
#endif
|
|
|
|
//
|
|
// Reset and synchronize the performance counters of all processors.
|
|
//
|
|
|
|
KeAdjustInterruptTime (0);
|
|
return;
|
|
}
|
|
|
|
#if !defined(NT_UP)
|
|
VOID
|
|
KiAllProcessorsStarted(
|
|
VOID
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine is called once all processors in the system
|
|
have been started.
|
|
|
|
Arguments:
|
|
|
|
None.
|
|
|
|
Return Value:
|
|
|
|
None.
|
|
|
|
--*/
|
|
|
|
{
|
|
ULONG i;
|
|
|
|
#if defined(KE_MULTINODE)
|
|
|
|
//
|
|
// Make sure there are no references to the temporary nodes
|
|
// used during initialization.
|
|
//
|
|
|
|
for (i = 0; i < KeNumberNodes; i++) {
|
|
if (KeNodeBlock[i] == &KiNodeInit[i]) {
|
|
|
|
//
|
|
// No processor started on this node so no new node
|
|
// structure has been allocated. This is possible
|
|
// if the node contains only memory or IO busses. At
|
|
// this time we need to allocate a permanent node
|
|
// structure for the node.
|
|
//
|
|
|
|
KeNodeBlock[i] = ExAllocatePoolWithTag(NonPagedPool,
|
|
sizeof(KNODE),
|
|
' eK');
|
|
if (KeNodeBlock[i]) {
|
|
*KeNodeBlock[i] = KiNodeInit[i];
|
|
}
|
|
}
|
|
|
|
//
|
|
// Set the node number.
|
|
//
|
|
|
|
KeNodeBlock[i]->NodeNumber = (UCHAR)i;
|
|
}
|
|
|
|
for (i = KeNumberNodes; i < MAXIMUM_CCNUMA_NODES; i++) {
|
|
KeNodeBlock[i] = NULL;
|
|
}
|
|
|
|
#endif
|
|
|
|
if (KeNumberNodes == 1) {
|
|
|
|
//
|
|
// For Non NUMA machines, Node 0 gets all processors.
|
|
//
|
|
|
|
KeNodeBlock[0]->ProcessorMask = KeActiveProcessors;
|
|
}
|
|
}
|
|
#endif
|
|
|
|
|
|
#if defined(KE_MULTINODE)
|
|
|
|
NTSTATUS
|
|
KiNotNumaQueryProcessorNode(
|
|
IN ULONG ProcessorNumber,
|
|
OUT PUSHORT Identifier,
|
|
OUT PUCHAR Node
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine is a stub used on non NUMA systems to provide a
|
|
consistent method of determining the NUMA configuration rather
|
|
than checking for the presense of multiple nodes inline.
|
|
|
|
Arguments:
|
|
|
|
ProcessorNumber supplies the system logical processor number.
|
|
Identifier supplies the address of a variable to receive
|
|
the unique identifier for this processor.
|
|
NodeNumber supplies the address of a variable to receive
|
|
the number of the node this processor resides on.
|
|
|
|
Return Value:
|
|
|
|
Returns success.
|
|
|
|
--*/
|
|
|
|
{
|
|
*Identifier = (USHORT)ProcessorNumber;
|
|
*Node = 0;
|
|
return STATUS_SUCCESS;
|
|
}
|
|
|
|
#endif
|
|
|
|
#ifdef ALLOC_DATA_PRAGMA
|
|
#pragma data_seg()
|
|
#endif
|
|
|