|
|
/*++
Copyright (c) 2000 Microsoft Corporation
Module Name:
allproc.c
Abstract:
This module allocates and initializes kernel resources required to start a new processor, and passes a complete process state structure to the hal to obtain a new processor.
Author:
David N. Cutler (davec) 5-May-2000
Environment:
Kernel mode only.
Revision History:
--*/
#include "ki.h"
#include "pool.h"
//
// Define local macros.
//
#define ROUNDUP16(x) (((x) + 15) & ~15)
//
// Define prototypes for forward referenced functions.
//
#if !defined(NT_UP)
VOID KiCopyDescriptorMemory ( IN PKDESCRIPTOR Source, IN PKDESCRIPTOR Destination, IN PVOID Base );
NTSTATUS KiNotNumaQueryProcessorNode ( IN ULONG ProcessorNumber, OUT PUSHORT Identifier, OUT PUCHAR Node );
VOID KiSetDescriptorBase ( IN USHORT Selector, IN PKGDTENTRY64 GdtBase, IN PVOID Base );
PHALNUMAQUERYPROCESSORNODE KiQueryProcessorNode = KiNotNumaQueryProcessorNode;
//
// Statically allocate enough KNODE structures to allow memory management
// to allocate pages by node during system initialization. As processors
// are brought online, real KNODE structures are allocated in the correct
// memory for the node.
//
#pragma data_seg("INITDATA")
KNODE KiNodeInit[MAXIMUM_CCNUMA_NODES];
#pragma data_seg()
#pragma alloc_text(INIT, KiAllProcessorsStarted)
#pragma alloc_text(INIT, KiCopyDescriptorMemory)
#pragma alloc_text(INIT, KiNotNumaQueryProcessorNode)
#pragma alloc_text(INIT, KiSetDescriptorBase)
#endif // !defined(NT_UP)
#pragma alloc_text(INIT, KeStartAllProcessors)
ULONG KiBarrierWait = 0;
VOID KeStartAllProcessors ( VOID )
/*++
Routine Description:
This function is called during phase 1 initialization on the master boot processor to start all of the other registered processors.
Arguments:
None.
Return Value:
None.
--*/
{
#if !defined(NT_UP)
ULONG AllocationSize; PUCHAR Base; PKPCR CurrentPcr = KeGetPcr(); PVOID DataBlock; PVOID DpcStack; PKGDTENTRY64 GdtBase; ULONG GdtOffset; ULONG IdtOffset; PVOID KernelStack; PKNODE Node; UCHAR NodeNumber; UCHAR Number; PKPCR PcrBase; USHORT ProcessorId; KPROCESSOR_STATE ProcessorState; NTSTATUS Status; PKTSS64 SysTssBase; PETHREAD Thread;
//
// Do not start additional processors if the RELOCATEPHYSICAL loader
// switch has been specified.
//
if (KeLoaderBlock->LoadOptions != NULL) { if (strstr(KeLoaderBlock->LoadOptions, "RELOCATEPHYSICAL") != NULL) { return; } }
//
// If processor zero is not on node zero, then move it to the appropriate
// node.
//
if (KeNumberNodes > 1) { Status = KiQueryProcessorNode(0, &ProcessorId, &NodeNumber); if (NT_SUCCESS(Status)) {
//
// Adjust the data structures to reflect that P0 is not on Node 0.
//
if (NodeNumber != 0) {
ASSERT(KeNodeBlock[0] == &KiNode0);
KeNodeBlock[0]->ProcessorMask &= ~1; KiNodeInit[0] = *KeNodeBlock[0]; KeNodeBlock[0] = &KiNodeInit[0]; KiNode0 = *KeNodeBlock[NodeNumber]; KeNodeBlock[NodeNumber] = &KiNode0; KeNodeBlock[NodeNumber]->ProcessorMask |= 1; } } }
//
// Calculate the size of the per processor data structures.
//
// This includes:
//
// PCR (including the PRCB)
// System TSS
// Idle Thread Object
// Double Fault/NMI Panic Stack
// Machine Check Stack
// GDT
// IDT
//
// If this is a multinode system, the KNODE structure is also allocated.
//
// A DPC and Idle stack are also allocated, but they are done separately.
//
AllocationSize = ROUNDUP16(sizeof(KPCR)) + ROUNDUP16(sizeof(KTSS64)) + ROUNDUP16(sizeof(ETHREAD)) + ROUNDUP16(DOUBLE_FAULT_STACK_SIZE) + ROUNDUP16(KERNEL_MCA_EXCEPTION_STACK_SIZE);
AllocationSize += ROUNDUP16(sizeof(KNODE));
//
// Save the offset of the GDT in the allocation structure and add in
// the size of the GDT.
//
GdtOffset = AllocationSize; AllocationSize += CurrentPcr->Prcb.ProcessorState.SpecialRegisters.Gdtr.Limit + 1;
//
// Save the offset of the IDT in the allocation structure and add in
// the size of the IDT.
//
IdtOffset = AllocationSize; AllocationSize += CurrentPcr->Prcb.ProcessorState.SpecialRegisters.Idtr.Limit + 1;
//
// If the registered number of processors is greater than the maximum
// number of processors supported, then only allow the maximum number
// of supported processors.
//
if (KeRegisteredProcessors > MAXIMUM_PROCESSORS) { KeRegisteredProcessors = MAXIMUM_PROCESSORS; }
//
// Set barrier that will prevent any other processor from entering the
// idle loop until all processors have been started.
//
KiBarrierWait = 1;
//
// Initialize the fixed part of the processor state that will be used to
// start processors. Each processor starts in the system initialization
// code with address of the loader parameter block as an argument.
//
RtlZeroMemory(&ProcessorState, sizeof(KPROCESSOR_STATE)); ProcessorState.ContextFrame.Rcx = (ULONG64)KeLoaderBlock; ProcessorState.ContextFrame.Rip = (ULONG64)KiSystemStartup; ProcessorState.ContextFrame.SegCs = KGDT64_R0_CODE; ProcessorState.ContextFrame.SegDs = KGDT64_R3_DATA | RPL_MASK; ProcessorState.ContextFrame.SegEs = KGDT64_R3_DATA | RPL_MASK; ProcessorState.ContextFrame.SegFs = KGDT64_R3_CMTEB | RPL_MASK; ProcessorState.ContextFrame.SegGs = KGDT64_R3_DATA | RPL_MASK; ProcessorState.ContextFrame.SegSs = KGDT64_NULL;
//
// Loop trying to start a new processors until a new processor can't be
// started or an allocation failure occurs.
//
Number = 0; while ((ULONG)KeNumberProcessors < KeRegisteredProcessors) { Number += 1; Status = KiQueryProcessorNode(Number, &ProcessorId, &NodeNumber); if (!NT_SUCCESS(Status)) {
//
// No such processor, advance to next.
//
continue; }
Node = KeNodeBlock[NodeNumber];
//
// Allocate memory for the new processor specific data. If the
// allocation fails, then stop starting processors.
//
DataBlock = MmAllocateIndependentPages(AllocationSize, NodeNumber); if (DataBlock == NULL) { break; }
//
// Allocate a pool tag table for the new processor.
//
if (ExCreatePoolTagTable(Number, NodeNumber) == NULL) { MmFreeIndependentPages(DataBlock, AllocationSize); break; }
//
// Zero the allocated memory.
//
Base = (PUCHAR)DataBlock; RtlZeroMemory(DataBlock, AllocationSize);
//
// Copy and initialize the GDT for the next processor.
//
KiCopyDescriptorMemory(&CurrentPcr->Prcb.ProcessorState.SpecialRegisters.Gdtr, &ProcessorState.SpecialRegisters.Gdtr, Base + GdtOffset);
GdtBase = (PKGDTENTRY64)ProcessorState.SpecialRegisters.Gdtr.Base;
//
// Copy and initialize the IDT for the next processor.
//
KiCopyDescriptorMemory(&CurrentPcr->Prcb.ProcessorState.SpecialRegisters.Idtr, &ProcessorState.SpecialRegisters.Idtr, Base + IdtOffset);
//
// Set the PCR base address for the next processor and set the
// processor number.
//
// N.B. The PCR address is passed to the next processor by computing
// the containing address with respect to the PRCB.
//
PcrBase = (PKPCR)Base; PcrBase->Number = Number; PcrBase->Prcb.Number = Number; Base += ROUNDUP16(sizeof(KPCR));
//
// Set the system TSS descriptor base for the next processor.
//
SysTssBase = (PKTSS64)Base; KiSetDescriptorBase(KGDT64_SYS_TSS / 16, GdtBase, SysTssBase); Base += ROUNDUP16(sizeof(KTSS64));
//
// Initialize the panic stack address for double fault and NMI.
//
Base += DOUBLE_FAULT_STACK_SIZE; SysTssBase->Ist[TSS_IST_PANIC] = (ULONG64)Base;
//
// Initialize the machine check stack address.
//
Base += KERNEL_MCA_EXCEPTION_STACK_SIZE; SysTssBase->Ist[TSS_IST_MCA] = (ULONG64)Base;
//
// Idle Thread thread object.
//
Thread = (PETHREAD)Base; Base += ROUNDUP16(sizeof(ETHREAD));
//
// Set other special registers in the processor state.
//
ProcessorState.SpecialRegisters.Cr0 = ReadCR0(); ProcessorState.SpecialRegisters.Cr3 = ReadCR3(); ProcessorState.ContextFrame.EFlags = 0; ProcessorState.SpecialRegisters.Tr = KGDT64_SYS_TSS; GdtBase[KGDT64_SYS_TSS / 16].Bytes.Flags1 = 0x89; ProcessorState.SpecialRegisters.Cr4 = ReadCR4();
//
// Allocate a kernel stack and a DPC stack for the next processor.
//
KernelStack = MmCreateKernelStack(FALSE, NodeNumber); if (KernelStack == NULL) { MmFreeIndependentPages(DataBlock, AllocationSize); break; }
DpcStack = MmCreateKernelStack(FALSE, NodeNumber); if (DpcStack == NULL) { MmDeleteKernelStack(KernelStack, FALSE); MmFreeIndependentPages(DataBlock, AllocationSize); break; }
//
// Initialize the kernel stack for the system TSS.
//
SysTssBase->Rsp0 = (ULONG64)KernelStack - sizeof(PVOID) * 4; ProcessorState.ContextFrame.Rsp = (ULONG64)KernelStack;
//
// If this is the first processor on this node, then use the space
// allocated for KNODE as the KNODE.
//
if (KeNodeBlock[NodeNumber] == &KiNodeInit[NodeNumber]) { Node = (PKNODE)Base; *Node = KiNodeInit[NodeNumber]; KeNodeBlock[NodeNumber] = Node; }
Base += ROUNDUP16(sizeof(KNODE)); PcrBase->Prcb.ParentNode = Node;
//
// Adjust the loader block so it has the next processor state. Ensure
// that the KernelStack has space for home registers for up to four
// parameters.
//
KeLoaderBlock->KernelStack = (ULONG64)DpcStack - (sizeof(PVOID) * 4); KeLoaderBlock->Thread = (ULONG64)Thread; KeLoaderBlock->Prcb = (ULONG64)(&PcrBase->Prcb);
//
// Attempt to start the next processor. If a processor cannot be
// started, then deallocate memory and stop starting processors.
//
if (HalStartNextProcessor(KeLoaderBlock, &ProcessorState) == 0) { ExDeletePoolTagTable (Number); MmFreeIndependentPages(DataBlock, AllocationSize); MmDeleteKernelStack(KernelStack, FALSE); MmDeleteKernelStack(DpcStack, FALSE); break; }
Node->ProcessorMask |= AFFINITY_MASK(Number);
//
// Wait for processor to initialize.
//
while (*((volatile ULONG64 *)&KeLoaderBlock->Prcb) != 0) { KeYieldProcessor(); } }
//
// All processors have been stated.
//
KiAllProcessorsStarted();
//
// Reset and synchronize the performance counters of all processors, by
// applying a null adjustment to the interrupt time
//
KeAdjustInterruptTime(0);
//
// Allow all processors that were started to enter the idle loop and
// begin execution.
//
KiBarrierWait = 0;
#endif // !defined(NT_UP)
return; }
#if !defined(NT_UP)
VOID KiSetDescriptorBase ( IN USHORT Selector, IN PKGDTENTRY64 GdtBase, IN PVOID Base )
/*++
Routine Description:
This function sets the base address of a descriptor to the specified base address.
Arguments:
Selector - Supplies the selector for the descriptor.
GdtBase - Supplies a pointer to the GDT.
Base - Supplies a pointer to the base address.
Return Value:
None.
--*/
{
GdtBase = &GdtBase[Selector]; GdtBase->BaseLow = (USHORT)((ULONG64)Base); GdtBase->Bytes.BaseMiddle = (UCHAR)((ULONG64)Base >> 16); GdtBase->Bytes.BaseHigh = (UCHAR)((ULONG64)Base >> 24); GdtBase->BaseUpper = (ULONG)((ULONG64)Base >> 32); return; }
VOID KiCopyDescriptorMemory ( IN PKDESCRIPTOR Source, IN PKDESCRIPTOR Destination, IN PVOID Base )
/*++
Routine Description:
This function copies the specified descriptor memory to the new memory and initializes a descriptor for the new memory.
Arguments:
Source - Supplies a pointer to the source descriptor that describes the memory to copy.
Destination - Supplies a pointer to the destination descriptor to be initialized.
Base - Supplies a pointer to the new memory.
Return Value:
None.
--*/
{
Destination->Limit = Source->Limit; Destination->Base = Base; RtlCopyMemory(Base, Source->Base, Source->Limit + 1); return; }
VOID KiAllProcessorsStarted ( VOID )
/*++
Routine Description:
This routine is called once all processors in the system have been started.
Arguments:
None.
Return Value:
None.
--*/
{
ULONG i;
//
// Make sure there are no references to the temporary nodes used during
// initialization.
//
for (i = 0; i < KeNumberNodes; i += 1) { if (KeNodeBlock[i] == &KiNodeInit[i]) {
//
// No processor started on this node so no new node structure has
// been allocated. This is possible if the node contains memory
// only or IO busses. At this time we need to allocate a permanent
// node structure for the node.
//
KeNodeBlock[i] = ExAllocatePoolWithTag(NonPagedPool, sizeof(KNODE), ' eK');
if (KeNodeBlock[i]) { *KeNodeBlock[i] = KiNodeInit[i]; } }
//
// Set the node number.
//
KeNodeBlock[i]->NodeNumber = (UCHAR)i; }
for (i = KeNumberNodes; i < MAXIMUM_CCNUMA_NODES; i += 1) { KeNodeBlock[i] = NULL; }
if (KeNumberNodes == 1) {
//
// For Non NUMA machines, Node 0 gets all processors.
//
KeNodeBlock[0]->ProcessorMask = KeActiveProcessors; }
return; }
NTSTATUS KiNotNumaQueryProcessorNode ( IN ULONG ProcessorNumber, OUT PUSHORT Identifier, OUT PUCHAR Node )
/*++
Routine Description:
This routine is a stub used on non NUMA systems to provide a consistent method of determining the NUMA configuration rather than checking for the presense of multiple nodes inline.
Arguments:
ProcessorNumber supplies the system logical processor number. Identifier supplies the address of a variable to receive the unique identifier for this processor. NodeNumber supplies the address of a variable to receive the number of the node this processor resides on.
Return Value:
Returns success.
--*/
{ *Identifier = (USHORT)ProcessorNumber; *Node = 0; return STATUS_SUCCESS; }
#endif // !defined(NT_UP)
|