/*++
Copyright (c) 1993 Microsoft Corporation
Module Name:
kiinit.c
Abstract:
This module implements architecture independent kernel initialization.
Author:
David N. Cutler 11-May-1993
Environment:
Kernel mode only.
Revision History:
--*/
#include "ki.h"
//
// External data.
//
extern KSPIN_LOCK AfdWorkQueueSpinLock;
extern KSPIN_LOCK CcBcbSpinLock;
extern KSPIN_LOCK CcMasterSpinLock;
extern KSPIN_LOCK CcVacbSpinLock;
extern KSPIN_LOCK CcWorkQueueSpinLock;
extern KSPIN_LOCK IopCancelSpinLock;
extern KSPIN_LOCK IopCompletionLock;
extern KSPIN_LOCK IopDatabaseLock;
extern KSPIN_LOCK IopVpbSpinLock;
extern KSPIN_LOCK KiContextSwapLock;
extern KSPIN_LOCK KiDispatcherLock;
extern KSPIN_LOCK NtfsStructLock;
extern KSPIN_LOCK MmPfnLock;
extern KSPIN_LOCK NonPagedPoolLock;
extern KSPIN_LOCK MmSystemSpaceLock;
#if DBG && !defined(_X86_)
extern KSPIN_LOCK KipGlobalAlignmentDatabaseLock;
#endif
//
//
// The following exist to allow testing of NUMA support.
//
ULONG KeVerifyNumaPageShift;
ULONG KeVerifyNumaAffinityShift;
ULONG KeVerifyNumaNodeCount;
ULONG KeVerifyNumaAffinity;
ULONG KeVerifyNumaPageMask;
// End numa test support variables.
//
// Put all code for kernel initialization in the INIT section. It will be
// deallocated by memory management when phase 1 initialization is completed.
//
#if defined(ALLOC_PRAGMA)
#pragma alloc_text(INIT, KeInitSystem)
#pragma alloc_text(INIT, KiInitQueuedSpinLocks)
#pragma alloc_text(INIT, KiInitSystem)
#pragma alloc_text(INIT, KiComputeReciprocal)
#pragma alloc_text(INIT, KeNumaInitialize)
#endif
BOOLEAN
KeInitSystem (
VOID
)
/*++
Routine Description:
This function initializes executive structures implemented by the
kernel.
N.B. This function is only called during phase 1 initialization.
Arguments:
None.
Return Value:
A value of TRUE is returned if initialization is successful. Otherwise,
a value of FALSE is returned.
--*/
{
//
// Perform platform dependent initialization.
//
return KiInitMachineDependent();
}
VOID
KiInitQueuedSpinLocks (
PKPRCB Prcb,
ULONG Number
)
/*++
Routine Description:
This function initializes the queued spinlock structures in the per
processor PRCB. This function is called once for each processor as
it is initialized in an MP system.
Arguments:
Prcb - Supplies a pointer to a PRCB.
Number - Supplies the number of respective processor.
Return Value:
None.
--*/
{
//
// Initialize queued spinlock structures.
//
Prcb->LockQueue[LockQueueDispatcherLock].Next = NULL;
Prcb->LockQueue[LockQueueDispatcherLock].Lock = &KiDispatcherLock;
Prcb->LockQueue[LockQueueContextSwapLock].Next = NULL;
Prcb->LockQueue[LockQueueContextSwapLock].Lock = &KiContextSwapLock;
Prcb->LockQueue[LockQueuePfnLock].Next = NULL;
Prcb->LockQueue[LockQueuePfnLock].Lock = &MmPfnLock;
Prcb->LockQueue[LockQueueSystemSpaceLock].Next = NULL;
Prcb->LockQueue[LockQueueSystemSpaceLock].Lock = &MmSystemSpaceLock;
Prcb->LockQueue[LockQueueBcbLock].Next = NULL;
Prcb->LockQueue[LockQueueBcbLock].Lock = &CcBcbSpinLock;
Prcb->LockQueue[LockQueueMasterLock].Next = NULL;
Prcb->LockQueue[LockQueueMasterLock].Lock = &CcMasterSpinLock;
Prcb->LockQueue[LockQueueVacbLock].Next = NULL;
Prcb->LockQueue[LockQueueVacbLock].Lock = &CcVacbSpinLock;
Prcb->LockQueue[LockQueueWorkQueueLock].Next = NULL;
Prcb->LockQueue[LockQueueWorkQueueLock].Lock = &CcWorkQueueSpinLock;
Prcb->LockQueue[LockQueueNonPagedPoolLock].Next = NULL;
Prcb->LockQueue[LockQueueNonPagedPoolLock].Lock = &NonPagedPoolLock;
Prcb->LockQueue[LockQueueIoCancelLock].Next = NULL;
Prcb->LockQueue[LockQueueIoCancelLock].Lock = &IopCancelSpinLock;
Prcb->LockQueue[LockQueueIoVpbLock].Next = NULL;
Prcb->LockQueue[LockQueueIoVpbLock].Lock = &IopVpbSpinLock;
Prcb->LockQueue[LockQueueIoDatabaseLock].Next = NULL;
Prcb->LockQueue[LockQueueIoDatabaseLock].Lock = &IopDatabaseLock;
Prcb->LockQueue[LockQueueIoCompletionLock].Next = NULL;
Prcb->LockQueue[LockQueueIoCompletionLock].Lock = &IopCompletionLock;
Prcb->LockQueue[LockQueueNtfsStructLock].Next = NULL;
Prcb->LockQueue[LockQueueNtfsStructLock].Lock = &NtfsStructLock;
Prcb->LockQueue[LockQueueAfdWorkQueueLock].Next = NULL;
Prcb->LockQueue[LockQueueAfdWorkQueueLock].Lock = &AfdWorkQueueSpinLock;
//
// If this is processor zero, then also initialize the queued spin lock
// home address.
//
if (Number == 0) {
KeInitializeSpinLock(&KiContextSwapLock);
KeInitializeSpinLock(&KiDispatcherLock);
KeInitializeSpinLock(&MmPfnLock);
KeInitializeSpinLock(&MmSystemSpaceLock);
KeInitializeSpinLock(&CcBcbSpinLock);
KeInitializeSpinLock(&CcMasterSpinLock);
KeInitializeSpinLock(&CcVacbSpinLock);
KeInitializeSpinLock(&CcWorkQueueSpinLock);
KeInitializeSpinLock(&IopCancelSpinLock);
KeInitializeSpinLock(&IopCompletionLock);
KeInitializeSpinLock(&IopDatabaseLock);
KeInitializeSpinLock(&IopVpbSpinLock);
KeInitializeSpinLock(&NonPagedPoolLock);
KeInitializeSpinLock(&NtfsStructLock);
KeInitializeSpinLock(&AfdWorkQueueSpinLock);
}
return;
}
VOID
KiInitSystem (
VOID
)
/*++
Routine Description:
This function initializes architecture independent kernel structures.
Arguments:
None.
Return Value:
None.
--*/
{
ULONG Index;
//
// Initialize dispatcher ready queue listheads.
//
for (Index = 0; Index < MAXIMUM_PRIORITY; Index += 1) {
InitializeListHead(&KiDispatcherReadyListHead[Index]);
}
//
// Initialize bug check callback listhead and spinlock.
//
InitializeListHead(&KeBugCheckCallbackListHead);
InitializeListHead(&KeBugCheckReasonCallbackListHead);
KeInitializeSpinLock(&KeBugCheckCallbackLock);
//
// Initialize the timer expiration DPC object.
//
KeInitializeDpc(&KiTimerExpireDpc,
(PKDEFERRED_ROUTINE)KiTimerExpiration, NIL);
//
// Initialize the profile listhead and profile locks
//
KeInitializeSpinLock(&KiProfileLock);
InitializeListHead(&KiProfileListHead);
#if DBG && !defined(_X86_)
//
// Initialize the global alignment fault database lock
//
KeInitializeSpinLock(&KipGlobalAlignmentDatabaseLock);
#endif
//
// Initialize the active profile source listhead.
//
InitializeListHead(&KiProfileSourceListHead);
//
// Initialize the timer table, the timer completion listhead, and the
// timer completion DPC.
//
for (Index = 0; Index < TIMER_TABLE_SIZE; Index += 1) {
InitializeListHead(&KiTimerTableListHead[Index]);
}
//
// Initialize the swap event, the process inswap listhead, the
// process outswap listhead, the kernel stack inswap listhead,
// the wait in listhead, and the wait out listhead.
//
KeInitializeEvent(&KiSwapEvent,
SynchronizationEvent,
FALSE);
KiProcessInSwapListHead.Next = NULL;
KiProcessOutSwapListHead.Next = NULL;
KiStackInSwapListHead.Next = NULL;
InitializeListHead(&KiWaitListHead);
//
// Initialize the system service descriptor table.
//
KeServiceDescriptorTable[0].Base = &KiServiceTable[0];
KeServiceDescriptorTable[0].Count = NULL;
KeServiceDescriptorTable[0].Limit = KiServiceLimit;
//
// The global pointer associated with the table base is placed just
// before the service table on the ia64.
//
#if defined(_IA64_)
KeServiceDescriptorTable[0].TableBaseGpOffset =
(LONG)(*(KiServiceTable-1) - (ULONG_PTR)KiServiceTable);
#endif
KeServiceDescriptorTable[0].Number = &KiArgumentTable[0];
for (Index = 1; Index < NUMBER_SERVICE_TABLES; Index += 1) {
KeServiceDescriptorTable[Index].Limit = 0;
}
//
// Copy the system service descriptor table to the shadow table
// which is used to record the Win32 system services.
//
RtlCopyMemory(KeServiceDescriptorTableShadow,
KeServiceDescriptorTable,
sizeof(KeServiceDescriptorTable));
//
// Initialize call performance data structures.
//
#if defined(_COLLECT_FLUSH_SINGLE_CALLDATA_)
ExInitializeCallData(&KiFlushSingleCallData);
#endif
#if defined(_COLLECT_SET_EVENT_CALLDATA_)
ExInitializeCallData(&KiSetEventCallData);
#endif
#if defined(_COLLECT_WAIT_SINGLE_CALLDATA_)
ExInitializeCallData(&KiWaitSingleCallData);
#endif
return;
}
LARGE_INTEGER
KiComputeReciprocal (
IN LONG Divisor,
OUT PCCHAR Shift
)
/*++
Routine Description:
This function computes the large integer reciprocal of the specified
value.
Arguments:
Divisor - Supplies the value for which the large integer reciprocal is
computed.
Shift - Supplies a pointer to a variable that receives the computed
shift count.
Return Value:
The large integer reciprocal is returned as the fucntion value.
--*/
{
LARGE_INTEGER Fraction;
LONG NumberBits;
LONG Remainder;
//
// Compute the large integer reciprocal of the specified value.
//
NumberBits = 0;
Remainder = 1;
Fraction.LowPart = 0;
Fraction.HighPart = 0;
while (Fraction.HighPart >= 0) {
NumberBits += 1;
Fraction.HighPart = (Fraction.HighPart << 1) | (Fraction.LowPart >> 31);
Fraction.LowPart <<= 1;
Remainder <<= 1;
if (Remainder >= Divisor) {
Remainder -= Divisor;
Fraction.LowPart |= 1;
}
}
if (Remainder != 0) {
if ((Fraction.LowPart == 0xffffffff) && (Fraction.HighPart == 0xffffffff)) {
Fraction.LowPart = 0;
Fraction.HighPart = 0x80000000;
NumberBits -= 1;
} else {
if (Fraction.LowPart == 0xffffffff) {
Fraction.LowPart = 0;
Fraction.HighPart += 1;
} else {
Fraction.LowPart += 1;
}
}
}
//
// Compute the shift count value and return the reciprocal fraction.
//
*Shift = (CCHAR)(NumberBits - 64);
return Fraction;
}
VOID
KeNumaInitialize (
VOID
)
/*++
Routine Description:
Initialize ntos kernel structures needed to support NUMA.
Arguments:
None.
Return Value:
None.
--*/
{
#if defined(KE_MULTINODE)
NTSTATUS Status;
HAL_NUMA_TOPOLOGY_INTERFACE HalNumaInfo;
ULONG ReturnedLength;
extern PHALNUMAQUERYPROCESSORNODE KiQueryProcessorNode;
extern PHALNUMAPAGETONODE MmPageToNode;
//
// Mega Kludge: For testing purposes on NON-Numa systems,
// we can have non-numa MP systems report a NUMA configuration.
//
// Pass the information obtained from the registry to the HAL
// in the return info buffer.
//
if (KeVerifyNumaNodeCount &&
(KeVerifyNumaNodeCount < 8) &&
KeVerifyNumaAffinity &&
KeVerifyNumaAffinityShift &&
(KeVerifyNumaAffinityShift < 32) &&
KeVerifyNumaPageMask &&
(KeVerifyNumaPageMask < KeVerifyNumaNodeCount)&&
KeVerifyNumaPageShift &&
(KeVerifyNumaPageShift < 32) ) {
struct {
ULONG Nodes:3;
ULONG AffinityShift:6;
ULONG PageShift:6;
ULONG Signature:17;
ULONG Affinity;
ULONG Mask;
} Fake;
C_ASSERT(sizeof(Fake) <= sizeof(HalNumaInfo));
Fake.Signature = 0x15a5a;
Fake.PageShift = KeVerifyNumaPageShift;
Fake.AffinityShift = KeVerifyNumaAffinityShift;
Fake.Nodes = KeVerifyNumaNodeCount;
Fake.Affinity = KeVerifyNumaAffinity;
Fake.Mask = KeVerifyNumaPageMask;
RtlCopyMemory(&HalNumaInfo, &Fake, sizeof(Fake));
}
//
// End Mega Kludge.
//
Status = HalQuerySystemInformation (HalNumaTopologyInterface,
sizeof(HalNumaInfo),
&HalNumaInfo,
&ReturnedLength);
if (NT_SUCCESS(Status)) {
ASSERT (ReturnedLength == sizeof(HalNumaInfo));
ASSERT (HalNumaInfo.NumberOfNodes <= MAXIMUM_CCNUMA_NODES);
ASSERT (HalNumaInfo.QueryProcessorNode);
ASSERT (HalNumaInfo.PageToNode);
if (HalNumaInfo.NumberOfNodes > 1) {
KeNumberNodes = (UCHAR)HalNumaInfo.NumberOfNodes;
MmPageToNode = HalNumaInfo.PageToNode;
KiQueryProcessorNode = HalNumaInfo.QueryProcessorNode;
}
}
#endif
}