mirror of https://github.com/tongzx/nt5src
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
543 lines
13 KiB
543 lines
13 KiB
/*++
|
|
|
|
Copyright (c) 1993 Microsoft Corporation
|
|
|
|
Module Name:
|
|
|
|
kiinit.c
|
|
|
|
Abstract:
|
|
|
|
This module implements architecture independent kernel initialization.
|
|
|
|
Author:
|
|
|
|
David N. Cutler 11-May-1993
|
|
|
|
Environment:
|
|
|
|
Kernel mode only.
|
|
|
|
Revision History:
|
|
|
|
--*/
|
|
|
|
#include "ki.h"
|
|
|
|
//
|
|
// External data.
|
|
//
|
|
|
|
extern KSPIN_LOCK AfdWorkQueueSpinLock;
|
|
extern KSPIN_LOCK CcBcbSpinLock;
|
|
extern KSPIN_LOCK CcMasterSpinLock;
|
|
extern KSPIN_LOCK CcVacbSpinLock;
|
|
extern KSPIN_LOCK CcWorkQueueSpinLock;
|
|
extern KSPIN_LOCK IopCancelSpinLock;
|
|
extern KSPIN_LOCK IopCompletionLock;
|
|
extern KSPIN_LOCK IopDatabaseLock;
|
|
extern KSPIN_LOCK IopVpbSpinLock;
|
|
extern KSPIN_LOCK KiContextSwapLock;
|
|
extern KSPIN_LOCK KiDispatcherLock;
|
|
extern KSPIN_LOCK NtfsStructLock;
|
|
extern KSPIN_LOCK MmPfnLock;
|
|
extern KSPIN_LOCK NonPagedPoolLock;
|
|
extern KSPIN_LOCK MmSystemSpaceLock;
|
|
|
|
#if DBG && !defined(_X86_)
|
|
extern KSPIN_LOCK KipGlobalAlignmentDatabaseLock;
|
|
#endif
|
|
//
|
|
//
|
|
// The following exist to allow testing of NUMA support.
|
|
//
|
|
|
|
ULONG KeVerifyNumaPageShift;
|
|
ULONG KeVerifyNumaAffinityShift;
|
|
ULONG KeVerifyNumaNodeCount;
|
|
ULONG KeVerifyNumaAffinity;
|
|
ULONG KeVerifyNumaPageMask;
|
|
|
|
// End numa test support variables.
|
|
|
|
|
|
//
|
|
// Put all code for kernel initialization in the INIT section. It will be
|
|
// deallocated by memory management when phase 1 initialization is completed.
|
|
//
|
|
|
|
#if defined(ALLOC_PRAGMA)
|
|
|
|
#pragma alloc_text(INIT, KeInitSystem)
|
|
#pragma alloc_text(INIT, KiInitQueuedSpinLocks)
|
|
#pragma alloc_text(INIT, KiInitSystem)
|
|
#pragma alloc_text(INIT, KiComputeReciprocal)
|
|
#pragma alloc_text(INIT, KeNumaInitialize)
|
|
|
|
#endif
|
|
|
|
BOOLEAN
|
|
KeInitSystem (
|
|
VOID
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This function initializes executive structures implemented by the
|
|
kernel.
|
|
|
|
N.B. This function is only called during phase 1 initialization.
|
|
|
|
Arguments:
|
|
|
|
None.
|
|
|
|
Return Value:
|
|
|
|
A value of TRUE is returned if initialization is successful. Otherwise,
|
|
a value of FALSE is returned.
|
|
|
|
--*/
|
|
|
|
{
|
|
|
|
//
|
|
// Perform platform dependent initialization.
|
|
//
|
|
|
|
return KiInitMachineDependent();
|
|
}
|
|
|
|
VOID
|
|
KiInitQueuedSpinLocks (
|
|
PKPRCB Prcb,
|
|
ULONG Number
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This function initializes the queued spinlock structures in the per
|
|
processor PRCB. This function is called once for each processor as
|
|
it is initialized in an MP system.
|
|
|
|
Arguments:
|
|
|
|
Prcb - Supplies a pointer to a PRCB.
|
|
|
|
Number - Supplies the number of respective processor.
|
|
|
|
Return Value:
|
|
|
|
None.
|
|
|
|
--*/
|
|
|
|
{
|
|
|
|
//
|
|
// Initialize queued spinlock structures.
|
|
//
|
|
|
|
Prcb->LockQueue[LockQueueDispatcherLock].Next = NULL;
|
|
Prcb->LockQueue[LockQueueDispatcherLock].Lock = &KiDispatcherLock;
|
|
|
|
Prcb->LockQueue[LockQueueContextSwapLock].Next = NULL;
|
|
Prcb->LockQueue[LockQueueContextSwapLock].Lock = &KiContextSwapLock;
|
|
|
|
Prcb->LockQueue[LockQueuePfnLock].Next = NULL;
|
|
Prcb->LockQueue[LockQueuePfnLock].Lock = &MmPfnLock;
|
|
|
|
Prcb->LockQueue[LockQueueSystemSpaceLock].Next = NULL;
|
|
Prcb->LockQueue[LockQueueSystemSpaceLock].Lock = &MmSystemSpaceLock;
|
|
|
|
Prcb->LockQueue[LockQueueBcbLock].Next = NULL;
|
|
Prcb->LockQueue[LockQueueBcbLock].Lock = &CcBcbSpinLock;
|
|
|
|
Prcb->LockQueue[LockQueueMasterLock].Next = NULL;
|
|
Prcb->LockQueue[LockQueueMasterLock].Lock = &CcMasterSpinLock;
|
|
|
|
Prcb->LockQueue[LockQueueVacbLock].Next = NULL;
|
|
Prcb->LockQueue[LockQueueVacbLock].Lock = &CcVacbSpinLock;
|
|
|
|
Prcb->LockQueue[LockQueueWorkQueueLock].Next = NULL;
|
|
Prcb->LockQueue[LockQueueWorkQueueLock].Lock = &CcWorkQueueSpinLock;
|
|
|
|
Prcb->LockQueue[LockQueueNonPagedPoolLock].Next = NULL;
|
|
Prcb->LockQueue[LockQueueNonPagedPoolLock].Lock = &NonPagedPoolLock;
|
|
|
|
Prcb->LockQueue[LockQueueIoCancelLock].Next = NULL;
|
|
Prcb->LockQueue[LockQueueIoCancelLock].Lock = &IopCancelSpinLock;
|
|
|
|
Prcb->LockQueue[LockQueueIoVpbLock].Next = NULL;
|
|
Prcb->LockQueue[LockQueueIoVpbLock].Lock = &IopVpbSpinLock;
|
|
|
|
Prcb->LockQueue[LockQueueIoDatabaseLock].Next = NULL;
|
|
Prcb->LockQueue[LockQueueIoDatabaseLock].Lock = &IopDatabaseLock;
|
|
|
|
Prcb->LockQueue[LockQueueIoCompletionLock].Next = NULL;
|
|
Prcb->LockQueue[LockQueueIoCompletionLock].Lock = &IopCompletionLock;
|
|
|
|
Prcb->LockQueue[LockQueueNtfsStructLock].Next = NULL;
|
|
Prcb->LockQueue[LockQueueNtfsStructLock].Lock = &NtfsStructLock;
|
|
|
|
Prcb->LockQueue[LockQueueAfdWorkQueueLock].Next = NULL;
|
|
Prcb->LockQueue[LockQueueAfdWorkQueueLock].Lock = &AfdWorkQueueSpinLock;
|
|
|
|
//
|
|
// If this is processor zero, then also initialize the queued spin lock
|
|
// home address.
|
|
//
|
|
|
|
if (Number == 0) {
|
|
KeInitializeSpinLock(&KiContextSwapLock);
|
|
KeInitializeSpinLock(&KiDispatcherLock);
|
|
KeInitializeSpinLock(&MmPfnLock);
|
|
KeInitializeSpinLock(&MmSystemSpaceLock);
|
|
KeInitializeSpinLock(&CcBcbSpinLock);
|
|
KeInitializeSpinLock(&CcMasterSpinLock);
|
|
KeInitializeSpinLock(&CcVacbSpinLock);
|
|
KeInitializeSpinLock(&CcWorkQueueSpinLock);
|
|
KeInitializeSpinLock(&IopCancelSpinLock);
|
|
KeInitializeSpinLock(&IopCompletionLock);
|
|
KeInitializeSpinLock(&IopDatabaseLock);
|
|
KeInitializeSpinLock(&IopVpbSpinLock);
|
|
KeInitializeSpinLock(&NonPagedPoolLock);
|
|
KeInitializeSpinLock(&NtfsStructLock);
|
|
KeInitializeSpinLock(&AfdWorkQueueSpinLock);
|
|
}
|
|
|
|
return;
|
|
}
|
|
|
|
VOID
|
|
KiInitSystem (
|
|
VOID
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This function initializes architecture independent kernel structures.
|
|
|
|
Arguments:
|
|
|
|
None.
|
|
|
|
Return Value:
|
|
|
|
None.
|
|
|
|
--*/
|
|
|
|
{
|
|
|
|
ULONG Index;
|
|
|
|
//
|
|
// Initialize dispatcher ready queue listheads.
|
|
//
|
|
|
|
for (Index = 0; Index < MAXIMUM_PRIORITY; Index += 1) {
|
|
InitializeListHead(&KiDispatcherReadyListHead[Index]);
|
|
}
|
|
|
|
//
|
|
// Initialize bug check callback listhead and spinlock.
|
|
//
|
|
|
|
InitializeListHead(&KeBugCheckCallbackListHead);
|
|
InitializeListHead(&KeBugCheckReasonCallbackListHead);
|
|
KeInitializeSpinLock(&KeBugCheckCallbackLock);
|
|
|
|
//
|
|
// Initialize the timer expiration DPC object.
|
|
//
|
|
|
|
KeInitializeDpc(&KiTimerExpireDpc,
|
|
(PKDEFERRED_ROUTINE)KiTimerExpiration, NIL);
|
|
|
|
//
|
|
// Initialize the profile listhead and profile locks
|
|
//
|
|
|
|
KeInitializeSpinLock(&KiProfileLock);
|
|
InitializeListHead(&KiProfileListHead);
|
|
|
|
#if DBG && !defined(_X86_)
|
|
|
|
//
|
|
// Initialize the global alignment fault database lock
|
|
//
|
|
|
|
KeInitializeSpinLock(&KipGlobalAlignmentDatabaseLock);
|
|
#endif
|
|
|
|
//
|
|
// Initialize the active profile source listhead.
|
|
//
|
|
|
|
InitializeListHead(&KiProfileSourceListHead);
|
|
|
|
//
|
|
// Initialize the timer table, the timer completion listhead, and the
|
|
// timer completion DPC.
|
|
//
|
|
|
|
for (Index = 0; Index < TIMER_TABLE_SIZE; Index += 1) {
|
|
InitializeListHead(&KiTimerTableListHead[Index]);
|
|
}
|
|
|
|
//
|
|
// Initialize the swap event, the process inswap listhead, the
|
|
// process outswap listhead, the kernel stack inswap listhead,
|
|
// the wait in listhead, and the wait out listhead.
|
|
//
|
|
|
|
KeInitializeEvent(&KiSwapEvent,
|
|
SynchronizationEvent,
|
|
FALSE);
|
|
|
|
KiProcessInSwapListHead.Next = NULL;
|
|
KiProcessOutSwapListHead.Next = NULL;
|
|
KiStackInSwapListHead.Next = NULL;
|
|
InitializeListHead(&KiWaitListHead);
|
|
|
|
//
|
|
// Initialize the system service descriptor table.
|
|
//
|
|
|
|
KeServiceDescriptorTable[0].Base = &KiServiceTable[0];
|
|
KeServiceDescriptorTable[0].Count = NULL;
|
|
KeServiceDescriptorTable[0].Limit = KiServiceLimit;
|
|
|
|
//
|
|
// The global pointer associated with the table base is placed just
|
|
// before the service table on the ia64.
|
|
//
|
|
|
|
#if defined(_IA64_)
|
|
|
|
KeServiceDescriptorTable[0].TableBaseGpOffset =
|
|
(LONG)(*(KiServiceTable-1) - (ULONG_PTR)KiServiceTable);
|
|
|
|
#endif
|
|
|
|
KeServiceDescriptorTable[0].Number = &KiArgumentTable[0];
|
|
for (Index = 1; Index < NUMBER_SERVICE_TABLES; Index += 1) {
|
|
KeServiceDescriptorTable[Index].Limit = 0;
|
|
}
|
|
|
|
//
|
|
// Copy the system service descriptor table to the shadow table
|
|
// which is used to record the Win32 system services.
|
|
//
|
|
|
|
RtlCopyMemory(KeServiceDescriptorTableShadow,
|
|
KeServiceDescriptorTable,
|
|
sizeof(KeServiceDescriptorTable));
|
|
|
|
//
|
|
// Initialize call performance data structures.
|
|
//
|
|
|
|
#if defined(_COLLECT_FLUSH_SINGLE_CALLDATA_)
|
|
|
|
ExInitializeCallData(&KiFlushSingleCallData);
|
|
|
|
#endif
|
|
|
|
#if defined(_COLLECT_SET_EVENT_CALLDATA_)
|
|
|
|
ExInitializeCallData(&KiSetEventCallData);
|
|
|
|
#endif
|
|
|
|
#if defined(_COLLECT_WAIT_SINGLE_CALLDATA_)
|
|
|
|
ExInitializeCallData(&KiWaitSingleCallData);
|
|
|
|
#endif
|
|
|
|
return;
|
|
}
|
|
|
|
LARGE_INTEGER
|
|
KiComputeReciprocal (
|
|
IN LONG Divisor,
|
|
OUT PCCHAR Shift
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This function computes the large integer reciprocal of the specified
|
|
value.
|
|
|
|
Arguments:
|
|
|
|
Divisor - Supplies the value for which the large integer reciprocal is
|
|
computed.
|
|
|
|
Shift - Supplies a pointer to a variable that receives the computed
|
|
shift count.
|
|
|
|
Return Value:
|
|
|
|
The large integer reciprocal is returned as the fucntion value.
|
|
|
|
--*/
|
|
|
|
{
|
|
|
|
LARGE_INTEGER Fraction;
|
|
LONG NumberBits;
|
|
LONG Remainder;
|
|
|
|
//
|
|
// Compute the large integer reciprocal of the specified value.
|
|
//
|
|
|
|
NumberBits = 0;
|
|
Remainder = 1;
|
|
Fraction.LowPart = 0;
|
|
Fraction.HighPart = 0;
|
|
while (Fraction.HighPart >= 0) {
|
|
NumberBits += 1;
|
|
Fraction.HighPart = (Fraction.HighPart << 1) | (Fraction.LowPart >> 31);
|
|
Fraction.LowPart <<= 1;
|
|
Remainder <<= 1;
|
|
if (Remainder >= Divisor) {
|
|
Remainder -= Divisor;
|
|
Fraction.LowPart |= 1;
|
|
}
|
|
}
|
|
|
|
if (Remainder != 0) {
|
|
if ((Fraction.LowPart == 0xffffffff) && (Fraction.HighPart == 0xffffffff)) {
|
|
Fraction.LowPart = 0;
|
|
Fraction.HighPart = 0x80000000;
|
|
NumberBits -= 1;
|
|
|
|
} else {
|
|
if (Fraction.LowPart == 0xffffffff) {
|
|
Fraction.LowPart = 0;
|
|
Fraction.HighPart += 1;
|
|
|
|
} else {
|
|
Fraction.LowPart += 1;
|
|
}
|
|
}
|
|
}
|
|
|
|
//
|
|
// Compute the shift count value and return the reciprocal fraction.
|
|
//
|
|
|
|
*Shift = (CCHAR)(NumberBits - 64);
|
|
return Fraction;
|
|
}
|
|
|
|
VOID
|
|
KeNumaInitialize (
|
|
VOID
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
Initialize ntos kernel structures needed to support NUMA.
|
|
|
|
Arguments:
|
|
|
|
None.
|
|
|
|
Return Value:
|
|
|
|
None.
|
|
|
|
--*/
|
|
|
|
{
|
|
|
|
#if defined(KE_MULTINODE)
|
|
|
|
NTSTATUS Status;
|
|
HAL_NUMA_TOPOLOGY_INTERFACE HalNumaInfo;
|
|
ULONG ReturnedLength;
|
|
|
|
extern PHALNUMAQUERYPROCESSORNODE KiQueryProcessorNode;
|
|
extern PHALNUMAPAGETONODE MmPageToNode;
|
|
|
|
//
|
|
// Mega Kludge: For testing purposes on NON-Numa systems,
|
|
// we can have non-numa MP systems report a NUMA configuration.
|
|
//
|
|
// Pass the information obtained from the registry to the HAL
|
|
// in the return info buffer.
|
|
//
|
|
|
|
if (KeVerifyNumaNodeCount &&
|
|
(KeVerifyNumaNodeCount < 8) &&
|
|
KeVerifyNumaAffinity &&
|
|
KeVerifyNumaAffinityShift &&
|
|
(KeVerifyNumaAffinityShift < 32) &&
|
|
KeVerifyNumaPageMask &&
|
|
(KeVerifyNumaPageMask < KeVerifyNumaNodeCount)&&
|
|
KeVerifyNumaPageShift &&
|
|
(KeVerifyNumaPageShift < 32) ) {
|
|
|
|
struct {
|
|
ULONG Nodes:3;
|
|
ULONG AffinityShift:6;
|
|
ULONG PageShift:6;
|
|
ULONG Signature:17;
|
|
ULONG Affinity;
|
|
ULONG Mask;
|
|
} Fake;
|
|
|
|
C_ASSERT(sizeof(Fake) <= sizeof(HalNumaInfo));
|
|
|
|
Fake.Signature = 0x15a5a;
|
|
Fake.PageShift = KeVerifyNumaPageShift;
|
|
Fake.AffinityShift = KeVerifyNumaAffinityShift;
|
|
Fake.Nodes = KeVerifyNumaNodeCount;
|
|
Fake.Affinity = KeVerifyNumaAffinity;
|
|
Fake.Mask = KeVerifyNumaPageMask;
|
|
|
|
RtlCopyMemory(&HalNumaInfo, &Fake, sizeof(Fake));
|
|
}
|
|
|
|
//
|
|
// End Mega Kludge.
|
|
//
|
|
|
|
Status = HalQuerySystemInformation (HalNumaTopologyInterface,
|
|
sizeof(HalNumaInfo),
|
|
&HalNumaInfo,
|
|
&ReturnedLength);
|
|
|
|
if (NT_SUCCESS(Status)) {
|
|
|
|
ASSERT (ReturnedLength == sizeof(HalNumaInfo));
|
|
ASSERT (HalNumaInfo.NumberOfNodes <= MAXIMUM_CCNUMA_NODES);
|
|
ASSERT (HalNumaInfo.QueryProcessorNode);
|
|
ASSERT (HalNumaInfo.PageToNode);
|
|
|
|
if (HalNumaInfo.NumberOfNodes > 1) {
|
|
KeNumberNodes = (UCHAR)HalNumaInfo.NumberOfNodes;
|
|
MmPageToNode = HalNumaInfo.PageToNode;
|
|
KiQueryProcessorNode = HalNumaInfo.QueryProcessorNode;
|
|
}
|
|
}
|
|
|
|
|
|
#endif
|
|
|
|
}
|