/*++ Copyright (c) 1989 Microsoft Corporation Module Name: ki.h Abstract: This module contains the private (internal) header file for the kernel. Author: David N. Cutler (davec) 28-Feb-1989 Revision History: --*/ #ifndef _KI_ #define _KI_ #pragma warning(disable:4214) // bit field types other than int #pragma warning(disable:4201) // nameless struct/union #pragma warning(disable:4324) // alignment sensitive to declspec #pragma warning(disable:4127) // condition expression is constant #pragma warning(disable:4115) // named type definition in parentheses #pragma warning(disable:4706) // assignment within conditional expression #pragma warning(disable:4206) // translation unit empty #include "ntos.h" #include "stdio.h" #include "stdlib.h" #include "zwapi.h" // // Private (internal) constant definitions. // // Priority increment value definitions // #define ALERT_INCREMENT 2 // Alerted unwait priority increment #define BALANCE_INCREMENT 10 // Balance set priority increment #define RESUME_INCREMENT 0 // Resume thread priority increment #define TIMER_EXPIRE_INCREMENT 0 // Timer expiration priority increment // // Define time critical priority class base. // #define TIME_CRITICAL_PRIORITY_BOUND 14 // // Define NIL pointer value. // #define NIL (PVOID)NULL // Null pointer to void // // Define macros which are used in the kernel only // // Clear member in set // #define ClearMember(Member, Set) \ Set = Set & (~((ULONG_PTR)1 << (Member))) // // Set member in set // #define SetMember(Member, Set) \ Set = Set | ((ULONG_PTR)1 << (Member)) #ifdef CAPKERN_SYNCH_POINTS VOID __cdecl CAP_Log_NInt_Clothed ( IN ULONG Bcode_Bts_Scount, ... ); #endif FORCEINLINE SCHAR KiComputeNewPriority ( IN PKTHREAD Thread, IN SCHAR Adjustment ) /*++ Routine Description: This function computes a new priority for the specified thread by subtracting the priority decrement value plus the adjustment from the thread priority. Arguments: Thread - Supplies a pointer to a thread object. Adjustment - Supplies an additional adjustment value. Return Value: The new priority is returned as the function value. --*/ { SCHAR Priority; // // Compute the new thread priority. // ASSERT((Thread->PriorityDecrement >= 0) && (Thread->PriorityDecrement <= Thread->Priority)); ASSERT((Thread->Priority < LOW_REALTIME_PRIORITY) ? TRUE : (Thread->PriorityDecrement == 0)); Priority = Thread->Priority; if (Priority < LOW_REALTIME_PRIORITY) { Priority = Priority - Thread->PriorityDecrement - Adjustment; if (Priority < Thread->BasePriority) { Priority = Thread->BasePriority; } Thread->PriorityDecrement = 0; } return Priority; } VOID FASTCALL KiExitDispatcher ( IN KIRQL OldIrql ); FORCEINLINE KIRQL FASTCALL KiAcquireSpinLockForDpc ( IN PKSPIN_LOCK SpinLock ) /*++ Routine Description: This function conditionally raises IRQL to DISPATCH_LEVEL and acquires the specified spin lock. N.B. The conditional IRQL raise is predicated on whether a thread DPC is enabled. Arguments: SpinLock - Supplies the address of a spin lock. Return Value: If the IRQL is raised, then the previous IRQL is returned. Otherwise, zero is returned. --*/ { KIRQL OldIrql; // // If the DPC thread is active, then raise IRQL and acquire the specified // spin lock. Otherwise, zero the previous IRQL and acquire the specified // spin lock at DISPATCH_LEVEL. // if (KeGetCurrentPrcb()->DpcThreadActive != FALSE) { KeAcquireSpinLock(SpinLock, &OldIrql); } else { ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL); OldIrql = DISPATCH_LEVEL; KeAcquireSpinLockAtDpcLevel(SpinLock); } return OldIrql; } FORCEINLINE VOID FASTCALL KiReleaseSpinLockForDpc ( IN PKSPIN_LOCK SpinLock, IN KIRQL OldIrql ) /*++ Routine Description: This function releases the specified spin lock and conditionally lowers IRQL to its previous value. N.B. The conditional IRQL raise is predicated on whether a thread DPC is enabled. Arguments: SpinLock - Supplies the address of a spin lock. OldIrql - Supplies the previous IRQL. Return Value: None. --*/ { // // If the DPC thread is active, then release the specified spin lock and // lower IRQL to its previous value. Otherwise, release specified spin // lock from DISPATCH_LEVEL. // if (KeGetCurrentPrcb()->DpcThreadActive != FALSE) { KeReleaseSpinLock(SpinLock, OldIrql); } else { ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL); KeReleaseSpinLockFromDpcLevel(SpinLock); } return; } FORCEINLINE VOID FASTCALL KiAcquireInStackQueuedSpinLockForDpc ( IN PKSPIN_LOCK SpinLock, IN PKLOCK_QUEUE_HANDLE LockHandle ) /*++ Routine Description: This function conditionally raises IRQL to DISPATCH_LEVEL and acquires the specified in-stack spin lock. N.B. The conditional IRQL raise is predicated on whether a thread DPC is enabled. Arguments: SpinLock - Supplies the address of a spin lock. LockHandle - Supplies the address of a lock handle. Return Value: None. --*/ { // // If the DPC thread is active, then raise IRQL and acquire the specified // in-stack spin lock. Otherwise, acquire the specified in-stack spin lock // at DISPATCH_LEVEL. // if (KeGetCurrentPrcb()->DpcThreadActive != FALSE) { KeAcquireInStackQueuedSpinLock(SpinLock, LockHandle); } else { ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL); KeAcquireInStackQueuedSpinLockAtDpcLevel(SpinLock, LockHandle); } return; } FORCEINLINE VOID FASTCALL KiReleaseInStackQueuedSpinLockForDpc ( IN PKLOCK_QUEUE_HANDLE LockHandle ) /*++ Routine Description: This function releases the specified in-stack spin lock and conditionally lowers IRQL to its previous value. N.B. The conditional IRQL raise is predicated on whether a thread DPC is enabled. Arguments: LockHandle - Supplies the address of a lock handle. Return Value: None. --*/ { // // If threaded DPCs are enabled, then release the specified in-stack // spin lock and lower IRQL to its previous value. Otherwise, release // the specified in-stack spin lock from DISPATCH_LEVEL. // if (KeGetCurrentPrcb()->DpcThreadActive != FALSE) { KeReleaseInStackQueuedSpinLock(LockHandle); } else { ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL); KeReleaseInStackQueuedSpinLockFromDpcLevel(LockHandle); } return; } FORCEINLINE VOID KzAcquireSpinLock ( IN PKSPIN_LOCK SpinLock ) /*++ Routine Description: This function acquires a spin lock at the current IRQL. Arguments: SpinLock - Supplies a pointer to an spin lock. Return Value: None. --*/ { #if !defined(NT_UP) #ifdef CAPKERN_SYNCH_POINTS ULONG Count = 0; CAP_Log_NInt_Clothed(0x00010101, (PVOID)SpinLock); #endif #if defined(_WIN64) #if defined(_AMD64_) while (InterlockedBitTestAndSet64((LONG64 *)SpinLock, 0)) { #else while (InterlockedExchangeAcquire64((PLONGLONG)SpinLock, 1) != 0) { #endif #else while (InterlockedExchange((PLONG)SpinLock, 1) != 0) { #endif do { #ifdef CAPKERN_SYNCH_POINTS Count += 1; #endif KeYieldProcessor(); #if defined(_AMD64_) KeMemoryBarrierWithoutFence(); } while (BitTest64((LONG64 *)SpinLock, 0)); #else } while (*(volatile LONG_PTR *)SpinLock != 0); #endif } #ifdef CAPKERN_SYNCH_POINTS if (Count != 0) { CAP_Log_NInt_Clothed(0x00020102, Count, (PVOID)SpinLock); } #endif #else UNREFERENCED_PARAMETER(SpinLock); #endif // !defined(NT_UP) return; } FORCEINLINE VOID KiAcquirePrcbLock ( IN PKPRCB Prcb ) /*++ Routine Description: This routine acquires the PRCB lock for the specified processor. N.B. This routine must be called from an IRQL greater than or equal to dispatch level. Arguments: Prcb - Supplies a pointer to a processor control block. Return Value: None. --*/ { ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL); KzAcquireSpinLock(&Prcb->PrcbLock); return; } FORCEINLINE VOID KiAcquireTwoPrcbLocks ( IN PKPRCB FirstPrcb, IN PKPRCB SecondPrcb ) /*++ Routine Description: This routine acquires the specified PRCB locks in address order. N.B. This routine must be called from an IRQL greater than or equal to dispatch level. Arguments: FirstPrcb - Supplies a pointer to a processor control block. SecondPrcb - Supplies a pointer to a processor control block. Return Value: None. --*/ { ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL); if (FirstPrcb < SecondPrcb) { KzAcquireSpinLock(&FirstPrcb->PrcbLock); KzAcquireSpinLock(&SecondPrcb->PrcbLock); } else { if (FirstPrcb != SecondPrcb) { KzAcquireSpinLock(&SecondPrcb->PrcbLock); } KzAcquireSpinLock(&FirstPrcb->PrcbLock); } return; } FORCEINLINE VOID KiReleasePrcbLock ( IN PKPRCB Prcb ) /*++ Routine Description: This routine release the PRCB lock for the specified processor. N.B. This routine must be called from an IRQL greater than or equal to dispatch level. Arguments: Prcb - Supplies a pointer to a processor control block. Return Value: None. --*/ { ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL); #if !defined(NT_UP) #ifdef CAPKERN_SYNCH_POINTS CAP_Log_NInt_Clothed(0x00010107, (PVOID)&Prcb->PrcbLock); #endif ASSERT(Prcb->PrcbLock != 0); #if defined (_X86_) InterlockedAnd ((volatile LONG *)&Prcb->PrcbLock, 0); #else KeMemoryBarrierWithoutFence(); *((volatile ULONG_PTR *)&Prcb->PrcbLock) = 0; #endif #else UNREFERENCED_PARAMETER(Prcb); #endif return; } FORCEINLINE VOID KiReleaseTwoPrcbLocks ( IN PKPRCB FirstPrcb, IN PKPRCB SecondPrcb ) /*++ Routine Description: This routine releases the specified PRCB locks. N.B. This routine must be called from an IRQL greater than or equal to dispatch level. Arguments: FirstPrcb - Supplies a pointer to a processor control block. SecondPrcb - Supplies a pointer to a processor control block. Return Value: None. --*/ { ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL); #if !defined(NT_UP) #ifdef CAPKERN_SYNCH_POINTS CAP_Log_NInt_Clothed(0x00010107, (PVOID)&FirstPrcb->PrcbLock); #endif KiReleasePrcbLock (FirstPrcb); if (FirstPrcb != SecondPrcb) { KiReleasePrcbLock (SecondPrcb); } #ifdef CAPKERN_SYNCH_POINTS CAP_Log_NInt_Clothed(0x00010107, (PVOID)&SecondPrcb->PrcbLock); #endif #else UNREFERENCED_PARAMETER(FirstPrcb); UNREFERENCED_PARAMETER(SecondPrcb); #endif return; } FORCEINLINE VOID KiAcquireThreadLock ( IN PKTHREAD Thread ) /*++ Routine Description: This routine acquires the thread lock for the specified thread. N.B. This routine must be called from an IRQL greater than or equal to dispatch level. Arguments: Thread - Supplies a pointer to a thread object. Return Value: None. --*/ { ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL); KzAcquireSpinLock(&Thread->ThreadLock); return; } FORCEINLINE VOID KiReleaseThreadLock ( IN PKTHREAD Thread ) /*++ Routine Description: This routine releases the thread lock for the specified thread. N.B. This routine must be called from an IRQL greater than or equal to dispatch level. Arguments: Thread - Supplies a pointer to a thread object. Return Value: None. --*/ { ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL); #if !defined(NT_UP) KeMemoryBarrierWithoutFence(); #if defined (_X86_) InterlockedAnd ((volatile LONG *)&Thread->ThreadLock, 0); #else *((volatile ULONG_PTR *)&Thread->ThreadLock) = 0; #endif #else UNREFERENCED_PARAMETER(Thread); #endif return; } FORCEINLINE VOID KiClearIdleSummary ( IN KAFFINITY Mask ) /*++ Routine Description: This function interlocked clears the specified mask into the current idle summary. Arguments: Mask - Supplies the affinity mask to merge. Return Value: None. --*/ { #if defined(NT_UP) KiIdleSummary &= ~Mask; #else #if defined(_X86_) InterlockedAnd((volatile LONG *)&KiIdleSummary, ~(LONG)Mask); #else InterlockedAnd64((volatile LONG64 *)&KiIdleSummary, ~(LONG64)Mask); #endif #endif return; } FORCEINLINE VOID KiSetIdleSummary ( IN KAFFINITY Mask ) /*++ Routine Description: This function interlocked merges the specified mask into the current idle summary. Arguments: Mask - Supplies the affinity mask to merge. Return Value: None. --*/ { #if defined(NT_UP) KiIdleSummary |= Mask; #else #if defined(_X86_) InterlockedOr((volatile LONG *)&KiIdleSummary, (LONG)Mask); #else InterlockedOr64((volatile LONG64 *)&KiIdleSummary, (LONG64)Mask); #endif #endif return; } extern volatile KAFFINITY KiIdleSMTSummary; FORCEINLINE VOID KiClearSMTSummary ( IN KAFFINITY Mask ) /*++ Routine Description: This function interlocked clears the specified mask into the current SMT summary. Arguments: Mask - Supplies the affinity mask to merge. Return Value: None. --*/ { #if defined(NT_SMT) #if defined(_X86_) InterlockedAnd((volatile LONG *)&KiIdleSMTSummary, ~(LONG)Mask); #else InterlockedAnd64((volatile LONG64 *)&KiIdleSMTSummary, ~(LONG64)Mask); #endif #else UNREFERENCED_PARAMETER(Mask); #endif return; } FORCEINLINE VOID KiSetSMTSummary ( IN KAFFINITY Mask ) /*++ Routine Description: This function interlocked merges the specified mask into the current SMT summary. Arguments: Mask - Supplies the affinity mask to merge. Return Value: None. --*/ { #if defined(NT_SMT) #if defined(_X86_) InterlockedOr((volatile LONG *)&KiIdleSMTSummary, (LONG)Mask); #else InterlockedOr64((volatile LONG64 *)&KiIdleSMTSummary, (LONG64)Mask); #endif #else UNREFERENCED_PARAMETER(Mask); #endif return; } FORCEINLINE VOID KiBoostPriorityThread ( IN PKTHREAD Thread, IN KPRIORITY Increment ) /*++ Routine Description: This function boosts the priority of the specified thread using the same algorithm used when a thread gets a boost from a wait operation. Arguments: Thread - Supplies a pointer to a dispatcher object of type thread. Increment - Supplies the priority increment that is to be applied to the thread's priority. Return Value: None. --*/ { KPRIORITY NewPriority; PKPROCESS Process; // // If the thread is not a real time thread and does not already // have an unusual boost, then boost the priority as specified. // KiAcquireThreadLock(Thread); if ((Thread->Priority < LOW_REALTIME_PRIORITY) && (Thread->PriorityDecrement == 0)) { NewPriority = Thread->BasePriority + Increment; if (NewPriority > Thread->Priority) { if (NewPriority >= LOW_REALTIME_PRIORITY) { NewPriority = LOW_REALTIME_PRIORITY - 1; } Process = Thread->ApcState.Process; Thread->Quantum = Process->ThreadQuantum; KiSetPriorityThread(Thread, NewPriority); } } KiReleaseThreadLock(Thread); return; } BOOLEAN KiHandleNmi ( VOID ); FORCEINLINE LOGICAL KiIsKernelStackSwappable ( IN KPROCESSOR_MODE WaitMode, IN PKTHREAD Thread ) /*++ Routine Description: This function determines whether the kernel stack is swappabel for the the specified thread in a wait operation. Arguments: WaitMode - Supplies the processor mode of the wait operation. Thread - Supplies a pointer to a dispatcher object of type thread. Return Value: If the kernel stack for the specified thread is swappable, then TRUE is returned. Otherwise, FALSE is returned. --*/ { return ((WaitMode != KernelMode) && (Thread->EnableStackSwap != FALSE) && (Thread->Priority < (LOW_REALTIME_PRIORITY + 9))); } VOID FASTCALL KiRetireDpcList ( PKPRCB Prcb ); FORCEINLINE VOID FASTCALL KiUnlockDispatcherDatabase ( IN KIRQL OldIrql ) /*++ Routine Description: This function unlocks the dispatcher database and exits the scheduler. Arguments: OldIrql - Supplies the previous IRQL. Return Value: None. --*/ { KiUnlockDispatcherDatabaseFromSynchLevel(); KiExitDispatcher(OldIrql); return; } // // Private (internal) structure definitions. // // APC Parameter structure. // typedef struct _KAPC_RECORD { PKNORMAL_ROUTINE NormalRoutine; PVOID NormalContext; PVOID SystemArgument1; PVOID SystemArgument2; } KAPC_RECORD, *PKAPC_RECORD; // // Executive initialization. // VOID ExpInitializeExecutive ( IN ULONG Number, IN PLOADER_PARAMETER_BLOCK LoaderBlock ); // // Interprocessor interrupt function definitions. // // Define immediate interprocessor commands. // #define IPI_APC 1 // APC interrupt request #define IPI_DPC 2 // DPC interrupt request #define IPI_FREEZE 4 // freeze execution request #define IPI_PACKET_READY 8 // packet ready request #define IPI_SYNCH_REQUEST 16 // reverse stall packet request // // Define interprocess interrupt types. // typedef ULONG KIPI_REQUEST; #if NT_INST #define IPI_INSTRUMENT_COUNT(a,b) KiIpiCounts[a].b++; #else #define IPI_INSTRUMENT_COUNT(a,b) #endif #if defined(_AMD64_) || defined(_IA64_) ULONG KiIpiProcessRequests ( VOID ); #endif // defined(_AMD64_) || defined(_IA64_) VOID FASTCALL KiIpiSend ( IN KAFFINITY TargetProcessors, IN KIPI_REQUEST Request ); VOID KiIpiSendPacket ( IN KAFFINITY TargetProcessors, IN PKIPI_WORKER WorkerFunction, IN PVOID Parameter1, IN PVOID Parameter2, IN PVOID Parameter3 ); VOID FASTCALL KiIpiSignalPacketDone ( IN PKIPI_CONTEXT SignalDone ); FORCEINLINE VOID KiIpiStallOnPacketTargets ( KAFFINITY TargetSet ) /*++ Routine Description: This function waits until the specified set of processors have signaled their completion of a requested function. N.B. The exact protocol used between the source and the target of an interprocessor request is not specified. Minimally the source must construct an appropriate packet and send the packet to a set of specified targets. Each target receives the address of the packet address as an argument, and minimally must clear the packet address when the mutually agreed upon protocol allows. The target has three options: 1. Capture necessary information, release the source by clearing the packet address, execute the request in parallel with the source, and return from the interrupt. 2. Execute the request in series with the source, release the source by clearing the packet address, and return from the interrupt. 3. Execute the request in series with the source, release the source, wait for a reply from the source based on a packet parameter, and return from the interrupt. This function is provided to enable the source to synchronize with the target for cases 2 and 3 above. N.B. There is no support for method 3 above. Arguments: TargetSet - Supplies the the target set of IPI processors. Return Value: None. --*/ { KAFFINITY volatile *Barrier; PKPRCB Prcb; // // If there is one and only one bit set in the target set, then wait // on the target set. Otherwise, wait on the packet barrier. // Prcb = KeGetCurrentPrcb(); Barrier = &Prcb->TargetSet; if ((TargetSet & (TargetSet - 1)) != 0) { Barrier = &Prcb->PacketBarrier; } while (*Barrier != 0) { KeYieldProcessor(); } return; } // // Private (internal) function definitions. // VOID FASTCALL KiUnwaitThread ( IN PKTHREAD Thread, IN LONG_PTR WaitStatus, IN KPRIORITY Increment ); FORCEINLINE VOID KiActivateWaiterQueue ( IN PRKQUEUE Queue ) /*++ Routine Description: This function is called when the current thread is about to enter a wait state and is currently processing a queue entry. The current number of threads processign entries for the queue is decrement and an attempt is made to activate another thread if the current count is less than the maximum count, there is a waiting thread, and the queue is not empty. N.B. It is possible that this function is called on one processor holding the dispatcher database lock while the state of the specified queue object is being modified on another processor while holding only the queue object lock. This does not cause a problem since holding the queue object lock ensures that there are no waiting threads. Arguments: Queue - Supplies a pointer to a dispatcher object of type event. Return Value: None. --*/ { PRLIST_ENTRY Entry; PRKTHREAD Thread; PRKWAIT_BLOCK WaitBlock; PRLIST_ENTRY WaitEntry; // // Decrement the current count of active threads and check if another // thread can be activated. If the current number of active threads is // less than the target maximum number of threads, there is a entry in // in the queue, and a thread is waiting, then remove the entry from the // queue, decrement the number of entries in the queue, and unwait the // respectiive thread. // Queue->CurrentCount -= 1; if (Queue->CurrentCount < Queue->MaximumCount) { Entry = Queue->EntryListHead.Flink; WaitEntry = Queue->Header.WaitListHead.Blink; if ((Entry != &Queue->EntryListHead) && (WaitEntry != &Queue->Header.WaitListHead)) { RemoveEntryList(Entry); Entry->Flink = NULL; Queue->Header.SignalState -= 1; WaitBlock = CONTAINING_RECORD(WaitEntry, KWAIT_BLOCK, WaitListEntry); Thread = WaitBlock->Thread; KiUnwaitThread(Thread, (LONG_PTR)Entry, 0); } } return; } VOID KiAllProcessorsStarted ( VOID ); VOID KiApcInterrupt ( VOID ); NTSTATUS KiCallUserMode ( IN PVOID *OutputBuffer, IN PULONG OutputLength ); typedef struct { ULONGLONG Adjustment; LARGE_INTEGER NewCount; volatile LONG KiNumber; volatile LONG HalNumber; volatile LONG Barrier; } ADJUST_INTERRUPT_TIME_CONTEXT, *PADJUST_INTERRUPT_TIME_CONTEXT; VOID KiCalibrateTimeAdjustment ( PADJUST_INTERRUPT_TIME_CONTEXT Adjust ); VOID KiChainedDispatch ( VOID ); #if DBG VOID KiCheckTimerTable ( IN ULARGE_INTEGER SystemTime ); #endif LARGE_INTEGER KiComputeReciprocal ( IN LONG Divisor, OUT PCCHAR Shift ); extern LARGE_INTEGER KiTimeIncrementReciprocal; extern CCHAR KiTimeIncrementShiftCount; #if defined(_AMD64_) __forceinline ULONG KiComputeTimerTableIndex ( IN LARGE_INTEGER Interval, IN LARGE_INTEGER CurrentTime, IN PKTIMER Timer ) /*++ Routine Description: This function computes the timer table index for the specified timer object and stores the due time in the timer object. N.B. The interval parameter is guaranteed to be negative since it is expressed as relative time. The formula for due time calculation is: Due Time = Current Time - Interval The formula for the index calculation is: Index = (Due Time / Maximum time increment) & (Table Size - 1) The time increment division is performed using reciprocal multiplication. N.B. The maximum time increment determines the interval corresponding to a tick. Arguments: Interval - Supplies the relative time at which the timer is to expire. CurrentCount - Supplies the current system tick count. Timer - Supplies a pointer to a dispatch object of type timer. Return Value: The time table index is returned as the function value and the due time is stored in the timer object. --*/ { ULONG64 DueTime; ULONG64 HighTime; ULONG Index; // // Compute the due time of the timer. // DueTime = CurrentTime.QuadPart - Interval.QuadPart; Timer->DueTime.QuadPart = DueTime; // // Compute the timer table index. // HighTime = UnsignedMultiplyHigh(DueTime, KiTimeIncrementReciprocal.QuadPart); Index = (ULONG)(HighTime >> KiTimeIncrementShiftCount); return (Index & (TIMER_TABLE_SIZE - 1)); } #else ULONG KiComputeTimerTableIndex ( IN LARGE_INTEGER Interval, IN LARGE_INTEGER CurrentCount, IN PKTIMER Timer ); #endif PLARGE_INTEGER FASTCALL KiComputeWaitInterval ( IN PLARGE_INTEGER OriginalTime, IN PLARGE_INTEGER DueTime, IN OUT PLARGE_INTEGER NewTime ); NTSTATUS KiContinue ( IN PCONTEXT ContextRecord, IN PKEXCEPTION_FRAME ExceptionFrame, IN PKTRAP_FRAME TrapFrame ); VOID KiDeliverApc ( IN KPROCESSOR_MODE PreviousMode, IN PKEXCEPTION_FRAME ExceptionFrame, IN PKTRAP_FRAME TrapFrame ); VOID KiDispatchException ( IN PEXCEPTION_RECORD ExceptionRecord, IN PKEXCEPTION_FRAME ExceptionFrame, IN PKTRAP_FRAME TrapFrame, IN KPROCESSOR_MODE PreviousMode, IN BOOLEAN FirstChance ); VOID KiExecuteDpc ( IN PVOID Context ); KCONTINUE_STATUS KiSetDebugProcessor ( IN PKTRAP_FRAME TrapFrame, IN PKEXCEPTION_FRAME ExceptionFrame, IN KPROCESSOR_MODE PreviousMode ); ULONG KiCopyInformation ( IN OUT PEXCEPTION_RECORD ExceptionRecord1, IN PEXCEPTION_RECORD ExceptionRecord2 ); VOID KiDispatchInterrupt ( VOID ); VOID FASTCALL KiDeferredReadyThread ( IN PKTHREAD Thread ); PKTHREAD FASTCALL KiFindReadyThread ( IN ULONG Processor, IN PKPRCB Prcb ); VOID KiFloatingDispatch ( VOID ); FORCEINLINE VOID KiSetTbFlushTimeStampBusy ( VOID ) /*++ Routine Description: This function sets the TB flush time stamp busy by setting the high order bit of the TB flush time stamp. All readers of the time stamp value will spin until the bit is cleared. Arguments: None. Return Value: None. --*/ { LONG Value; // // While the TB flush time stamp counter is being updated the high // order bit of the time stamp value is set. Otherwise, the bit is // clear. // do { do { } while ((Value = KiTbFlushTimeStamp) < 0); // // Attempt to set the high order bit. // } while (InterlockedCompareExchange((PLONG)&KiTbFlushTimeStamp, Value | 0x80000000, Value) != Value); return; } FORCEINLINE VOID KiClearTbFlushTimeStampBusy ( VOID ) /*++ Routine Description: This function ckears the TB flush time stamp busy by clearing the high order bit of the TB flush time stamp and incrementing the low 32-bit value. N.B. It is assumed that the high order bit of the time stamp value is set on entry to this routine. Arguments: None. Return Value: None. --*/ { LONG Value; // // Get the current TB flush time stamp value, compute the next value, // and store the result clearing the busy bit. // Value = (KiTbFlushTimeStamp + 1) & 0x7fffffff; InterlockedExchange((PLONG)&KiTbFlushTimeStamp, Value); return; } PULONG KiGetUserModeStackAddress ( VOID ); VOID KiInitializeContextThread ( IN PKTHREAD Thread, IN PKSYSTEM_ROUTINE SystemRoutine, IN PKSTART_ROUTINE StartRoutine OPTIONAL, IN PVOID StartContext OPTIONAL, IN PCONTEXT ContextFrame OPTIONAL ); VOID KiInitializeKernel ( IN PKPROCESS Process, IN PKTHREAD Thread, IN PVOID IdleStack, IN PKPRCB Prcb, IN CCHAR Number, IN PLOADER_PARAMETER_BLOCK LoaderBlock ); VOID KiInitSpinLocks ( PKPRCB Prcb, ULONG Number ); VOID KiInitSystem ( VOID ); BOOLEAN KiInitMachineDependent ( VOID ); VOID KiInitializeUserApc ( IN PKEXCEPTION_FRAME ExceptionFrame, IN PKTRAP_FRAME TrapFrame, IN PKNORMAL_ROUTINE NormalRoutine, IN PVOID NormalContext, IN PVOID SystemArgument1, IN PVOID SystemArgument2 ); FORCEINLINE VOID FASTCALL KiInsertDeferredReadyList ( IN PKTHREAD Thread ) /*++ Routine Description: This function pushes an entry onto the current processor's deferred ready list. Arguments: Thread - Supplies a pointer to a thread object. Return Value: None. --*/ { // // On the MP system, insert the specified thread in the deferred ready // list. On the UP system, ready the thread immediately. // #if defined(NT_UP) Thread->State = DeferredReady; Thread->DeferredProcessor = 0; KiDeferredReadyThread(Thread); #else PKPRCB Prcb; Prcb = KeGetCurrentPrcb(); Thread->State = DeferredReady; Thread->DeferredProcessor = Prcb->Number; PushEntryList(&Prcb->DeferredReadyListHead, &Thread->SwapListEntry); #endif return; } LONG FASTCALL KiInsertQueue ( IN PKQUEUE Queue, IN PLIST_ENTRY Entry, IN BOOLEAN Head ); VOID FASTCALL KiInsertQueueApc ( IN PKAPC Apc, IN KPRIORITY Increment ); LOGICAL FASTCALL KiInsertTreeTimer ( IN PKTIMER Timer, IN LARGE_INTEGER Interval ); VOID KiInterruptDispatch ( VOID ); VOID KiInterruptDispatchRaise ( IN PKINTERRUPT Interrupt ); VOID KiInterruptDispatchSame ( IN PKINTERRUPT Interrupt ); VOID KiPassiveRelease ( VOID ); VOID FASTCALL KiProcessDeferredReadyList ( IN PKPRCB CurrentPrcb ); VOID KiQuantumEnd ( VOID ); NTSTATUS KiRaiseException ( IN PEXCEPTION_RECORD ExceptionRecord, IN PCONTEXT ContextRecord, IN PKEXCEPTION_FRAME ExceptionFrame, IN PKTRAP_FRAME TrapFrame, IN BOOLEAN FirstChance ); VOID FASTCALL KiReadyThread ( IN PKTHREAD Thread ); FORCEINLINE VOID KxQueueReadyThread ( IN PKTHREAD Thread, IN PKPRCB Prcb ) /*++ Routine Description: This function inserts the previously current thread in the current processor's dispatcher ready queues if the thread can run on the curent processor. Otherwise, the specified thread is readied for execution. N.B. This function is called with the current PRCB lock held and returns with the PRCB lock not held. Arguments: Thread - Supplies a pointer to a thread object. Prcb - Supplies a pointer to a the current PRCB. Return Value: None. --*/ { BOOLEAN Preempted; KPRIORITY Priority; ASSERT(Prcb == KeGetCurrentPrcb()); ASSERT(Thread->State == Running); ASSERT(Thread->NextProcessor == Prcb->Number); // // If the thread can run on the specified processor, then insert the // thread in the appropriate dispatcher ready queue for the specified // processor and release the specified PRCB lock. Otherwise, release // the specified PRCB lock and ready the thread for execution. // #if !defined(NT_UP) if ((Thread->Affinity & Prcb->SetMember) != 0) { #endif Thread->State = Ready; Preempted = Thread->Preempted; Thread->Preempted = FALSE; Thread->WaitTime = KiQueryLowTickCount(); Priority = Thread->Priority; ASSERT((Priority >= 0) && (Priority <= HIGH_PRIORITY)); if (Preempted != FALSE) { InsertHeadList(&Prcb->DispatcherReadyListHead[Priority], &Thread->WaitListEntry); } else { InsertTailList(&Prcb->DispatcherReadyListHead[Priority], &Thread->WaitListEntry); } Prcb->ReadySummary |= PRIORITY_MASK(Priority); ASSERT(Priority == Thread->Priority); KiReleasePrcbLock(Prcb); #if !defined(NT_UP) } else { Thread->State = DeferredReady; Thread->DeferredProcessor = Prcb->Number; KiReleasePrcbLock(Prcb); KiDeferredReadyThread(Thread); } #endif return; } LOGICAL FASTCALL KiReinsertTreeTimer ( IN PKTIMER Timer, IN ULARGE_INTEGER DueTime ); #if DBG #define KiRemoveTreeTimer(Timer) \ (Timer)->Header.Inserted = FALSE; \ RemoveEntryList(&(Timer)->TimerListEntry); \ (Timer)->TimerListEntry.Flink = NULL; \ (Timer)->TimerListEntry.Blink = NULL #else #define KiRemoveTreeTimer(Timer) \ (Timer)->Header.Inserted = FALSE; \ RemoveEntryList(&(Timer)->TimerListEntry) #endif #if defined(NT_UP) #define KiRequestApcInterrupt(Processor) KiRequestSoftwareInterrupt(APC_LEVEL) #else #define KiRequestApcInterrupt(Processor) \ if (KeGetCurrentProcessorNumber() == Processor) { \ KiRequestSoftwareInterrupt(APC_LEVEL); \ } else { \ KiIpiSend(AFFINITY_MASK(Processor), IPI_APC); \ } #endif #if defined(NT_UP) #define KiRequestDispatchInterrupt(Processor) #else #define KiRequestDispatchInterrupt(Processor) \ if (KeGetCurrentProcessorNumber() != Processor) { \ KiIpiSend(AFFINITY_MASK(Processor), IPI_DPC); \ } #endif PKTHREAD FASTCALL KiSelectNextThread ( IN PKPRCB Prcb ); KAFFINITY FASTCALL KiSetAffinityThread ( IN PKTHREAD Thread, IN KAFFINITY Affinity ); FORCEINLINE VOID KiSetContextSwapBusy ( IN PKTHREAD Thread ) /*++ Routine Description: This routine sets context swap busy for the specified thread. Arguments: Thread - Supplies a pointer to a dispatcher object of type thread. Return Value: None. --*/ { #if !defined(NT_UP) ASSERT(Thread->SwapBusy == FALSE); Thread->SwapBusy = TRUE; #else UNREFERENCED_PARAMETER(Thread); #endif return; } FORCEINLINE VOID KiSetContextSwapIdle ( IN PKTHREAD Thread ) /*++ Routine Description: This routine sets context swap idle for the specified thread. Arguments: Thread - Supplies a pointer to a dispatcher object of type thread. Return Value: None. --*/ { #if !defined(NT_UP) ASSERT(Thread->SwapBusy == TRUE); Thread->SwapBusy = FALSE; #else UNREFERENCED_PARAMETER(Thread); #endif return; } VOID KiSetSystemTime ( IN PLARGE_INTEGER NewTime, OUT PLARGE_INTEGER OldTime ); VOID KiSuspendNop ( IN struct _KAPC *Apc, IN OUT PKNORMAL_ROUTINE *NormalRoutine, IN OUT PVOID *NormalContext, IN OUT PVOID *SystemArgument1, IN OUT PVOID *SystemArgument2 ); VOID KiSuspendRundown ( IN PKAPC Apc ); VOID KiSuspendThread ( IN PVOID NormalContext, IN PVOID SystemArgument1, IN PVOID SystemArgument2 ); BOOLEAN KiSwapProcess ( IN PKPROCESS NewProcess, IN PKPROCESS OldProcess ); LONG_PTR FASTCALL KiSwapThread ( IN PKTHREAD OldThread, IN PKPRCB CurrentPrcb ); VOID KiThreadStartup ( IN PVOID StartContext ); VOID KiTimerExpiration ( IN PKDPC Dpc, IN PVOID DeferredContext, IN PVOID SystemArgument1, IN PVOID SystemArgument2 ); VOID FASTCALL KiTimerListExpire ( IN PLIST_ENTRY ExpiredListHead, IN KIRQL OldIrql ); VOID KiUnexpectedInterrupt ( VOID ); FORCEINLINE VOID FASTCALL KiUnlinkThread ( IN PRKTHREAD Thread, IN LONG_PTR WaitStatus ) /*++ Routine Description: This function unlinks a thread from the appropriate wait queues and sets the thread's wait completion status. Arguments: Thread - Supplies a pointer to a dispatcher object of type thread. WaitStatus - Supplies the wait completion status. Return Value: None. --*/ { PKQUEUE Queue; PKTIMER Timer; PRKWAIT_BLOCK WaitBlock; // // Set wait completion status, remove wait blocks from object wait // lists, and remove thread from wait list. // Thread->WaitStatus |= WaitStatus; WaitBlock = Thread->WaitBlockList; do { RemoveEntryList(&WaitBlock->WaitListEntry); WaitBlock = WaitBlock->NextWaitBlock; } while (WaitBlock != Thread->WaitBlockList); if (Thread->WaitListEntry.Flink != NULL) { RemoveEntryList(&Thread->WaitListEntry); } // // If thread timer is still active, then cancel thread timer. // Timer = &Thread->Timer; if (Timer->Header.Inserted != FALSE) { KiRemoveTreeTimer(Timer); } // // If the thread is processing a queue entry, then increment the // count of currently active threads. // Queue = Thread->Queue; if (Queue != NULL) { Queue->CurrentCount += 1; } return; } VOID KiUserApcDispatcher ( IN PVOID NormalContext, IN PVOID SystemArgument1, IN PVOID SystemArgument2, IN PKNORMAL_ROUTINE NormalRoutine ); VOID KiUserExceptionDispatcher ( IN PEXCEPTION_RECORD ExceptionRecord, IN PCONTEXT ContextFrame ); VOID KiVerifyReadySummary ( PKPRCB Prcb ); BOOLEAN FASTCALL KiSwapContext ( IN PKTHREAD OldThread, IN PKTHREAD NewThread ); // // VOID // FASTCALL // KiWaitSatisfyAny ( // IN PKMUTANT Object, // IN PKTHREAD Thread // ) // // // Routine Description: // // This function satisfies a wait for any type of object and performs // any side effects that are necessary. // // Arguments: // // Object - Supplies a pointer to a dispatcher object. // // Thread - Supplies a pointer to a dispatcher object of type thread. // // Return Value: // // None. // #define KiWaitSatisfyAny(_Object_, _Thread_) { \ if (((_Object_)->Header.Type & DISPATCHER_OBJECT_TYPE_MASK) == EventSynchronizationObject) { \ (_Object_)->Header.SignalState = 0; \ \ } else if ((_Object_)->Header.Type == SemaphoreObject) { \ (_Object_)->Header.SignalState -= 1; \ \ } else if ((_Object_)->Header.Type == MutantObject) { \ (_Object_)->Header.SignalState -= 1; \ if ((_Object_)->Header.SignalState == 0) { \ (_Thread_)->KernelApcDisable = (_Thread_)->KernelApcDisable - (_Object_)->ApcDisable; \ (_Object_)->OwnerThread = (_Thread_); \ if ((_Object_)->Abandoned == TRUE) { \ (_Object_)->Abandoned = FALSE; \ (_Thread_)->WaitStatus = STATUS_ABANDONED; \ } \ \ InsertHeadList((_Thread_)->MutantListHead.Blink, \ &(_Object_)->MutantListEntry); \ } \ } \ } // // VOID // FASTCALL // KiWaitSatisfyMutant ( // IN PKMUTANT Object, // IN PKTHREAD Thread // ) // // // Routine Description: // // This function satisfies a wait for a mutant object. // // Arguments: // // Object - Supplies a pointer to a dispatcher object. // // Thread - Supplies a pointer to a dispatcher object of type thread. // // Return Value: // // None. // #define KiWaitSatisfyMutant(_Object_, _Thread_) { \ (_Object_)->Header.SignalState -= 1; \ if ((_Object_)->Header.SignalState == 0) { \ (_Thread_)->KernelApcDisable = (_Thread_)->KernelApcDisable - (_Object_)->ApcDisable; \ (_Object_)->OwnerThread = (_Thread_); \ if ((_Object_)->Abandoned == TRUE) { \ (_Object_)->Abandoned = FALSE; \ (_Thread_)->WaitStatus = STATUS_ABANDONED; \ } \ \ InsertHeadList((_Thread_)->MutantListHead.Blink, \ &(_Object_)->MutantListEntry); \ } \ } // // VOID // FASTCALL // KiWaitSatisfyOther ( // IN PKMUTANT Object // ) // // // Routine Description: // // This function satisfies a wait for any type of object except a mutant // and performs any side effects that are necessary. // // Arguments: // // Object - Supplies a pointer to a dispatcher object. // // Return Value: // // None. // #define KiWaitSatisfyOther(_Object_) { \ if (((_Object_)->Header.Type & DISPATCHER_OBJECT_TYPE_MASK) == EventSynchronizationObject) { \ (_Object_)->Header.SignalState = 0; \ \ } else if ((_Object_)->Header.Type == SemaphoreObject) { \ (_Object_)->Header.SignalState -= 1; \ \ } \ } VOID FASTCALL KiWaitTest ( IN PVOID Object, IN KPRIORITY Increment ); FORCEINLINE VOID KiWaitTestSynchronizationObject ( IN PVOID Object, IN KPRIORITY Increment ) /*++ Routine Description: This function tests if a wait can be satisfied when a synchronization dispatcher object attains a state of signaled. Synchronization objects include synchronization events and synchronization timers. Arguments: Object - Supplies a pointer to an event object. Increment - Supplies the priority increment. Return Value: None. --*/ { PKEVENT Event = Object; PLIST_ENTRY ListHead; PRKTHREAD Thread; PRKWAIT_BLOCK WaitBlock; PLIST_ENTRY WaitEntry; // // As long as the signal state of the specified event is signaled and // there are waiters in the event wait list, then try to satisfy a wait. // ListHead = &Event->Header.WaitListHead; ASSERT(IsListEmpty(&Event->Header.WaitListHead) == FALSE); WaitEntry = ListHead->Flink; do { // // Get the address of the wait block and the thread doing the wait. // WaitBlock = CONTAINING_RECORD(WaitEntry, KWAIT_BLOCK, WaitListEntry); Thread = WaitBlock->Thread; // // If the wait type is wait any, then satisfy the wait, unwait the // thread with the wait key status, and exit loop. Otherwise, unwait // the thread with a kernel APC status and continue the loop. // if (WaitBlock->WaitType == WaitAny) { Event->Header.SignalState = 0; KiUnwaitThread(Thread, (NTSTATUS)WaitBlock->WaitKey, Increment); break; } KiUnwaitThread(Thread, STATUS_KERNEL_APC, Increment); WaitEntry = ListHead->Flink; } while (WaitEntry != ListHead); return; } FORCEINLINE VOID KiWaitTestWithoutSideEffects ( IN PVOID Object, IN KPRIORITY Increment ) /*++ Routine Description: This function tests if a wait can be satisfied when a dispatcher object without side effects attains a state of signaled. Dispatcher objects that have no side effects when a wait is satisfied include notification events, notification timers, processes, and threads. Arguments: Object - Supplies a pointer to a dispatcher object that has no side effects when a wait is satisfied. Increment - Supplies the priority increment. Return Value: None. --*/ { PKEVENT Event = Object; PLIST_ENTRY ListHead; PRKTHREAD Thread; PRKWAIT_BLOCK WaitBlock; PLIST_ENTRY WaitEntry; // // Empty the entire list of waiters since the specified object has // no side effects when a wait is satisfied. // ListHead = &Event->Header.WaitListHead; ASSERT(IsListEmpty(&Event->Header.WaitListHead) == FALSE); WaitEntry = ListHead->Flink; do { // // Get the address of the wait block and the thread doing the wait. // WaitBlock = CONTAINING_RECORD(WaitEntry, KWAIT_BLOCK, WaitListEntry); Thread = WaitBlock->Thread; // // If the wait type is wait any, then unwait the thread with the // wait key status. Otherwise, unwait the thread with a kernel APC // status. // if (WaitBlock->WaitType == WaitAny) { KiUnwaitThread(Thread, (NTSTATUS)WaitBlock->WaitKey, Increment); } else { KiUnwaitThread(Thread, STATUS_KERNEL_APC, Increment); } WaitEntry = ListHead->Flink; } while (WaitEntry != ListHead); return; } VOID KiFreezeTargetExecution ( IN PKTRAP_FRAME TrapFrame, IN PKEXCEPTION_FRAME ExceptionFrame ); VOID KiPollFreezeExecution ( VOID ); VOID KiSaveProcessorState ( IN PKTRAP_FRAME TrapFrame, IN PKEXCEPTION_FRAME ExceptionFrame ); VOID KiSaveProcessorControlState ( IN PKPROCESSOR_STATE ProcessorState ); VOID KiRestoreProcessorState ( IN PKTRAP_FRAME TrapFrame, IN PKEXCEPTION_FRAME ExceptionFrame ); VOID KiRestoreProcessorControlState ( IN PKPROCESSOR_STATE ProcessorState ); #define KiEnableAlignmentExceptions() #define KiDisableAlignmentExceptions() BOOLEAN KiHandleAlignmentFault( IN PEXCEPTION_RECORD ExceptionRecord, IN PKEXCEPTION_FRAME ExceptionFrame, IN PKTRAP_FRAME TrapFrame, IN KPROCESSOR_MODE PreviousMode, IN BOOLEAN FirstChance, OUT BOOLEAN *ExceptionForwarded ); // // External references to private kernel data structures // extern PMESSAGE_RESOURCE_DATA KiBugCodeMessages; extern FAST_MUTEX KiGenericCallDpcMutex; extern ULONG KiDmaIoCoherency; extern ULONG KiMaximumDpcQueueDepth; extern ULONG KiMinimumDpcRate; extern ULONG KiAdjustDpcThreshold; extern PKDEBUG_ROUTINE KiDebugRoutine; extern PKDEBUG_SWITCH_ROUTINE KiDebugSwitchRoutine; extern const CCHAR KiFindFirstSetLeft[256]; extern CALL_PERFORMANCE_DATA KiFlushSingleCallData; extern ULONG_PTR KiHardwareTrigger; extern KEVENT KiSwapEvent; extern PKTHREAD KiSwappingThread; extern KNODE KiNode0; extern KNODE KiNodeInit[]; extern SINGLE_LIST_ENTRY KiProcessInSwapListHead; extern SINGLE_LIST_ENTRY KiProcessOutSwapListHead; extern SINGLE_LIST_ENTRY KiStackInSwapListHead; extern const ULONG KiPriorityMask[]; extern LIST_ENTRY KiProfileSourceListHead; extern BOOLEAN KiProfileAlignmentFixup; extern ULONG KiProfileAlignmentFixupInterval; extern ULONG KiProfileAlignmentFixupCount; extern KSPIN_LOCK KiReverseStallIpiLock; #if defined(_X86_) extern ULONG KiLog2MaximumIncrement; extern ULONG KiMaximumIncrementReciprocal; extern ULONG KeTimerReductionModulus; extern ULONG KiUpperModMul; #endif #if defined(_IA64_) extern ULONG KiMaxIntervalPerTimerInterrupt; // KiProfileInterval value should be replaced by a call: // HalQuerySystemInformation(HalProfileSourceInformation) #else // _IA64_ extern ULONG KiProfileInterval; #endif // _IA64_ extern LIST_ENTRY KiProfileListHead; extern KSPIN_LOCK KiProfileLock; extern UCHAR KiArgumentTable[]; extern ULONG KiServiceLimit; extern ULONG_PTR KiServiceTable[]; extern CALL_PERFORMANCE_DATA KiSetEventCallData; extern ULONG KiTickOffset; extern LIST_ENTRY KiTimerTableListHead[TIMER_TABLE_SIZE]; extern KAFFINITY KiTimeProcessor; extern KDPC KiTimerExpireDpc; extern KSPIN_LOCK KiFreezeExecutionLock; extern BOOLEAN KiSlavesStartExecution; extern CALL_PERFORMANCE_DATA KiWaitSingleCallData; extern ULONG KiEnableTimerWatchdog; #if defined(_IA64_) extern ULONG KiMasterRid; extern ULONGLONG KiMasterSequence; extern ULONG KiIdealDpcRate; extern KSPIN_LOCK KiRegionSwapLock; #if !defined(UP_NT) extern KSPIN_LOCK KiMasterRidLock; #endif VOID KiSaveEmDebugContext ( IN OUT PCONTEXT Context ); VOID KiLoadEmDebugContext ( IN PCONTEXT Context ); VOID KiFlushRse ( VOID ); VOID KiInvalidateStackedRegisters ( VOID ); NTSTATUS Ki386CheckDivideByZeroTrap( IN PKTRAP_FRAME Frame ); #endif // defined(_IA64_) #if defined(_IA64_) extern KINTERRUPT KxUnexpectedInterrupt; #endif #if NT_INST extern KIPI_COUNTS KiIpiCounts[MAXIMUM_PROCESSORS]; #endif extern KSPIN_LOCK KiFreezeLockBackup; extern ULONG KiFreezeFlag; extern volatile ULONG KiSuspendState; #if DBG extern ULONG KiMaximumSearchCount; #endif // // Define context switch data collection macro. // //#define _COLLECT_SWITCH_DATA_ 1 #if defined(_COLLECT_SWITCH_DATA_) #define KiIncrementSwitchCounter(Member) KeThreadSwitchCounters.Member += 1 #else #define KiIncrementSwitchCounter(Member) #endif FORCEINLINE PKTHREAD KiSelectReadyThread ( IN KPRIORITY LowPriority, IN PKPRCB Prcb ) /*++ Routine Description: This function searches the dispatcher ready queues from the specified low priority to the highest priority in an attempt to find a thread that can execute on the specified processor. Arguments: LowPriority - Supplies the lowest priority dispatcher ready queue to examine. Prcb - Supplies a pointer to a processor control block. Return Value: If a thread is located that can execute on the specified processor, then the address of the thread object is returned. Otherwise a null pointer is returned. --*/ { ULONG HighPriority; PRLIST_ENTRY ListEntry; ULONG PrioritySet; PKTHREAD Thread; // // Compute the set of priority levels that should be scanned in an attempt // to find a thread that can run on the current processor. // PrioritySet = KiPriorityMask[LowPriority] & Prcb->ReadySummary; Thread = NULL; if (PrioritySet != 0) { KeFindFirstSetLeftMember(PrioritySet, &HighPriority); ASSERT((PrioritySet & PRIORITY_MASK(HighPriority)) != 0); ASSERT(IsListEmpty(&Prcb->DispatcherReadyListHead[HighPriority]) == FALSE); ListEntry = Prcb->DispatcherReadyListHead[HighPriority].Flink; Thread = CONTAINING_RECORD(ListEntry, KTHREAD, WaitListEntry); ASSERT((KPRIORITY)HighPriority == Thread->Priority); ASSERT((Thread->Affinity & AFFINITY_MASK(Prcb->Number)) != 0); ASSERT(Thread->NextProcessor == Prcb->Number); if (RemoveEntryList(&Thread->WaitListEntry) != FALSE) { Prcb->ReadySummary ^= PRIORITY_MASK(HighPriority); } } // // Return thread address if one could be found. // return Thread; } VOID KiSetInternalEvent ( IN PKEVENT Event, IN PKTHREAD Thread ); // // Include platform specific internal kernel header file. // #if defined(_AMD64_) #include "amd64\kiamd64.h" #elif defined(_X86_) #include "i386\kix86.h" #endif // defined(_AMD64_) #endif // defined(_KI_)