// TITLE("Context Swap") //++ // // Copyright (c) 1994 IBM Corporation // // Module Name: // // ctxswap.s // // Abstract: // // This module implements the PowerPC machine dependent code necessary to // field the dispatch interrupt and to perform kernel initiated context // switching. // // Author: // // Peter L. Johnston (plj@vnet.ibm.com) August 1993 // Adapted from code by David N. Cutler (davec) 1-Apr-1991 // // Environment: // // Kernel mode only. // // Revision History: // // plj Apr-94 Upgraded to NT 3.5. // //-- #include "ksppc.h" // Module Constants #define rPrcb r.29 #define OTH r.30 #define NTH r.31 // Global externals .extern ..KdPollBreakIn .extern ..KeFlushCurrentTb .extern ..KiActivateWaiterQueue .extern ..KiContinueClientWait .extern ..KiDeliverApc .extern ..KiQuantumEnd .extern ..KiReadyThread .extern ..KiWaitTest .extern KdDebuggerEnabled .extern KeTickCount .extern KiDispatcherReadyListHead .extern KiIdleSummary .extern KiReadySummary .extern KiWaitInListHead .extern KiWaitOutListHead .extern __imp_HalProcessorIdle .extern __imp_KeLowerIrql #if DBG .extern ..DbgBreakPoint .extern ..DbgBreakPointWithStatus #endif #if !defined(NT_UP) .extern KeNumberProcessors .extern KiBarrierWait .extern KiContextSwapLock .extern KiDispatcherLock .extern KiProcessorBlock #if SPINDBG .extern ..KiAcquireSpinLockDbg .extern ..KiTryToAcquireSpinLockDbg #endif #endif .extern KiMasterSequence .extern KiMasterPid .globl KiScheduleCount .data KiScheduleCount: .long 0 #if COLLECT_PAGING_DATA .globl KiFlushOnProcessSwap .data KiFlushOnProcessSwap: .long 0 #endif // // ThreadSwitchFrame // // This is the layout of the beginning of the stack frame that must be // established by any routine that calls SwapContext. // // The caller of SwapContext must have saved r.14 and 26 thru 31. // SwapContext will take care of r.15 thru r.25, f.14 thru f.31 and // the condition register. // // Note: this is not a complete stack frame, the caller must allocate // additional space for any additional registers it needs to // save (eg Link Register). (Also, the following has not been // padded to 8 bytes). // // WARNING: KiInitializeContextThread() is aware of the layout of // a ThreadSwitchFrame. // .struct 0 .space StackFrameHeaderLength swFrame:.space SwapFrameLength swFrameLength: // SBTTL("Switch To Thread") //++ // // NTSTATUS // KiSwitchToThread ( // IN PKTHREAD NextThread, // IN ULONG WaitReason, // IN ULONG WaitMode, // IN PKEVENT WaitObject // ) // // Routine Description: // // This function performs an optimal switch to the specified target thread // if possible. No timeout is associated with the wait, thus the issuing // thread will wait until the wait event is signaled or an APC is deliverd. // // N.B. This routine is called with the dispatcher database locked. // // N.B. The wait IRQL is assumed to be set for the current thread and the // wait status is assumed to be set for the target thread. // // N.B. It is assumed that if a queue is associated with the target thread, // then the concurrency count has been incremented. // // N.B. Control is returned from this function with the dispatcher database // unlocked. // // Arguments: // // NextThread - Supplies a pointer to a dispatcher object of type thread. // // WaitReason - supplies the reason for the wait operation. // // WaitMode - Supplies the processor wait mode. // // WaitObject - Supplies a pointer to a dispatcher object of type event // or semaphore. // // Return Value: // // The wait completion status. A value of STATUS_SUCCESS is returned if // the specified object satisfied the wait. A value of STATUS_USER_APC is // returned if the wait was aborted to deliver a user APC to the current // thread. //-- .struct 0 .space swFrameLength sttLR: .space 4 sttReason: .space 4 sttMode: .space 4 sttObject: .space 4 .align 3 // ensure 8 byte alignment sttFrameLength: SPECIAL_ENTRY_S(KiSwitchToThread,_TEXT$00) mflr r.0 // get return address stwu r.sp, -sttFrameLength(r.sp) // buy stack frame stw r.14, swFrame + ExGpr14(r.sp) // save gpr 14 stw r.26, swFrame + ExGpr26(r.sp) // save gprs 26 through 31 stw r.27, swFrame + ExGpr27(r.sp) stw r.28, swFrame + ExGpr28(r.sp) stw r.29, swFrame + ExGpr29(r.sp) stw r.30, swFrame + ExGpr30(r.sp) stw r.31, swFrame + ExGpr31(r.sp) ori NTH, r.3, 0 stw r.0, sttLR(r.sp) // save return address li r.0, 0 PROLOGUE_END(KiSwitchToThread) // // Save the wait reason, the wait mode, and the wait object address. // stw r.4, sttReason(r.sp) // save wait reason stw r.5, sttMode(r.sp) // save wait mode stw r.6, sttObject(r.sp) // save wait object address // // If the target thread's kernel stack is resident, the target thread's // process is in the balance set, the target thread can run on the // current processor, and another thread has not already been selected // to run on the current processor, then do a direct dispatch to the // target thread bypassing all the general wait logic, thread priorities // permiting. // lwz r.7, ThApcState + AsProcess(NTH) // get target process address lbz r.8, ThKernelStackResident(NTH) // get kernel stack resident lwz rPrcb, KiPcr + PcPrcb(r.0) // get address of PRCB lbz r.10, PrState(r.7) // get target process state lwz OTH, KiPcr + PcCurrentThread(r.0) // get current thread address cmpwi r.8, 0 // kernel stack resident? beq LongWay // if eq, kernel stack not resident cmpwi r.10, ProcessInMemory // process in memory? bne LongWay // if ne, process not in memory #if !defined(NT_UP) lwz r.8, PbNextThread(rPrcb) // get address of next thread lbz r.10, ThNextProcessor(OTH) // get current processor number lwz r.14, ThAffinity(NTH) // get target thread affinity lwz r.26, KiPcr + PcSetMember(r.0) // get processor set member cmpwi r.8, 0 // next thread selected? bne LongWay // if ne, next thread selected and. r.26, r.26, r.14 // check for compatible affinity beq LongWay // if eq, affinity not compatible #endif // // Compute the new thread priority. // lbz r.14, ThPriority(OTH) // get client thread priority lbz r.26, ThPriority(NTH) // get server thread priority cmpwi r.14, LOW_REALTIME_PRIORITY // check if realtime client cmpwi cr.7, r.26, LOW_REALTIME_PRIORITY // check if realtime server bge stt60 // if ge, realtime client lbz r.27, ThPriorityDecrement(NTH) // get priority decrement value lbz r.28, ThBasePriority(NTH) // get server base priority bge cr.7, stt50 // if ge, realtime server addi r.9, r.28, 1 // computed boosted priority cmpwi r.27, 0 // server boot active? bne stt30 // if ne, server boost active // // Both the client and the server are not realtime and a priority boost // is not currently active for the server. Under these conditions an // optimal switch to the server can be performed if the base priority // of the server is above a minimum threshold or the boosted priority // of the server is not less than the client priority. // cmpw r.9, r.14 // check if high enough boost cmpwi cr.7, r.9, LOW_REALTIME_PRIORITY // check if less than realtime blt stt20 // if lt, boosted priority less stb r.9, ThPriority(NTH) // asssume boosted priority is okay blt cr.7, stt70 // if lt, less than realtime li r.9, LOW_REALTIME_PRIORITY - 1 // set high server priority stb r.9, ThPriority(NTH) // b stt70 stt20: // // The boosted priority of the server is less than the current priority of // the client. If the server base priority is above the required threshold, // then a optimal switch to the server can be performed by temporarily // raising the priority of the server to that of the client. // cmpwi r.28, BASE_PRIORITY_THRESHOLD // check if above threshold sub r.9, r.14, r.28 // compute priority decrement value blt LongWay // if lt, priority below threshold li r.28, ROUND_TRIP_DECREMENT_COUNT // get system decrement count value stb r.9, ThPriorityDecrement(NTH) // set priority decrement value stb r.14, ThPriority(NTH) // set current server priority stb r.28, ThDecrementCount(NTH) // set server decrement count b stt70 stt30: // // A server boost has previously been applied to the server thread. Count // down the decrement count to determine if another optimal server switch // is allowed. // lbz r.9, ThDecrementCount(NTH) // decrement server count value subic. r.9, r.9, 1 // stb r.9, ThDecrementCount(NTH) // store updated decrement count beq stt40 // if eq, no more switches allowed // // Another optimal switch to the server is allowed provided that the // server priority is not less than the client priority. // cmpw r.26, r.14 // check if server lower priority bge stt70 // if ge, server not lower priority b LongWay stt40: // // The server has exhausted the number of times an optimal switch may // be performed without reducing its priority. Reduce the priority of // the server to its original unboosted value minus one. // stb r.0, ThPriorityDecrement(NTH) // clear server priority decrement stb r.28, ThPriority(NTH) // set server priority to base b LongWay stt50: // // The client is not realtime and the server is realtime. An optimal switch // to the server can be performed. // lbz r.9, PrThreadQuantum(r.7) // get process quantum value b stt65 stt60: // // The client is realtime. In order for an optimal switch to occur, the // server must also be realtime and run at a high or equal priority. // cmpw r.26, r.14 // check if server is lower priority lbz r.9, PrThreadQuantum(r.7) // get process quantum value blt LongWay // if lt, server is lower priority stt65: stb r.9, ThQuantum(NTH) // set server thread quantum stt70: // // Set the next processor for the server thread. // #if !defined(NT_UP) stb r.10, ThNextProcessor(NTH) // set server next processor number #endif // // Set the address of the wait block list in the client thread, initialize // the event wait block, and insert the wait block in client event wait list. // addi r.8, OTH, EVENT_WAIT_BLOCK_OFFSET // compute wait block address stw r.8, ThWaitBlockList(OTH) // set address of wait block list stw r.0, ThWaitStatus(OTH) // set initial wait status stw r.6, WbObject(r.8) // set address of wait object stw r.8, WbNextWaitBlock(r.8) // set next wait block address lis r.10, WaitAny // get wait type and wait key stw r.10, WbWaitKey(r.8) // set wait key and wait type addi r.10, r.6, EvWaitListHead // compute wait object listhead address lwz r.14, LsBlink(r.10) // get backward link of listhead addi r.26, r.8, WbWaitListEntry // compute wait block list entry address stw r.26, LsBlink(r.10) // set backward link of listhead stw r.26, LsFlink(r.14) // set forward link in last entry stw r.10, LsFlink(r.26) // set forward link in wait entry stw r.14, LsBlink(r.26) // set backward link in wait entry // // Set the client thread wait parameters, set the thread state to Waiting, // and insert the thread in the proper wait list. // stb r.0, ThAlertable(OTH) // set alertable FALSE. stb r.4, ThWaitReason(OTH) // set wait reason stb r.5, ThWaitMode(OTH) // set the wait mode lbz r.6, ThEnableStackSwap(OTH) // get kernel stack swap enable lwz r.10, [toc]KeTickCount(r.toc) // get &KeTickCount lwz r.10, 0(r.10) // get low part of tick count stw r.10, ThWaitTime(OTH) // set thread wait time li r.8, Waiting // set thread state stb r.8, ThState(OTH) // lwz r.8,[toc]KiWaitInListHead(r.toc) // get address of wait in listhead cmpwi r.5, 0 // is wait mode kernel? beq stt75 // if eq, wait mode is kernel cmpwi r.6, 0 // is kernel stack swap disabled? beq stt75 // if eq, kernel stack swap disabled cmpwi r.14, LOW_REALTIME_PRIORITY + 9 // check if priority in range blt stt76 // if lt, thread priority in range stt75: lwz r.8,[toc]KiWaitOutListHead(r.toc) // get address of wait in listhead stt76: lwz r.14, LsBlink(r.8) // get backlink of wait listhead addi r.26, OTH, ThWaitListEntry // compute wait list entry address stw r.26, LsBlink(r.8) // set backward link of listhead stw r.26, LsFlink(r.14) // set forward link in last entry stw r.8, LsFlink(r.26) // set forward link in wait entry stw r.14, LsBlink(r.26) // set backward link in wait entry stt77: // // If the current thread is processing a queue entry, then attempt to // activate another thread that is blocked on the queue object. // // N.B. The next thread address can change if the routine to activate // a queue waiter is called. // lwz r.3, ThQueue(OTH) // get queue object address cmpwi r.3, 0 // queue object attached? beq stt78 // if eq, no queue object attached stw NTH, PbNextThread(rPrcb) // set next thread address bl ..KiActivateWaiterQueue // attempt to activate a blocked thread lwz NTH, PbNextThread(rPrcb) // get next thread address li r.0, 0 stw r.0, PbNextThread(rPrcb) // set next thread address to NULL stt78: #if !defined(NT_UP) lwz r.27, [toc]KiContextSwapLock(r.2)// get &KiContextSwapLock lwz r.28, [toc]KiDispatcherLock(r.2) // get &KiDispatcherLock #endif stw NTH, PbCurrentThread(rPrcb) // set address of current thread object bl ..SwapContext // swap context // // Lower IRQL to its previous level. // // N.B. SwapContext releases the dispatcher database lock. // // N.B. Register NTH (r.31) contains the address of the new thread on return. // // In the following, we could lower IRQL, isync then check for pending // interrupts. I believe it is faster to disable interrupts and get // both loads going. We need to avoid the situation where a DPC or // APC could be queued between the time we load PcSoftwareInterrupt // and actually lowering IRQL. (plj) // // We load the thread's WaitStatus in this block for scheduling // reasons in the hope that the normal case will be there is NOT // a DPC or APC pending. // lbz r.27, ThWaitIrql(NTH) // get original IRQL DISABLE_INTERRUPTS(r.5, r.6) lhz r.4, KiPcr+PcSoftwareInterrupt(r.0) lwz r.26, ThWaitStatus(NTH) // get wait completion status stb r.27, KiPcr+PcCurrentIrql(r.0) // set new IRQL ENABLE_INTERRUPTS(r.5) cmpw r.4, r.27 // check if new IRQL allows ble+ stt79 // APC/DPC and one is pending bl ..KiDispatchSoftwareInterrupt // process software interrupt stt79: // // If the wait was not interrupted to deliver a kernel APC, then return the // completion status. // cmpwi r.26, STATUS_KERNEL_APC // check if awakened for kernel APC ori r.3, r.26, 0 // set return status bne stt90 // if ne, normal wait completion // // Disable interrupts and acquire the dispatcher database lock. // #if !defined(NT_UP) DISABLE_INTERRUPTS(r.5, r.6) // // WARNING: The address of KiDispatcherLock was intentionally left in // r.28 by SwapContext for use here. // ACQUIRE_SPIN_LOCK(r.28, NTH, r.7, stt80, stt82) #endif // // Raise IRQL to synchronization level and save wait IRQL. // li r.7, SYNCH_LEVEL stb r.7, KiPcr+PcCurrentIrql(r.0) #if !defined(NT_UP) ENABLE_INTERRUPTS(r.5) #endif stb r.27, ThWaitIrql(NTH) // set client wait IRQL b ContinueWait #if !defined(NT_UP) SPIN_ON_SPIN_LOCK_ENABLED(r.28, r.7, stt80, stt82, stt85, r.5, r.6) #endif LongWay: // // Ready the target thread for execution and wait on the specified wait // object. // bl ..KiReadyThread // ready thread for execution // // Continue the wait and return the wait completion status. // // N.B. The wait continuation routine is called with the dispatcher // database locked. // ContinueWait: lwz r.3, sttObject(r.sp) // get wait object address lwz r.4, sttReason(r.sp) // get wait reason lwz r.5, sttMode(r.sp) // get wait mode bl ..KiContinueClientWait // continue client wait stt90: lwz r.0, sttLR(r.sp) // restore return address lwz r.26, swFrame + ExGpr26(r.sp) // restore gprs 26 thru 31 lwz r.27, swFrame + ExGpr27(r.sp) // lwz r.28, swFrame + ExGpr28(r.sp) // lwz r.29, swFrame + ExGpr29(r.sp) // lwz r.30, swFrame + ExGpr30(r.sp) // mtlr r.0 // set return address lwz r.31, swFrame + ExGpr31(r.sp) // lwz r.14, swFrame + ExGpr14(r.sp) // restore gpr 14 addi r.sp, r.sp, sttFrameLength // return stack frame blr DUMMY_EXIT(KiSwitchToThread) SBTTL("Dispatch Software Interrupt") //++ // // VOID // KiDispatchSoftwareInterrupt (VOID) // // Routine Description: // // This routine is called when the current irql drops below // DISPATCH_LEVEL and a software interrupt may be pending. A // software interrupt is either a pending DPC or a pending APC. // If a DPC is pending, Irql is raised to DISPATCH_LEVEL and // KiDispatchInterrupt is called. When KiDispatchInterrupt // returns, the Irql (before we raised it) is compared to // APC_LEVEL and if less and an APC interrupt is pending it // is processed. // // Note: this routine manipulates PCR->CurrentIrql directly to // avoid recursion by using Ke[Raise|Lower]Irql. // // Arguments: // // None. // // Return Value: // // None. // //-- .text .align 5 // cache block align LEAF_ENTRY_S(KiDispatchSoftwareInterrupt, _TEXT$00) li r.3, 1 // Flag to dispatch routines // to enable interrupts when // returning to IRQL 0 DISABLE_INTERRUPTS(r.7, r.4) ALTERNATE_ENTRY(KiDispatchSoftwareIntDisabled) stw r.3, -8(r.sp) // Flag to dispatch routines // indicating whether to enable // or not to enable interrupts // when returning to IRQL 0 lhz r.9, KiPcr+PcSoftwareInterrupt(r.0) // pending s/w interrupt? lbz r.3, KiPcr+PcCurrentIrql(r.0) // get current irql srwi. r.4, r.9, 8 // isolate DPC pending cmpw cr.6, r.9, r.3 // see if APC int and APC level cmpwi cr.7, r.3, APC_LEVEL // compare IRQL to APC LEVEL // // Possible values for SoftwareInterrupt (r.9) are // // 0x0101 DPC and APC interrupt pending // 0x0100 DPC interrupt pending // 0x0001 APC interrupt pending // 0x0000 No software interrupt pending (unlikely but possible) // // Possible values for current IRQL are zero or one. By comparing // SoftwareInterrupt against current IQRL (above) we can quickly see // if any software interrupts are valid at this time. // // Calculate correct IRQL for the interrupt we are processing. If DPC // then we need to be at DISPATCH_LEVEL which is one greater than APC_ // LEVEL. r.4 contains one if we are going to run a DPC, so we add // APC_LEVEL to r.4 to get the desired IRQL. // addi r.4, r.4, APC_LEVEL // calculate new IRQL ble cr.6,ExitEnabled // return if no valid interrupt #if DBG cmplwi cr.6, r.3, DISPATCH_LEVEL // sanity check, should only blt+ cr.6, $+12 // below DISPATCH_LEVEL twi 31, 0, 0x16 // BREAKPOINT blr // return if wrong IRQL #endif // // ..DispatchSoftwareInterrupt is an alternate entry used indirectly // by KeReleaseSpinLock (via KxReleaseSpinLock). KeReleaseSpinLock has // been carefully written to construct the same conditions as apply if // execution came from above. // ..DispatchSoftwareInterrupt: stb r.4, KiPcr+PcCurrentIrql(r.0) // set IRQL ENABLE_INTERRUPTS(r.7) beq cr.0, ..KiDispatchApc // jif not DPC interrupt beq cr.7, ..KiDispatchDpcOnly // jif DPC and old IRQL APC LEV. b ..KiDispatchDpc // DPC int, old IRQL < APC LEV. ExitEnabled: ENABLE_INTERRUPTS(r.7) blr DUMMY_EXIT(KiDispatchSoftwareInterrupt) SBTTL("Unlock Dispatcher Database") //++ // // VOID // KiUnlockDispatcherDatabase ( // IN KIRQL OldIrql // ) // // Routine Description: // // This routine is entered at synchronization level with the dispatcher // database locked. Its function is to either unlock the dispatcher // database and return or initiate a context switch if another thread // has been selected for execution. // // N.B. This code merges with the following swap context code. // // N.B. A context switch CANNOT be initiated if the previous IRQL // is greater than or equal to DISPATCH_LEVEL. // // N.B. This routine is carefully written to be a leaf function. If, // however, a context swap should be performed, the routine is // switched to a nested function. // // Arguments: // // OldIrql (r.3) - Supplies the IRQL when the dispatcher database // lock was acquired. // // Return Value: // // None. // //-- LEAF_ENTRY_S(KiUnlockDispatcherDatabase, _TEXT$00) // // Check if a thread has been scheduled to execute on the current processor. // cmpwi cr.1, r.3, APC_LEVEL // check if IRQL below dispatch level lwz r.7, KiPcr+PcPrcb(r.0) // get address of PRCB lhz r.9, KiPcr+PcSoftwareInterrupt(r.0) // pending s/w interrupt? lwz r.8, PbNextThread(r.7) // get next thread address cmpw cr.7, r.9, r.3 // compare pending against irql cmpwi r.8, 0 // check if next thread selected bne uddSwitch // jif new thread selected // // Not switching, release dispatcher database lock. // #if !defined(NT_UP) lwz r.5, [toc]KiDispatcherLock(r.toc) li r.6, 0 RELEASE_SPIN_LOCK(r.5, r.6) #endif // // If already at dispatch level, we're done. // bgtlr cr.1 // return // // Dropping below DISPATCH_LEVEL, we may need to inspire a software // interrupt if one is pending. // // // Above we compared r.9 (SoftwareInterrupt) with r.3 (OldIrql) (result in // cr.7). See KiDispatchSoftwareInterrupt (above) for possible values. // If we did not take the above branch then OldIrql is either zero or one. // If SoftwareInterrupt is greater than OldIrql then a pending software // interrupt can be taken at this time. // bgt cr.7, uddIntPending // jif pending interrupt stb r.3, KiPcr+PcCurrentIrql(r.0) // set new IRQL blr // return // // A new thread has been selected to run on the current processor, but // the new IRQL is not below dispatch level. Release the dispatcher lock. // If the current processor is not executing a DPC, then request a dispatch // interrupt on the current. IRQL is already at the right level. // uddCantSwitch: #if !defined(NT_UP) lwz r.5, [toc]KiDispatcherLock(r.toc) li r.6, 0 RELEASE_SPIN_LOCK(r.5, r.6) #endif lwz r.6, PbDpcRoutineActive(r.7) li r.9, 1 cmplwi r.6, 0 bnelr // return if DPC already active lwz r.5, [toc]KiScheduleCount(r.toc) stb r.9, KiPcr+PcDispatchInterrupt(r.0) // request dispatch interrupt lwz r.6, 0(r.5) addi r.6, r.6, 1 // bump schedule count stw r.6, 0(r.5) blr // return // // A software interrupt is pending with higher priority than OldIrql. // // cr.1 is the result of comparing OldIrql with APC_LEVEL. // uddIntPending: // // Set flag to enable interrupts after dispatching software interrupts. // r.9 must be non-zero because PcSoftwareInt(r.9) > OldIrql. Only // needs to be set once no matter which dispatch routine is called. // stw r.9, -8(r.sp) beq cr.1, ..KiDispatchDpcOnly // new IRQL doesn't allow APCs srwi. r.3, r.9, 8 // isolate DPC pending addi r.3, r.3, APC_LEVEL // calculate correct IRQL stb r.3, KiPcr+PcCurrentIrql(r.0) // set new IRQL bne ..KiDispatchDpc // jif DPC at APC_LEVEL // // IRQL dropped from DISPATCH_LEVEL to APC_LEVEL, make sure no DPCs // were queued while we were checking. We are now at APC level so // any new DPCs will happen without our having to check again. // lbz r.4, KiPcr+PcDispatchInterrupt(r.0) cmpwi r.4, 0 // new DPCs? beq ..KiDispatchApc // jif not li r.3, DISPATCH_LEVEL // re-raise to DISPATCH_LEVEL stb r.3, KiPcr+PcCurrentIrql(r.0) // set new IRQL b ..KiDispatchDpc DUMMY_EXIT(KiUnlockDispatcherDatabase) // // A new thread has been selected to run on the current processor. // // If the new IRQL is less than dispatch level, then switch to the new // thread. // uddSwitch: bgt cr.1, uddCantSwitch // jif new IRQL > apc level // // N.B. This routine is carefully written as a nested function. Control // drops into this function from above. // SPECIAL_ENTRY_S(KxUnlockDispatcherDatabase, _TEXT$00) mflr r.0 // get return address stwu r.sp, -kscFrameLength(r.sp) // buy stack frame stw r.29, swFrame + ExGpr29(r.sp) // save gpr 29 stw r.30, swFrame + ExGpr30(r.sp) // save gpr 30 ori rPrcb, r.7, 0 // copy PRCB address stw r.31, swFrame + ExGpr31(r.sp) // save gpr 31 lwz OTH, KiPcr+PcCurrentThread(r.0) // get current thread address stw r.27, swFrame + ExGpr27(r.sp) // save gpr 27 ori NTH, r.8, 0 // thread to switch to stw r.14, swFrame + ExGpr14(r.sp) // save gpr 14 li r.11,0 stw r.28, swFrame + ExGpr28(r.sp) // save gpr 28 stw r.26, swFrame + ExGpr26(r.sp) // save gpr 26 stw r.0, kscLR(r.sp) // save return address PROLOGUE_END(KxUnlockDispatcherDatabase) stw r.11, PbNextThread(rPrcb) // clear next thread address stb r.3, ThWaitIrql(OTH) // save previous IRQL // // Reready current thread for execution and swap context to the selected thread. // ori r.3, OTH, 0 // set address of previous thread object stw NTH, PbCurrentThread(rPrcb) // set address of current thread object #if !defined(NT_UP) lwz r.27,[toc]KiContextSwapLock(r.2)// get &KiContextSwapLock lwz r.28,[toc]KiDispatcherLock(r.2) // get &KiDispatcherLock #endif bl ..KiReadyThread // reready thread for execution b ksc130 // join common code DUMMY_EXIT(KxUnlockDispatcherDatabase) //++ // // VOID // KxReleaseSpinLock(VOID) // // Routine Description: // // This routine is entered when a call to KeReleaseSpinLock lowers // IRQL to a level sufficiently low for a pending software interrupt // to be deliverable. // // Although this routine has no arguments, the following entry conditions // apply. // // Interrupts are disabled. // // r.7 MSR prior to disabling interrupts. // r.4 IRQL to be raised to (PCR->CurrentIrql has been lowered // even though interrupts are currently disabled). // // cr.0 ne if DPC pending // cr.7 eq if ONLY DPCs can run (ie PCR->CurrentIrql == APC_LEVEL) // // Arguments: // // None. // // Return Value: // // None. // //-- .struct 0 .space StackFrameHeaderLength sp31: .space 4 // r.31 save .align 3 // ensure 8 byte alignment spLength: SPECIAL_ENTRY_S(KxReleaseSpinLock, _TEXT$01) stw r.31, sp31-spLength(r.sp) // save r.31 mflr r.31 // save return address (in r.31) stwu r.sp, -spLength(r.sp) // buy stack frame PROLOGUE_END(KxReleaseSpinLock) li r.3, 1 stw r.3, -8(r.sp) // flag to dispatch routines // to enable interrupts when // returning to irql 0. bl ..DispatchSoftwareInterrupt mtlr r.31 // set return address lwz r.31, sp31(r.sp) // restore r.31 addi r.sp, r.sp, spLength // release stack frame SPECIAL_EXIT(KxReleaseSpinLock) //++ // // VOID // KiDispatchDpcOnly (VOID) // // Routine Description: // // This routine is entered as a result of lowering IRQL to // APC_LEVEL with a DPC interrupt pending. IRQL is currently // at DISPATCH_LEVEL, the dispatcher database is unlocked. // // Arguments: // // None. // // Return Value: // // None. // //-- .set kdsiFrameLength, STK_MIN_FRAME+8 NESTED_ENTRY_S(KiDispatchDpcOnly, kdsiFrameLength, 0, 0, _TEXT$00) PROLOGUE_END(KiDispatchDpcOnly) kddo10: bl ..KiDispatchInterrupt li r.3, APC_LEVEL // get new IRQL DISABLE_INTERRUPTS(r.8, r.4) lbz r.4, KiPcr+PcDispatchInterrupt(r.0) // more DPCs pending? lwz r.5, KiPcr+PcPrcb(r.0) // get address of PRCB cmpwi r.4, 0 lwz r.6, PbInterruptCount(r.5) // bump interrupt count addi r.4, r.4, APC_LEVEL // calc new IRQL addi r.6, r.6, 1 stw r.6, PbInterruptCount(r.5) lwz r.6, STK_MIN_FRAME(r.sp) // parameter to enable stb r.4, KiPcr+PcCurrentIrql(r.0) // set new IRQL beq+ kddo20 ENABLE_INTERRUPTS(r.8) b kddo10 // jif more DPCs to run kddo20: cmpwi r.6, 0 // o.k to enable interrupts? beq kddo25 // return if not ENABLE_INTERRUPTS(r.8) // reenable interrupts and exit kddo25: NESTED_EXIT(KiDispatchDpcOnly, kdsiFrameLength, 0, 0) //++ // // VOID // KiDispatchDpc (VOID) // // Routine Description: // // This routine is entered as a result of lowering IRQL below // APC_LEVEL with a DPC interrupt pending. IRQL is currently // at DISPATCH_LEVEL, the dispatcher database is unlocked. // // Once DPC processing is complete, APC processing may be required. // // Arguments: // // None. // // Return Value: // // None. // //-- NESTED_ENTRY_S(KiDispatchDpc, kdsiFrameLength, 0, 0, _TEXT$00) PROLOGUE_END(KiDispatchDpc) kdd10: bl ..KiDispatchInterrupt DISABLE_INTERRUPTS(r.8, r.4) lwz r.5, KiPcr+PcPrcb(r.0) // get address of PRCB lhz r.4, KiPcr+PcSoftwareInterrupt(r.0) // more DPCs or APCs pending? lwz r.6, PbInterruptCount(r.5) // bump interrupt count cmpwi r.4, APC_LEVEL addi r.6, r.6, 1 stw r.6, PbInterruptCount(r.5) bgt- kdd20 // jif more DPCs lwz r.6, STK_MIN_FRAME(r.sp) // parameter to enable // interrupts stb r.4, KiPcr+PcCurrentIrql(r.0) // set new IRQL blt kdd30 // honor parameter to exit // enabled or disabled when // returning to IRQL 0 ENABLE_INTERRUPTS(r.8) // equal b kda10 // jif APCs to run kdd20: ENABLE_INTERRUPTS(r.8) // greater than b kdd10 // jif more DPCs to run kdd30: cmpwi r.6, 0 // o.k to enable interrupts? beq- kdd35 // return if not ENABLE_INTERRUPTS(r.8) // reenable interrupts and exit kdd35: NESTED_EXIT(KiDispatchDpc, kdsiFrameLength, 0, 0) //++ // // VOID // KiDispatchApc (VOID) // // Routine Description: // // This routine is entered as a result of lowering IRQL below // APC_LEVEL with an APC interrupt pending. IRQL is currently // at APC_LEVEL. // // Arguments: // // None. // // Return Value: // // None. // //-- NESTED_ENTRY_S(KiDispatchApc, kdsiFrameLength, 0, 0, _TEXT$00) PROLOGUE_END(KiDispatchApc) kda10: lwz r.6, KiPcr+PcPrcb(r.0) // get address of PRCB li r.3, 0 // PreviousMode = Kernel lwz r.7, PbApcBypassCount(r.6) // get APC bypass count li r.4, 0 // TrapFrame = 0 li r.5, 0 // ExceptionFrame = 0 addi r.7, r.7, 1 // increment APC bypass count stb r.3, KiPcr+PcApcInterrupt(r.0) // clear APC pending stw r.7, PbApcBypassCount(r.6) // store new APC bypass count bl ..KiDeliverApc li r.3, 0 // get new IRQL #if !defined(NT_UP) DISABLE_INTERRUPTS(r.8, r.4) #else stb r.3, KiPcr+PcCurrentIrql(r.0) // lower IRQL sync #endif lwz r.5, KiPcr+PcPrcb(r.0) // get address of PRCB lbz r.4, KiPcr+PcApcInterrupt(r.0) // more APCs pending? lwz r.6, PbInterruptCount(r.5) // bump interrupt count cmpwi r.4, 0 addi r.6, r.6, 1 stw r.6, PbInterruptCount(r.5) stb r.4, KiPcr+PcCurrentIrql(r.0) // Raise IRQL if more APCs #if !defined(NT_UP) ENABLE_INTERRUPTS(r.8) #endif bne- kda10 // jif more APCs to run NESTED_EXIT(KiDispatchApc, kdsiFrameLength, 0, 0) SBTTL("Swap Thread") //++ // // VOID // KiSwapThread ( // VOID // ) // // Routine Description: // // This routine is called to select the next thread to run on the // current processor and to perform a context switch to the thread. // // Arguments: // // None. // // Outputs: ( for call to SwapContext ) // // r.31 NTH pointer to new thread // r.30 OTH pointer to old thread // r.29 rPrcb pointer to processor control block // r.28 pointer to dispatcher database lock // r.27 pointer to context swap lock // // Return Value: // // Wait completion status (r.3). // //-- .struct 0 .space swFrameLength kscLR: .space 4 .align 3 // ensure 8 byte alignment kscFrameLength: .align 6 // cache line align SPECIAL_ENTRY_S(KiSwapThread,_TEXT$00) mflr r.0 // get return address stwu r.sp, -kscFrameLength(r.sp) // buy stack frame stw r.29, swFrame + ExGpr29(r.sp) // save gpr 29 stw r.30, swFrame + ExGpr30(r.sp) // save gpr 30 stw r.31, swFrame + ExGpr31(r.sp) // save gpr 31 lwz OTH, KiPcr+PcCurrentThread(r.0) // get current thread addr lwz rPrcb, KiPcr+PcPrcb(r.0) // get Processor Control Block stw r.27, swFrame + ExGpr27(r.sp) // save gpr 27 stw r.28, swFrame + ExGpr28(r.sp) // save gpr 28 stw r.14, swFrame + ExGpr14(r.sp) // save gpr 14 lwz r.27, [toc]KiReadySummary(r.toc) // get &KiReadySummary lwz NTH, PbNextThread(rPrcb) // get address of next thread stw r.26, swFrame + ExGpr26(r.sp) // save gpr 26 li r.28, 0 // load a 0 lwz r.26, 0(r.27) // get ready summary cmpwi NTH, 0 // next thread selected? stw r.0, kscLR(r.sp) // save return address PROLOGUE_END(KiSwapThread) stw r.28, PbNextThread(rPrcb) // zero address of next thread bne ksc120 // if ne, next thread selected #if !defined(NT_UP) lwz r.5, [toc]KeTickCount(r.toc) // get &KeTickCount lwz r.3, KiPcr+PcSetMember(r.0) // get processor affinity mask lbz r.4, PbNumber(rPrcb) // get current processor number lwz r.5, 0(r.5) // get low part of tick count #endif // // Find the highest priority ready thread. // cntlzw r.6, r.26 // count zero bits from left lwz r.8, [toc]KiDispatcherReadyListHead(r.toc) // get ready listhead base address slw. r.7, r.26, r.6 // shift first set bit into sign bit subfic r.6, r.6, 31 // convert shift count to priority beq kscIdle // if mask is zero, no ready threads kscReadyScan: // // If the thread can execute on the current processor, then remove it from // the dispatcher ready queue. // slwi r.9, r.6, 3 // compute ready listhead offset slwi r.7, r.7, 1 // position next ready summary bit add r.9, r.9, r.8 // compute ready queue address lwz r.10, LsFlink(r.9) // get address of first entry subi NTH, r.10, ThWaitListEntry // compute address of thread object #if !defined(NT_UP) kscAffinityScan: lwz r.11, ThAffinity(NTH) // get thread affinity lbz r.0, ThNextProcessor(NTH) // get last processor number and. r.11, r.11, r.3 // check for compatible thread affinity cmpw cr.6, r.0, r.4 // compare last processor with current bne kscAffinityOk // if ne, thread affinity compatible lwz r.10, LsFlink(r.10) // get address of next entry cmpw r.10, r.9 // compare with queue address subi NTH, r.10, ThWaitListEntry // compute address of thread object bne kscAffinityScan // if ne, not end of list ksc70: cmpwi r.7, 0 // more ready queues to scan? subi r.6, r.6, 1 // decrement ready queue priority blt kscReadyScan // if lt, queue contains an entry beq kscIdle // if eq, no ready threads slwi r.7, r.7, 1 // position next ready summary bit b ksc70 // check next bit kscAffinityOk: // // If the thread last ran on the current processor, has been waiting for // longer than a quantum, or its priority is greater than low realtime // plus 9, then select the thread. Otherwise, an attempt is made to find // a more appropriate candidate. // beq cr.6, kscReadyFound // if eq, processor number match lbz r.0, ThIdealProcessor(NTH) // get ideal processor number cmpwi r.6, LOW_REALTIME_PRIORITY + 9 // check if priority in range cmpw cr.6, r.0, r.4 // compare ideal processor with current beq cr.6, ksc100 // if eq, processor number match bge ksc100 // if ge, priority high enough lwz r.12, ThWaitTime(NTH) // get time of thread ready sub r.12, r.5, r.12 // compute length of wait cmpwi cr.7, r.12, READY_SKIP_QUANTUM + 1 // check if wait time exceeded bge cr.7, ksc100 // if ge, waited long enough // // Search forward in the ready queue until the end of the list is reached // or a more appropriate thread is found. // lwz r.14, LsFlink(r.10) // get address of next entry ksc80: cmpw r.14, r.9 // compare with queue address subi r.28, r.14, ThWaitListEntry // compute address of thread object beq ksc100 // if eq, end of list lwz r.11, ThAffinity(r.28) // get thread affinity lbz r.0, ThNextProcessor(r.28) // get last processor number and. r.11, r.11, r.3 // check for compatible thread affinity cmpw cr.6, r.0, r.4 // compare last processor with current beq ksc85 // if eq, thread affinity not compatible beq cr.6, ksc90 // if eq, processor number match lbz r.0, ThIdealProcessor(NTH) // get ideal processor number cmpw cr.6, r.0, r.4 // compare ideal processor with current beq cr.6, ksc90 // if eq, processor number match ksc85: lwz r.12, ThWaitTime(r.28) // get time of thread ready lwz r.14, LsFlink(r.14) // get address of next entry sub r.12, r.5, r.12 // compute length of wait cmpwi cr.7, r.12, READY_SKIP_QUANTUM + 1 // check if wait time exceeded blt cr.7, ksc80 // if lt, wait time not exceeded b ksc100 // wait time exceeded -- switch to // first matching thread in ready queue ksc90: ori NTH, r.28, 0 // set thread address ori r.10, r.14, 0 // set list entry address ksc100: stb r.4, ThNextProcessor(NTH) // set next processor number kscReadyFound: #endif // // Remove the selected thread from the ready queue. // lwz r.11, LsFlink(r.10) // get list entry forward link lwz r.12, LsBlink(r.10) // get list entry backward link li r.0, 1 // set bit for mask generation slw r.0, r.0, r.6 // compute ready summary set member cmpw r.11, r.12 // check for list empty stw r.11, LsFlink(r.12) // set forward link in previous entry stw r.12, LsBlink(r.11) // set backward link in next entry bne ksc120 // if ne, list is not empty xor r.26, r.26, r.0 // clear ready summary bit stw r.26, 0(r.27) // update ready summary ksc120: // // Swap context to the next thread. // #if !defined(NT_UP) lwz r.27,[toc]KiContextSwapLock(r.2)// get &KiContextSwapLock lwz r.28,[toc]KiDispatcherLock(r.2) // get &KiDispatcherLock #endif stw NTH, PbCurrentThread(rPrcb) // set new thread current ksc130: bl ..SwapContext // swap context // // Lower IRQL and return wait completion status. // // N.B. SwapContext releases the dispatcher database lock. // lbz r.4, ThWaitIrql(NTH) // get original wait IRQL DISABLE_INTERRUPTS(r.6, r.7) lhz r.5, KiPcr+PcSoftwareInterrupt(r.0) // check for pending s/w ints lwz r.14, kscLR(r.sp) // get return address lwz r.31, ThWaitStatus(NTH) // get wait completion status lwz r.26, swFrame + ExGpr26(r.sp) // restore gprs 26 thru 30 lwz r.27, swFrame + ExGpr27(r.sp) // lwz r.28, swFrame + ExGpr28(r.sp) // lwz r.29, swFrame + ExGpr29(r.sp) // lwz r.30, swFrame + ExGpr30(r.sp) // stb r.4, KiPcr+PcCurrentIrql(r.0) // set new IRQL ENABLE_INTERRUPTS(r.6) cmpw r.5, r.4 // see if s/w int could now run bgtl- ..KiDispatchSoftwareInterrupt // jif pending int can run mtlr r.14 // set return address ori r.3, r.31, 0 // set return status lwz r.31, swFrame + ExGpr31(r.sp) // restore r.31 lwz r.14, swFrame + ExGpr14(r.sp) // restore r.14 addi r.sp, r.sp, kscFrameLength // return stack frame SPECIAL_EXIT(KiSwapThread) kscIdle: // // All ready queues were scanned without finding a runnable thread so // default to the idle thread and set the appropriate bit in idle summary. // lwz r.5, [toc]KiIdleSummary(r.toc) // get &KiIdleSummary #if defined(NT_UP) li r.4, 1 // set current idle summary #else lwz r.4, 0(r.5) // get current idle summary or r.4, r.4, r.3 // set member bit in idle summary #endif stw r.4, 0(r.5) // set new idle summary lwz NTH,PbIdleThread(rPrcb) // set address of idle thread b ksc120 // switch to idle thread SBTTL("Swap Context to Next Thread") //++ // // Routine Description: // // This routine is called to swap context from one thread to the next. // // Arguments: // // r.sp Pointer to { StackFrameHeader, ExceptionFrame } // r.31 NTH Address of next thread object // r.30 OTH Address of previous thread object // r.29 rPrcb Address of processor control block // r.28 Address of KiDispatcherLock // r.27 Address of KiContextSwapLock // // Return value: // // r.31 NTH Address of current thread object. // r.29 rPrcb Address of processor control block // // Note that the TOC register is neither saved nor restored across a // thread switch. This is because we are in NTOSKRNL (actually in the // routine SwapContext) in both threads (ie the TOC does not change). // // GPR 13 is set to the address of the TEB where it will remain. // //-- SPECIAL_ENTRY_S(SwapContext,_TEXT$00) mfcr r.3 // get condition register // // Acquire the context swap lock so the address space of the old process // cannot be deleted and then release the dispatcher database lock. // // N.B. This lock is used to protect the address space until the context // switch has sufficiently progressed to the point where the address // space is no longer needed. This lock is also acquired by the reaper // thread before it finishes thread termination. // #if !defined(NT_UP) b LkCtxSw // skip spin code SPIN_ON_SPIN_LOCK(r.27,r.4,LkCtxSw,LkCtxSwSpin) // spin on context swap lock ACQUIRE_SPIN_LOCK(r.27,r.31,r.4,LkCtxSw,LkCtxSwSpin) // acquire context swap lock #endif // // Set the new thread's state to Running before releasing the dispatcher lock. // li r.8, Running // set state of new thread stb r.8, ThState(NTH) // to running. #if !defined(NT_UP) RELEASE_SPIN_LOCK(r.28,r.4) // release dispatcher lock #endif // // Save old thread non-volatile context. // mflr r.0 // get return address stw r.15, swFrame + ExGpr15(r.sp) // save gprs 15 thru 25 stw r.16, swFrame + ExGpr16(r.sp) // stw r.17, swFrame + ExGpr17(r.sp) // stw r.18, swFrame + ExGpr18(r.sp) // stw r.19, swFrame + ExGpr19(r.sp) // stw r.20, swFrame + ExGpr20(r.sp) // stw r.21, swFrame + ExGpr21(r.sp) // stw r.22, swFrame + ExGpr22(r.sp) // stw r.23, swFrame + ExGpr23(r.sp) // stw r.24, swFrame + ExGpr24(r.sp) // stw r.25, swFrame + ExGpr25(r.sp) // stfd f.14, swFrame + ExFpr14(r.sp) // save non-volatile stfd f.15, swFrame + ExFpr15(r.sp) // floating point regs stfd f.16, swFrame + ExFpr16(r.sp) // stfd f.17, swFrame + ExFpr17(r.sp) // stfd f.18, swFrame + ExFpr18(r.sp) // stfd f.19, swFrame + ExFpr19(r.sp) // stfd f.20, swFrame + ExFpr20(r.sp) // stfd f.21, swFrame + ExFpr21(r.sp) // stfd f.22, swFrame + ExFpr22(r.sp) // stfd f.23, swFrame + ExFpr23(r.sp) // stfd f.24, swFrame + ExFpr24(r.sp) // stfd f.25, swFrame + ExFpr25(r.sp) // stfd f.26, swFrame + ExFpr26(r.sp) // stfd f.27, swFrame + ExFpr27(r.sp) // stfd f.28, swFrame + ExFpr28(r.sp) // stfd f.29, swFrame + ExFpr29(r.sp) // stfd f.30, swFrame + ExFpr30(r.sp) // stfd f.31, swFrame + ExFpr31(r.sp) // stw r.3, swFrame + SwConditionRegister(r.sp)// save CR stw r.0, swFrame + SwSwapReturn(r.sp) // save return address PROLOGUE_END(SwapContext) // // The following entry point is used to switch from the idle thread to // another thread. // ..SwapFromIdle: #if DBG cmpw NTH, OTH bne th_ok twi 31, 0, 0x16 th_ok: #endif stw NTH, KiPcr+PcCurrentThread(r.0) // set new thread current // // Get the old and new process object addresses. // #define NPROC r.3 #define OPROC r.4 lwz NPROC, ThApcState + AsProcess(NTH) // get new process object lwz OPROC, ThApcState + AsProcess(OTH) // get old process object DISABLE_INTERRUPTS(r.15, r.6) lwz r.13, ThTeb(NTH) // get addr of user TEB lwz r.24, ThStackLimit(NTH) // get stack limit lwz r.23, ThInitialStack(NTH) // get initial kernel stk ptr lwz r.22, ThKernelStack(NTH) // get new thread stk ptr cmpw cr.0, NPROC, OPROC // same process ? stw r.sp, ThKernelStack(OTH) // save current kernel stk ptr stw r.13, KiPcr+PcTeb(r.0) // set addr of user TEB // // Although interrupts are disabled, someone may be attempting to // single step thru the following. I can't see anyway to perform // the two operations atomically so I am inserting a label that is // known externally and can be checked against the exception address // if we fail stack validation in common_exception_entry (in real0.s) // in which case it's really ok. This has no performance impact. // // *** WARNING ****** WARNING ****** WARNING ****** WARNING *** // // (1) these two instructions MUST stay together, // (2) the stack validation code in common_exception_entry // KNOWS that the second instruction is a 'ori r.sp, r.22, 0' // and will perform such an instruction in line to correct // the problem. If you change this sequence you will need // to make an equivalent change in real0.s and the correct- // ability is dependent on the second instruction destroying // the stack pointer. // (plj). // stw r.24, KiPcr+PcStackLimit(r.0) // set stack limit stw r.23, KiPcr+PcInitialStack(r.0) // set initial kernel stack ptr .globl KepSwappingContext KepSwappingContext: ori r.sp, r.22, 0 // switch stacks #if !defined(NT_UP) // // Old process address space is no longer required. Ensure all // stores are done prior to releasing the ContextSwap lock. // N.B. SwapContextLock is still needed to ensure KiMasterPid // integrity. // li r.16, 0 eieio bne cr.0, ksc10 stw r.16, 0(r.27) // release Context Swap lock b ksc20 ksc10: #else beq cr.0, ksc20 #endif // // If the process sequence number matches the system sequence number, then // use the process PID. Otherwise, allocate a new process PID. // // N.B. The following code is duplicated from KiSwapProcess and will // join KiSwapProcess at SwapProcessSlow if sequence numbers // don't match. Register usage from here to the branch should // match KiSwapProcess. // lwz r.10,[toc]KiMasterSequence(r.toc) // get &KiMasterSequence lwz r.9,PrProcessSequence(NPROC) // get process sequence number lwz r.11,0(r.10) // get master sequence number lwz r.7,PrProcessPid(NPROC) // get process PID cmpw r.11,r.9 // master sequence == process sequence? bne ksc15 // jif not equal, go the slow path #if !defined(NT_UP) stw r.16, 0(r.27) // release Context Swap lock #endif // // Swap address space to the specified process. // lwz r.5,PrDirectoryTableBase(r.3) // get page dir page real addr mtsr 0,r.7 // set sreg 0 addi r.7,r.7,1 // add 1 to VSID mtsr 1,r.7 // set sreg 1 addi r.7,r.7,1 // add 1 to VSID mtsr 2,r.7 // set sreg 2 addi r.7,r.7,1 // add 1 to VSID mtsr 3,r.7 // set sreg 3 addi r.7,r.7,1 // add 1 to VSID mtsr 4,r.7 // set sreg 4 addi r.7,r.7,1 // add 1 to VSID mtsr 5,r.7 // set sreg 5 addi r.7,r.7,1 // add 1 to VSID mtsr 6,r.7 // set sreg 6 addi r.7,r.7,1 // add 1 to VSID mtsr 7,r.7 // set sreg 7 addi r.7,r.7,12-7 // add 5 to VSID mtsr 12,r.7 // set sreg 12 isync // context synchronize stw r.5,KiPcr+PcPgDirRa(r.0) // store page dir page ra in PCR #if COLLECT_PAGING_DATA lwz r.10,[toc]KiFlushOnProcessSwap(r.toc) lwz r.10,0(r.10) cmpwi r.10,0 bnel ..KeFlushCurrentTb #endif b ksc20 ksc15: bl SwapProcessSlow ksc20: lbz r.5, ThApcState + AsKernelApcPending(NTH) lbz r.16, ThDebugActive(NTH) // get the active debug register // mask stb r.5, KiPcr+PcApcInterrupt(r.0) // set APC pending appropriately stb r.16, KiPcr+PcDebugActive(r.0) // set the active debug register // mask for the new thread lwz r.5, PbContextSwitches(rPrcb) // get context switch count lwz r.7, ThContextSwitches(NTH) addi r.5, r.5, 1 // bump context switch count stw r.5, PbContextSwitches(rPrcb) // for processor. addi r.7, r.7, 1 // bump context switch count stw r.7, ThContextSwitches(NTH) // for this thread ENABLE_INTERRUPTS(r.15) lwz r.0, swFrame + SwSwapReturn(r.sp) // get return address lwz r.5, swFrame + SwConditionRegister(r.sp)// get CR lwz r.15, swFrame + ExGpr15(r.sp) // restore gprs 15 thru 25 lwz r.16, swFrame + ExGpr16(r.sp) // lwz r.17, swFrame + ExGpr17(r.sp) // lwz r.18, swFrame + ExGpr18(r.sp) // lwz r.19, swFrame + ExGpr19(r.sp) // lwz r.20, swFrame + ExGpr20(r.sp) // lwz r.21, swFrame + ExGpr21(r.sp) // lwz r.22, swFrame + ExGpr22(r.sp) // lwz r.23, swFrame + ExGpr23(r.sp) // lwz r.24, swFrame + ExGpr24(r.sp) // lwz r.25, swFrame + ExGpr25(r.sp) // lfd f.14, swFrame + ExFpr14(r.sp) // restore non-volatile lfd f.15, swFrame + ExFpr15(r.sp) // floating point regs lfd f.16, swFrame + ExFpr16(r.sp) // lfd f.17, swFrame + ExFpr17(r.sp) // lfd f.18, swFrame + ExFpr18(r.sp) // lfd f.19, swFrame + ExFpr19(r.sp) // lfd f.20, swFrame + ExFpr20(r.sp) // lfd f.21, swFrame + ExFpr21(r.sp) // lfd f.22, swFrame + ExFpr22(r.sp) // lfd f.23, swFrame + ExFpr23(r.sp) // lfd f.24, swFrame + ExFpr24(r.sp) // lfd f.25, swFrame + ExFpr25(r.sp) // lfd f.26, swFrame + ExFpr26(r.sp) // lfd f.27, swFrame + ExFpr27(r.sp) // mtlr r.0 // set return address mtcrf 0xff, r.5 // set condition register lfd f.28, swFrame + ExFpr28(r.sp) // lfd f.29, swFrame + ExFpr29(r.sp) // lfd f.30, swFrame + ExFpr30(r.sp) // lfd f.31, swFrame + ExFpr31(r.sp) // SPECIAL_EXIT(SwapContext) #undef NTH #undef OTH //++ // // VOID // KiSwapProcess ( // IN PKPROCESS NewProcess, // IN PKPROCESS OldProcess // ) // // Routine Description: // // This function swaps the address space from one process to another by // moving to the PCR the real address of the process page directory page // and loading segment registers 0-7 and 12 with VSIDs derived therefrom. // // The fast path below is duplicated inline in SwapContext for speed. // SwapContext joins this code at SwapProcessSlow if sequence numbers // differ. // // Arguments: // // NewProcess (r3) - Supplies a pointer to a control object of type process // which represents the new process that is switched to. // // OldProcess (r4) - Supplies a pointer to a control object of type process // which represents the old process that is switched from. // // Return Value: // // None. // //-- LEAF_ENTRY_S(KiSwapProcess,_TEXT$00) // // Get the Context Swap lock. This lock is used to protect a // processes memory space, it serves double duty to protect access // to KiMasterSequence. // // N.B. It is already held if entry is via SwapProcessSlow, the // lock is ALWAYS released by this routine. // #if !defined(NT_UP) lwz r.6, [toc]KiContextSwapLock(r.2) ACQUIRE_SPIN_LOCK(r.6,r.3,r.5,LkCtxSw2,LkCtxSw2Spin) #endif // // If the process sequence number matches the system sequence number, then // use the process PID. Otherwise, allocate a new process PID. // // WARNING: if you change register usage in the following be sure to make // the same changes in SwapContext. // lwz r.10,[toc]KiMasterSequence(r.toc) // get &KiMasterSequence lwz r.9,PrProcessSequence(r.3) // get process sequence number lwz r.11,0(r.10) // get master sequence number lwz r.7,PrProcessPid(r.3) // get process PID cmpw r.11,r.9 // master sequence == process sequence? bne SwapProcessSlow // jif not equal, out of line // // Swap address space to the specified process. // spup: lwz r.5,PrDirectoryTableBase(r.3) // get page dir page real addr DISABLE_INTERRUPTS(r.8,r.0) // disable interrupts #if !defined(NT_UP) sync li r.10, 0 stw r.10, 0(r.6) // release KiContextSwapLock #endif stw r.5,KiPcr+PcPgDirRa(r.0) // store page dir page ra in PCR mtsr 0,r.7 // set sreg 0 addi r.7,r.7,1 // add 1 to VSID mtsr 1,r.7 // set sreg 1 addi r.7,r.7,1 // add 1 to VSID mtsr 2,r.7 // set sreg 2 addi r.7,r.7,1 // add 1 to VSID mtsr 3,r.7 // set sreg 3 addi r.7,r.7,1 // add 1 to VSID mtsr 4,r.7 // set sreg 4 addi r.7,r.7,1 // add 1 to VSID mtsr 5,r.7 // set sreg 5 addi r.7,r.7,1 // add 1 to VSID mtsr 6,r.7 // set sreg 6 addi r.7,r.7,1 // add 1 to VSID mtsr 7,r.7 // set sreg 7 addi r.7,r.7,12-7 // add 5 to VSID mtsr 12,r.7 // set sreg 12 isync // context synchronize ENABLE_INTERRUPTS(r.8) // enable interrupts #if COLLECT_PAGING_DATA lwz r.10,[toc]KiFlushOnProcessSwap(r.toc) lwz r.10,0(r.10) cmpwi r.10,0 bne ..KeFlushCurrentTb #endif ALTERNATE_EXIT(KiSwapProcess) // return // // We need a new PID, the dispatcher database lock is still held so // we can update KiMasterPid without further protection. // SwapProcessSlow: lwz r.8,[toc]KiMasterPid(r.toc) // get &KiMasterPid lwz r.7,0(r.8) // get KiMasterPid addi r.7,r.7,16 // bump master pid rlwinm. r.7,r.7,0,0x007ffff0 // detect PID wrap beq ..KxSwapProcess // jif PID wrap stw r.11,PrProcessSequence(r.3) // save new process sequence // // control returns here from KxSwapProcess // spnp: #if !defined(NT_UP) lwz r.6, [toc]KiContextSwapLock(r.2) #endif stw r.7,0(r.8) // save new master PID stw r.7,PrProcessPid(r.3) // save new process PID b spup // continue with main line code #if !defined(NT_UP) SPIN_ON_SPIN_LOCK(r.6,r.5,LkCtxSw2,LkCtxSw2Spin) #endif DUMMY_EXIT(KiSwapProcess) //++ // // VOID // KxSwapProcess ( // IN PKPROCESS NewProcess, // ) // // Routine Description: // // This function is called (only) from KiSwapProcess when PID wrap has // occured. KiSwapProcess is a LEAF function. The purpose of this // function is to alloacte a stack frame and save data that needs to // be restored for KiSwapProcess. This routine is called aproximately // once every 16 million new processes. The emphasis in KiSwapProcess // is to handle the other 16 million - 1 cases as fast as possible. // // Arguments: // // NewProcess (r3) - Supplies a pointer to a control object of // type process which represents the new process being switched to. // This must be saved and restored for KiSwapProcess. // // &KiMasterPid (r8) - Address of system global KiMasterPid // This must be restored for KiSwapProcess. // // &KiMasterSequence (r10) - Address of system global KiMasterSequence. // // KiMasterSequence (r11) - Current Value of the above variable. // // Return Value: // // None. // // Registers r3, r8 and the Link Register are restored. r7 contains // the new PID which will be 16. // //-- .struct 0 .space StackFrameHeaderLength spLR: .space 4 // link register save spR3: .space 4 // new process address save .align 3 // ensure correct alignment spFrameLength: SPECIAL_ENTRY_S(KxSwapProcess,_TEXT$00) mflr r.0 // get link register stwu r.sp,-spFrameLength(r.sp) // buy stack frame stw r.3,spR3(r.sp) // save new process address stw r.0,spLR(r.sp) // save swap process' return address PROLOGUE_END(KxSwapProcess) // // PID wrap has occured. On PowerPC we do not need to lock the process // id wrap lock because tlb synchronization is handled by hardware. // addic. r.11,r.11,1 // bump master sequence number bne+ spnsw // jif sequence number did not wrap // // The master sequence number has wrapped, this is 4 billion * 16 million // processes,... not too shabby. We start the sequence again at 2 in case // there are system processes that have been running since the system first // started. // li r.11,2 // start again at 2 spnsw: stw r.11,0(r.10) // save new master sequence number stw r.11,PrProcessSequence(r.3) // save new process sequence num bl ..KeFlushCurrentTb // flush entire HPT (and all processor // TLBs) lwz r.0,spLR(r.sp) // get swap process' return address lwz r.3,spR3(r.sp) // get new process address lwz r.8,[toc]KiMasterPid(r.toc) // get &KiMasterPid addi r.sp,r.sp,spFrameLength // return stack frame li r.7,16 // set new PID mtlr r.0 b spnp // continue in KiSwapProcess DUMMY_EXIT(KxSwapProcess) //++ // // VOID // KiIdleLoop ( // VOID // ) // // Routine Description: // // This is the idle loop for NT. This code runs in a thread for // each processor in the system. The idle thread runs at IRQL // DISPATCH_LEVEL and polls for work. // // Arguments: // // None. // // Return Value: // // None. (This routine never returns). // // Non-volatile register usage is as follows. // // r.14 --unused - available -- // r.15 Address of KdDebuggerEnabled // r.16 Kernel TOC (backup) // r.17 Idle loop MSR with Interrupts DISABLED // r.18 Idle loop MSR with Interrupts ENABLED // r.19 HalProcessorIdle entry point // r.20 HAL's TOC // r.21 Debugger poll count // r.22 Address of KeTickCount // r.23 Zero // r.24 Address of dispatcher database lock (MP) (backup for r28) // r.25 DpcListHead // r.26 --unused - available -- // r.27 Address of Context Swap lock // r.28 Address of dispatcher database lock (MP) // r.29 Address of Processor Control Block // // When another thread is selected to run, SwapContext is called. // Normally, callers of SwapContext are responsible for saving and // restoring non-volatile regs r.14 and r.26 thru r.31. SwapContext // saves/restores gprs r.15 thru r.25. The idle loop never returns so // previous contents of r.14 and r.26 thru r.31 are not saved. The // idle loop pre-initializes the storage area where SwapContext would // normally save r.15 thru r.25 with values that the idle loop needs // in those registers upon return from SwapContext and skips over the // register save on the way into SwapContext (alternate entry point // SwapFromIdle). All callers to SwapContext pass the following // arguments- // // r.27 Address of Context Swap lock (&KiContextSwapLock) // r.28 Address of dispatcher database lock (&KiDispatcherLock) // r.29 Address of PRCB // r.30 Address of OLD thread object // r.31 Address of NEW thread object // // The idle loop does not have a fixed use for regs r.30 and r.31. // r.29 contains the correct value for this processor. r.14 and // r.26 contents are unknown and must be regenerated upon return // from SwapContext. The assignment of function to these registers // was chosen for easy regeneration of content. // // Note also that r.21 was assigned in the range of registers // restored by SwapContext so that it is reset to its initial // values whenever SwapContext is called. // //-- #define rDbg r.15 #define rKTOC r.16 #define rIntOff r.17 #define rIntOn r.18 #define rHalIdle r.19 #define rHalToc r.20 #define rDbgCount r.21 #define rTickP r.22 #define rZero r.23 #define rDispLkSave r.24 #define rDPCHEAD r.25 #define rDispLk r.28 #define rPrcb r.29 SPECIAL_ENTRY_S(KiIdleLoop,_TEXT$00) mflr r.0 // get return address stwu r.sp, -kscFrameLength(r.sp) // buy stack frame stw r.0, kscLR(r.sp) // save return address PROLOGUE_END(KiIdleLoop) // // Setup initial global register values // ori rKTOC, r.toc, 0 // backup kernel's TOC lwz rPrcb, KiPcr+PcPrcb(r.0) // Address of PCB to rPrcb lwz rTickP, [toc]KeTickCount(r.toc) // Address of KeTickCount lwz rDbg, [toc]KdDebuggerEnabled(r.toc)// Addr KdDebuggerEnabled lwz rHalToc,[toc]__imp_HalProcessorIdle(r.toc) lwz rHalToc,0(rHalToc) lwz rHalIdle,0(rHalToc) // HalProcessorIdle entry lwz rHalToc,4(rHalToc) // HAL's TOC li rZero, 0 // Keep zero around, we use it mfmsr rIntOff // get current machine state rlwinm rIntOff, rIntOff, 0, 0xffff7fff // clear interrupt enable ori rIntOn, rIntOff, 0x00008000 // set interrupt enable #if !defined(NT_UP) lwz rDispLk, [toc]KiDispatcherLock(r.toc)// get &KiDispatcherLock lwz r.27, [toc]KiContextSwapLock(r.toc) // get &KiContextSwapLock #endif addi rDPCHEAD, rPrcb, PbDpcListHead // compute DPC listhead address li rDbgCount, 0 // Clear breakin loop counter #if !defined(NT_UP) ori rDispLkSave, rDispLk, 0 // copy &KiDispatcherLock #endif // // Registers 15 thru 25 are normally saved by SwapContext but the idle // loop uses an alternate entry that skips the save by SwapContext. // SwapContext will still restore them so we set up the stack so what // we want is what gets restored. This is especially useful for things // whose values need to be reset after SwapContext is called, eg rDbgCount. // lwz r.4, [toc]__imp_KeLowerIrql(r.toc) // &&fd(KeLowerIrql) stw r.15, swFrame + ExGpr15(r.sp) stw r.16, swFrame + ExGpr16(r.sp) stw r.17, swFrame + ExGpr17(r.sp) lwz r.4, 0(r.4) // &fd(KeLowerIrql) stw r.18, swFrame + ExGpr18(r.sp) stw r.19, swFrame + ExGpr19(r.sp) stw r.20, swFrame + ExGpr20(r.sp) lwz r.5, 0(r.4) // &KeLowerIrql stw r.21, swFrame + ExGpr21(r.sp) stw r.22, swFrame + ExGpr22(r.sp) stw r.23, swFrame + ExGpr23(r.sp) stw r.24, swFrame + ExGpr24(r.sp) stw r.25, swFrame + ExGpr25(r.sp) // // Control reaches here with IRQL at HIGH_LEVEL. Lower IRQL to // DISPATCH_LEVEL and set wait IRQL of idle thread. // mtctr r.5 lwz r.toc, 4(r.4) // HAL's TOC lwz r.11, KiPcr+PcCurrentThread(r.0) // Lower thread and processor li r.3, DISPATCH_LEVEL // IRQL to DISPATCH_LEVEL. stb r.3, ThWaitIrql(r.11) bctrl ori r.toc, rKTOC, 0 // restore our TOC // // In a multiprocessor system the boot processor proceeds directly into // the idle loop. As other processors start executing, however, they do // not directly enter the idle loop and spin until all processors have // been started and the boot master allows them to proceed. // #if !defined(NT_UP) lwz r.4, [toc]KiBarrierWait(r.toc) BarrierWait: lwz r.3, 0(r.4) // get barrier wait value cmpwi r.3, 0 // if ne spin until allowed bne BarrierWait // to proceed. lbz r.3, PbNumber(rPrcb) // get processor number cmpwi cr.4, r.3, 0 // save "processor == 0 ?" #endif // // Set condition register and swap return values in the swap frame. // mfcr r.3 // save condition register stw r.3, swFrame + SwConditionRegister(r.sp) bl FindIdleReturn FindIdleReturn: mflr r.3 addi r.3, r.3, KiIdleReturn - FindIdleReturn stw r.3, swFrame + SwSwapReturn(r.sp)// save return address // // The following loop is executed for the life of the system. // IdleLoop: #if DBG #if !defined(NT_UP) bne cr.4, CheckDpcList // if ne, not processor zero #endif // // Check if the debugger is enabled, and whether it is time to poll // for a debugger breakin. (This check is only performed on cpu 0). // subic. rDbgCount, rDbgCount, 1 // decrement poll counter bge+ CheckDpcList // jif not time yet. lbz r.3, 0(rDbg) // check if debugger enabled li rDbgCount, 20 * 1000 // set breakin loop counter cmpwi r.3, 0 beq+ CheckDpcList // jif debugger not enabled bl ..KdPollBreakIn // check if breakin requested cmpwi r.3, 0 beq+ CheckDpcList // jif no breakin request li r.3, DBG_STATUS_CONTROL_C // send status to debugger bl ..DbgBreakPointWithStatus #endif // // Disable interrupts and check if there is any work in the DPC list. // CheckDpcList: mtmsr rIntOn // give interrupts a chance isync // to interrupt spinning mtmsr rIntOff // disable interrupts cror 0,0,0 // N.B. 603e/ev Errata 15 // // Process the deferred procedure call list for the current processor. // lwz r.3, LsFlink(rDPCHEAD) // get address of first entry cmpw r.3, rDPCHEAD // is list empty? beq CheckNextThread // if eq, DPC list is empty ori r.31, rDPCHEAD, 0 bl ..KiProcessDpcList // process the DPC list // // Clear dispatch interrupt pending. // stb rZero, KiPcr+PcDispatchInterrupt(r.0) // clear pending DPC interrupt #if DBG li rDbgCount, 0 // clear breakin loop counter #endif // // Check if a thread has been selected to run on this processor // CheckNextThread: lwz r.31, PbNextThread(rPrcb) // get address of next thread cmpwi r.31, 0 beq Idle // jif no thread to execute // // A thread has been selected for execution on this processor. Acquire // dispatcher database lock, get the thread address again (it may have // changed), clear the address of the next thread in the processor block, // and call swap context to start execution of the selected thread. // // N.B. If the dispatcher database lock cannot be obtained immediately, // then attempt to process another DPC rather than spinning on the // dispatcher database lock. // #if !defined(NT_UP) TRY_TO_ACQUIRE_SPIN_LOCK(rDispLk, rDispLk, r.11, LkDisp, CheckDpcList) #endif mtmsr rIntOn // enable interrupts cror 0,0,0 // N.B. 603e/ev Errata 15 #if !defined(NT_UP) lwz r.31, PbNextThread(rPrcb) // get next thread address #endif lwz r.30, PbCurrentThread(rPrcb) // get current thread address stw rZero, PbNextThread(rPrcb) // clear address of next thread stw r.31, PbCurrentThread(rPrcb) // set new thread current // // Acquire the context swap lock so the address space of the old process // cannot be deleted and then release the dispatcher database lock. In // this case the old process is the system process, but the context swap // code releases the context swap lock so it must be acquired. // // N.B. This lock is used to protect the address space until the context // switch has sufficiently progressed to the point where the address // space is no longer needed. This lock is also acquired by the reaper // thread before it finishes thread termination. // #if !defined(NT_UP) ACQUIRE_SPIN_LOCK(r.27,r.31,r.3,LkCtxSw1,LkCtxSw1Spin) #endif // // Set the new thread's state to Running before releasing the dispatcher lock. // li r.3, Running // set state of new thread stb r.3, ThState(r.31) // to running. #if !defined(NT_UP) RELEASE_SPIN_LOCK(r.28,rZero) #endif bl ..SwapFromIdle // swap context to new thread KiIdleReturn: #if !defined(NT_UP) ori rDispLk, rDispLkSave, 0 // restore &KiDispatcherLock // // rDbgCount (r.21) will have been reset to 0 by the register restore // at the end of SwapContext. // // If processor 0, check for debugger breakin, otherwise just check for // DPCs again. // #endif b IdleLoop // // There are no entries in the DPC list and a thread has not been selected // for execution on this processor. Call the HAL so power managment can be // performed. // // N.B. The HAL is called with interrupts disabled. The HAL will return // with interrupts enabled. // Idle: mtctr rHalIdle // set entry point ori r.toc, rHalToc, 0 // set HAL's TOC bctrl // call HalProcessorIdle isync // give HAL's interrupt enable // a chance to take effect ori r.toc, rKTOC, 0 // restore ntoskrnl's TOC b IdleLoop #if !defined(NT_UP) SPIN_ON_SPIN_LOCK(r.27,r.3,LkCtxSw1,LkCtxSw1Spin) #endif DUMMY_EXIT(KiIdleLoop) #undef rDPCHEAD SBTTL("Process Deferred Procedure Call List") //++ // // Routine Description: // // This routine is called to process the given deferred procedure call // list. // // If called from KiDispatchInterrupt, we will have been switched to // the interrupt stack, the new stack pointer is in r.sp and entry is // at ..KiProcessDpcList.alt. // // If called from the idle loop, we run on the idle loop thread's // stack and no special action is needed. However, the idle loop // does not pass r.9 as we expect it and only passes r.0 = 0 on MP // systems. We take advantage of the separate entry points to set // these registers appropriately. // // N.B. Interrupts must be disabled on entry to this routine. Control // is returned to the caller with the same conditions true. // // Arguments: // // None. // // On entry: // // r.0 - Zero // r.9 - Machine State Register prior to disabling interrupts. // rPrcb r.29 - address of processor control block. // r.31 - address of DPC list head. // // On exit: // // r.0 - Zero // r.9 - Machine State Register prior to disabling interrupts. // rPrcb r.29 - address of processor control block. // r.31 - address of DPC list head. // // Return value: // // None. //-- .struct 0 .space StackFrameHeaderLength dp.lr: .space 4 dp.toc: .space 4 #if DBG dp.func:.space 4 dp.strt:.space 4 dp.cnt: .space 4 dp.time:.space 4 #endif .align 3 // ensure stack frame length is multile of 8 dp.framelen: .text #if DBG # define rDpStart r.22 # define rDpCount r.23 # define rDpTime r.24 #endif SPECIAL_ENTRY_S(KiProcessDpcList,_TEXT$00) stwu r.sp, -dp.framelen(r.sp) // buy stack frame // see routine description for why we do the following. ori r.9, rIntOn, 0 // get MSR interrupts enabled #if defined(NT_UP) li r.0, 0 #endif ..KiProcessDpcList.alt: mflr r.7 // save return address // save regs we will use,... don't need to save 29 and 31 as they // were saved by our caller and currently contain the values we want. stw r.toc, dp.toc(r.sp) #if DBG stw rDpTime, dp.time(r.sp) stw rDpCount, dp.cnt(r.sp) stw rDpStart, dp.strt(r.sp) #endif stw r.7, dp.lr(r.sp) // save Link Register PROLOGUE_END(KiProcessDpcList) DpcCallRestart: stw r.sp, PbDpcRoutineActive(rPrcb) // set DPC routine active // // Process the DPC list. // DpcCall: #if !defined(NT_UP) addi r.7, rPrcb, PbDpcLock // compute DPC lock address ACQUIRE_SPIN_LOCK(r.7, r.7, r.0, spinlk2, spinlk2spin) #endif lwz r.3, LsFlink(r.31) // get address of first entry lwz r.12, LsFlink(r.3) // get addr of next entry cmpw r.3, r.31 // is list empty? subi r.3, r.3, DpDpcListEntry // subtract DpcListEntry offset beq- UnlkDpc0 // if yes, release the lock. // // Get deferred routine address, this is done early as what // we actually have is a function descriptor's address and we // need to get the entry point address. // lwz r.11, DpDeferredRoutine(r.3) lwz r.8, PbDpcQueueDepth(rPrcb) // get DPC queue depth // // remove entry from list // stw r.12, LsFlink(r.31) // set addr of next in header stw r.31, LsBlink(r.12) // set addr of previous in next lwz r.10, 0(r.11) // get DPC code address // // entry removed, set up arguments for DPC proc // // args are- // dpc object address (r.3) // deferred context (r.4) // system argument 1 (r.5) // system argument 2 (r.6) // // note, the arguments must be loaded from the DPC object BEFORE // the inserted flag is cleared to prevent the object being // overwritten before its time. // lwz r.4, DpDeferredContext(r.3) lwz r.5, DpSystemArgument1(r.3) lwz r.6, DpSystemArgument2(r.3) subi r.8, r.8, 1 // decrement DPC queue depth stw r.8, PbDpcQueueDepth(rPrcb) // stw r.0, DpLock(r.3) // clear DPC inserted state #if !defined(NT_UP) RELEASE_SPIN_LOCK(r.7, r.0) #endif mtctr r.10 // ready address for branch ENABLE_INTERRUPTS(r.9) lwz r.toc, 4(r.11) // get DPC toc pointer #if DBG lwz rDpStart, KiPcr2 + Pc2TickCountLow(r.0) // get current time lwz rDpCount, PbInterruptCount(rPrcb)// get current interrupt count lwz rDpTime, PbInterruptTime(rPrcb) // get current interrupt time stw r.10, dp.func(r.sp) #endif bctrl // call DPC routine li r.0, 0 // reset zero constant #if DBG lbz r.10, KiPcr+PcCurrentIrql(r.0) // get current IRQL cmplwi r.10, DISPATCH_LEVEL // check if < DISPATCH_LEVEL blt DpcBadIrql // jif IRQL < DISPATCH_LEVEL DpcIrqlOk: lwz r.12, KiPcr2 + Pc2TickCountLow(r.0) // calculate time spent in sub r.12, r.12, rDpStart // r.12 = time cmpwi r.12, 100 bge DpcTookTooLong // jif >= 1 second DpcTimeOk: #endif // // Check to determine if any more DPCs are available to process. // DISABLE_INTERRUPTS(r.9, r.10) lwz r.3, LsFlink(r.31) // get address of first entry cmpw r.3, r.31 // is list empty? bne- DpcCall // if no, process it // // Clear DpcRoutineActive, then check one last time that the DPC queue is // empty. This is required to close a race condition with the DPC queueing // code where it appears that a DPC routine is active (and thus an // interrupt is not requested), but this code has decided that the queue // is empty and is clearing DpcRoutineActive. // stw r.0, PbDpcRoutineActive(rPrcb) stw r.0, PbDpcInterruptRequested(rPrcb) // clear DPC interrupt requested eieio // force writes out lwz r.3, LsFlink(r.31) cmpw r.3, r.31 bne- DpcCallRestart DpcDone: // // List is empty, restore non-volatile registers we have used. // lwz r.10, dp.lr(r.sp) // get link register #if DBG lwz rDpTime, dp.time(r.sp) lwz rDpCount, dp.cnt(r.sp) lwz rDpStart, dp.strt(r.sp) #endif // // Return to caller. // lwz r.toc, dp.toc(r.sp) // restore kernel toc mtlr r.10 // set return address lwz r.sp, 0(r.sp) // release stack frame blr // return UnlkDpc0: // // The DPC list became empty while we were acquiring the DPC queue lock. // Clear DPC routine active. The race condition mentioned above doesn't // exist here because we hold the DPC queue lock. // stw r.0, PbDpcRoutineActive(rPrcb) #if !defined(NT_UP) RELEASE_SPIN_LOCK(r.7, r.0) #endif b DpcDone // // DpcTookTooLong, DpcBadIrql // // Come here is it took >= 1 second to execute a DPC routine. This is way // too long, assume something is wrong and breakpoint. // // This code is out of line to avoid wasting cache space for something that // (hopefully) never happens. // #if DBG DpcTookTooLong: lwz r.toc, dp.toc(r.sp) // restore kernel's TOC lwz r.11, PbInterruptCount(rPrcb) // get current interrupt count lwz r.10, PbInterruptTime(rPrcb) // get current interrupt time lwz r.toc, dp.toc(r.sp) // restore our toc sub r.11, r.11, rDpCount // compute number of interrupts sub r.10, r.10, rDpTime // compute interrupt time bl ..DbgBreakPoint // execute debug breakpoint b DpcTimeOk // continue DpcBadIrql: lwz r.toc, dp.toc(r.sp) // restore kernel's TOC bl ..DbgBreakPoint // breakpoint b DpcIrqlOk // continue #endif #if !defined(NT_UP) SPIN_ON_SPIN_LOCK(r.7, r.11, spinlk2, spinlk2spin) #endif DUMMY_EXIT(KiProcessDpcList) SBTTL("Dispatch Interrupt") #define rDPCHEAD r.31 //++ // // Routine Description: // // This routine is entered as the result of a software interrupt generated // at DISPATCH_LEVEL. Its function is to process the Deferred Procedure Call // (DPC) list, and then perform a context switch if a new thread has been // selected for execution on the processor. // // This routine is entered at IRQL DISPATCH_LEVEL with the dispatcher // database unlocked. When a return to the caller finally occurs, the // IRQL remains at DISPATCH_LEVEL, and the dispatcher database is still // unlocked. // // Arguments: // // None. // // Outputs: // ( for call to KiProcessDpcList ) // r.3 - address of first dpc in list // r.0 - Zero // r.9 - Machine State Register prior to disabling interrupts. // rPrcb r.29 - address of processor control block. // rDPCHEAD r.31 - address of DPC listhead. // r.9 Machine State Register prior to disabling interrupts. // // ( for call to KiDispIntSwapContext ) // // r.28 pointer to Dispatcher Database Lock // r.29 rPrcb pointer to the processor control block // r.31 NTH pointer to new thread // // Return Value: // // None. // //-- .struct 0 .space StackFrameHeaderLength di.lr: .space 4 di.28: .space 4 di.29: .space 4 di.30: .space 4 di.31: .space 4 .align 3 // ensure stack frame length is multile of 8 di.framelen: SPECIAL_ENTRY_S(KiDispatchInterrupt,_TEXT$00) mflr r.0 stwu r.sp, -di.framelen(r.sp) stw r.29, di.29(r.sp) lwz rPrcb,KiPcr+PcPrcb(r.0) stw r.30, di.30(r.sp) stw r.31, di.31(r.sp) stw r.0, di.lr(r.sp) stw r.28, di.28(r.sp) PROLOGUE_END(KiDispatchInterrupt) // // Setup commonly used constants // lwz r.3, PbDpcBypassCount(rPrcb) // get DPC bypass count li r.0, 0 // zero addi rDPCHEAD, rPrcb, PbDpcListHead // compute DPC listhead address addi r.3, r.3, 1 // increment DPC bypass count stw r.3, PbDpcBypassCount(rPrcb) // store new DPC bypass count // // Process the deferred procedure call list. // PollDpcList: DISABLE_INTERRUPTS(r.9, r.8) lwz r.3, LsFlink(rDPCHEAD) // get address of first entry stb r.0, KiPcr+PcDispatchInterrupt(r.0) cmpw r.3, rDPCHEAD // list has entries? beq- di.empty // jif list is empty // // Switch to the interrupt stack // lwz r.6, KiPcr+PcInterruptStack(r.0)// get addr of interrupt stack lwz r.28, KiPcr+PcInitialStack(r.0) // get current stack base lwz r.30, KiPcr+PcStackLimit(r.0) // get current stack limit subi r.4, r.6, KERNEL_STACK_SIZE // compute stack limit stw r.sp,KiPcr+PcOnInterruptStack(r.0)// flag ON interrupt stack stw r.sp, -dp.framelen(r.6) // save new back pointer // // N.B. Can't step thru the next two instructions. // stw r.4, KiPcr+PcStackLimit(r.0) // set stack limit stw r.6, KiPcr+PcInitialStack(r.0) // set current base to int stk subi r.sp, r.6, dp.framelen // calc new sp bl ..KiProcessDpcList.alt // process all DPCs for this // processor. // // N.B. KiProcessDpcList left r.0, r.9 intact. // // Return from KiProcessDpcList switched back to the proper stack, // update PCR to reflect this. // stw r.30, KiPcr+PcStackLimit(r.0) // restore stack limit stw r.28, KiPcr+PcInitialStack(r.0) // set old stack current stw r.0, KiPcr+PcOnInterruptStack(r.0)// clear ON interrupt stack di.empty: ENABLE_INTERRUPTS(r.9) // // Check to determine if quantum end has occurred. // // N.B. If a new thread is selected as a result of processing a quantum // end request, then the new thread is returned with the dispatcher // database locked. Otherwise, NULL is returned with the dispatcher // database unlocked. // lwz r.3, KiPcr+PcQuantumEnd(r.0) // get quantum end indicator cmpwi r.3, 0 // if 0, no quantum end request beq di.CheckForNewThread stw r.0, KiPcr+PcQuantumEnd(r.0) // clear quantum end indicator bl ..KiQuantumEnd // process quantum end cmpwi r.3, 0 // new thread selected? li r.0, 0 // reset r.0 to zero // // If KiQuantumEnd returned no new thread to run, the dispatcher // database is unlocked, get out. // beq+ di.exit #if !defined(NT_UP) // // Even though the dispatcher database is already locked, we are expected // to pass the address of the lock in r.28. // lwz r.28, [toc]KiDispatcherLock(r.toc)// get &KiDispatcherLock #endif b di.Switch // switch to new thread // // Check to determine if a new thread has been selected for execution on // this processor. // di.CheckForNewThread: lwz r.3, PbNextThread(rPrcb) // get address of next thread cmpwi r.3, 0 // is there a new thread? beq di.exit // no, branch. #if !defined(NT_UP) lwz r.28, [toc]KiDispatcherLock(r.toc)// get &KiDispatcherLock // // Lock dispatcher database and reread address of next thread object since it // is possible for it to change in a multiprocessor system. (leave address // of lock in r.28). // TRY_TO_ACQUIRE_SPIN_LOCK(r.28, r.28, r.11, di.spinlk, PollDpcList) #endif di.Switch: lwz r.31, PbNextThread(rPrcb) // get thread address (again) stw r.0, PbNextThread(rPrcb) // clear addr of next thread obj // // Reready current thread for execution and swap context to the // selected thread. We do this indirectly thru KiDispIntSwapContext // to avoid saving and restoring so many registers for the cases // when KiDispatchInterrupt does not thread switch. // bl ..KiDispIntSwapContext // swap to new thread di.exit: lwz r.0, di.lr(r.sp) lwz r.31, di.31(r.sp) lwz r.30, di.30(r.sp) mtlr r.0 lwz r.29, di.29(r.sp) lwz r.28, di.28(r.sp) addi r.sp, r.sp, di.framelen SPECIAL_EXIT(KiDispatchInterrupt) //++ // // VOID // KiDispIntSwapContext ( // IN PKTHREAD Thread // ) // // Routine Description: // // This routine is called to perform a context switch to the specified // thread. The current (new previous) thread is re-readied for execution. // // Since this routine is called as subroutine all volatile registers are // considered free. // // Our caller has saved and will restore gprs 28 thru 31 and does not // care if we trash them. // // This routine is entered at IRQL DISPATCH_LEVEL with the dispatcher // database locked. When a return to the caller finally occurs, the // dispatcher database is unlocked. // // Arguments: // // r.28 pointer to Dispatcher Database Lock // r.29 rPrcb pointer to the processor control block // r.31 NTH pointer to new thread // // Outputs: ( for call to SwapContext ) // // r.27 pointer to KiContextSwapLock // r.28 pointer to Dispatcher Database Lock // r.29 rPrcb pointer to processor control block // r.30 OTH pointer to old thread // r.31 NTH pointer to new thread // // Return Value: // // Wait completion status (r.3). // //-- .struct 0 .space swFrameLength kdiscLR:.space 4 .align 3 // ensure 8 byte alignment kdiscFrameLength: .align 6 // cache line alignment SPECIAL_ENTRY_S(KiDispIntSwapContext,_TEXT$00) mflr r.0 // get return address lwz r.30, KiPcr+PcCurrentThread(r.0) // get current (old) thread stwu r.sp, -kdiscFrameLength(r.sp) // buy stack frame stw r.14, swFrame + ExGpr14(r.sp) // save gpr 14 stw r.26, swFrame + ExGpr26(r.sp) // save gprs 26 and 27 stw r.27, swFrame + ExGpr27(r.sp) // // // Gprs 28, 29, 30 and 31 saved/restored by KiDispatchInterrupt // stw r.0, kdiscLR(r.sp) // save return address PROLOGUE_END(KiDispIntSwapContext) stw r.31, PbCurrentThread(rPrcb) // set new thread current // // Reready current thread and swap context to the selected thread. // ori r.3, r.30, 0 #if !defined(NT_UP) lwz r.27, [toc]KiContextSwapLock(r.2) #endif bl ..KiReadyThread // reready current thread bl ..SwapContext // switch threads // // Restore registers and return. // lwz r.0, kdiscLR(r.sp) // restore return address lwz r.26, swFrame + ExGpr26(r.sp) // restore gpr 26 and 27 mtlr r.0 // set return address lwz r.27, swFrame + ExGpr27(r.sp) // // // Gprs 28, 29, 30 and 31 saved/restored by KiDispatchInterrupt // lwz r.14, swFrame + ExGpr14(r.sp) // restore gpr 14 addi r.sp, r.sp, kdiscFrameLength // return stack frame SPECIAL_EXIT(KiDispIntSwapContext) //++ // // VOID // KiRequestSoftwareInterrupt (KIRQL RequestIrql) // // Routine Description: // // This function requests a software interrupt at the specified IRQL // level. // // Arguments: // // RequestIrql (r.3) - Supplies the request IRQL value. // Allowable values are APC_LEVEL (1) // DPC_LEVEL (2) // // Return Value: // // None. // //-- LEAF_ENTRY(KiRequestSoftwareInterrupt) lbz r.6, KiPcr+PcCurrentIrql(r.0) // get current IRQL rlwinm r.4, r.3, 31, 0x1 // transform 1 or 2 to 0 or 1 li r.5, 1 // non-zero value cmpw r.6, r.3 // is current IRQL < requested IRQL? stb r.5, KiPcr+PcSoftwareInterrupt(r.4) // set interrupt pending blt ..KiDispatchSoftwareInterrupt // jump to dispatch interrupt if // current IRQL low enough (note // that this is a jump, not a call) LEAF_EXIT(KiRequestSoftwareInterrupt)