|
|
//++//// Copyright (c) 1989-2000 Microsoft Corporation//// Component Name://// NT / KE //// Module Name://// ctxswap.s//// Abstract://// This module implements the IA64 Process and Thread Context Swaps.//// Author://// David N. Cutler (davec) 5-Mar-1989//// Environment://// Kernel mode only//// Revision History:// // Bernard Lint Jul-12-1995//// Initial IA64 version////--
#include "ksia64.h"
.file "ctxswap.s" .text
//// Globals imported://
.global KeTickCount .global KiMasterSequence .global KiMasterRid .global PPerfGlobalGroupMask PublicFunction(KeBugCheck) PublicFunction(KiRetireDpcList) PublicFunction(KiSaveExceptionFrame) PublicFunction(KiRestoreExceptionFrame) PublicFunction(KiActivateWaiterQueue) PublicFunction(KiReadyThread) PublicFunction(KiQueueReadyThread) PublicFunction(KeFlushEntireTb) PublicFunction(KiQuantumEnd) PublicFunction(KiSyncNewRegionId) PublicFunction(KiCheckForSoftwareInterrupt) PublicFunction(KiSaveHigherFPVolatileAtDispatchLevel) PublicFunction(KiAcquireSpinLock) PublicFunction(KeReleaseQueuedSpinLockFromDpcLevel) PublicFunction(KeAcquireQueuedSpinLockRaiseToSynch) PublicFunction(KiTimerExpiration) PublicFunction(WmiTraceContextSwap)
#include "icecap.h"
#if DBG
PublicFunction(KeBugCheckEx)#endif // DBG
SBTTL("Swap Thread")//++//--------------------------------------------------------------------//// BOOLEAN// KiSwapContext (// IN PKTHREAD OldThread// IN PKTHREAD NewThread// )//// Routine Description://// This routine saves the non-volatile registers, marshals the// arguments for SwapContext and calls SwapContext to perform// the actual thread switch.//// Arguments://// OldThread - Supplies the address of the old thread.// NewThread - Supplies the address of the new thread.//// Return Value://// If a kernel APC is pending, then a value of TRUE is returned.// Otherwise, FALSE is returned.//// Notes://// GP valid on entry -- GP is not switched, just use kernel GP//--------------------------------------------------------------------//--
NESTED_ENTRY(KiSwapContext)
//// Register aliases//
pNoAPC = pt2 // do not dispatch APC
rpT1 = t0 // temp pointer rpT2 = t1 // temp pointer rT1 = t10 // temp regs
PROLOGUE_BEGIN
#ifdef _CAPKERN
.regstk 2, 2, 2, 0 alloc t16 = ar.pfs, 2, 2, 2, 0#else
.regstk 2, 2, 1, 0 alloc t16 = ar.pfs, 2, 2, 1, 0#endif
.save rp, loc0 mov loc0 = brp .fframe SwitchFrameLength add sp = -SwitchFrameLength, sp ;;
.save ar.unat, loc1 mov loc1 = ar.unat add t0 = ExFltS19+SwExFrame+STACK_SCRATCH_AREA, sp add t1 = ExFltS18+SwExFrame+STACK_SCRATCH_AREA, sp ;;
.save.gf 0x0, 0xC0000 stf.spill [t0] = fs19, ExFltS17-ExFltS19 stf.spill [t1] = fs18, ExFltS16-ExFltS18 ;;
.save.gf 0x0, 0x30000 stf.spill [t0] = fs17, ExFltS15-ExFltS17 stf.spill [t1] = fs16, ExFltS14-ExFltS16 mov t10 = bs4 ;;
.save.gf 0x0, 0xC000 stf.spill [t0] = fs15, ExFltS13-ExFltS15 stf.spill [t1] = fs14, ExFltS12-ExFltS14 mov t11 = bs3 ;;
.save.gf 0x0, 0x3000 stf.spill [t0] = fs13, ExFltS11-ExFltS13 stf.spill [t1] = fs12, ExFltS10-ExFltS12 mov t12 = bs2 ;;
.save.gf 0x0, 0xC00 stf.spill [t0] = fs11, ExFltS9-ExFltS11 stf.spill [t1] = fs10, ExFltS8-ExFltS10 mov t13 = bs1 ;;
.save.gf 0x0, 0x300 stf.spill [t0] = fs9, ExFltS7-ExFltS9 stf.spill [t1] = fs8, ExFltS6-ExFltS8 mov t14 = bs0 ;;
.save.gf 0x0, 0xC0 stf.spill [t0] = fs7, ExFltS5-ExFltS7 stf.spill [t1] = fs6, ExFltS4-ExFltS6 mov t15 = ar.lc ;;
.save.gf 0x0, 0x30 stf.spill [t0] = fs5, ExFltS3-ExFltS5 stf.spill [t1] = fs4, ExFltS2-ExFltS4 ;;
.save.f 0xC stf.spill [t0] = fs3, ExFltS1-ExFltS3 // save fs3 stf.spill [t1] = fs2, ExFltS0-ExFltS2 // save fs2 ;;
.save.f 0x3 stf.spill [t0] = fs1, ExBrS4-ExFltS1 // save fs1 stf.spill [t1] = fs0, ExBrS3-ExFltS0 // save fs0 ;;
.save.b 0x18 st8 [t0] = t10, ExBrS2-ExBrS4 // save bs4 st8 [t1] = t11, ExBrS1-ExBrS3 // save bs3 ;;
.save.b 0x6 st8 [t0] = t12, ExBrS0-ExBrS2 // save bs2 st8 [t1] = t13, ExIntS2-ExBrS1 // save bs1 ;;
.save.b 0x1 st8 [t0] = t14, ExIntS3-ExBrS0 // save bs0 ;;
.save.gf 0xC, 0x0 .mem.offset 0,0 st8.spill [t0] = s3, ExIntS1-ExIntS3 // save s3 .mem.offset 8,0 st8.spill [t1] = s2, ExIntS0-ExIntS2 // save s2 ;;
.save.gf 0x3, 0x0 .mem.offset 0,0 st8.spill [t0] = s1, ExApLC-ExIntS1 // save s1 .mem.offset 8,0 st8.spill [t1] = s0, ExApEC-ExIntS0 // save s0 ;;
.savepsp ar.pfs, ExceptionFrameLength-ExApEC-STACK_SCRATCH_AREA st8 [t1] = t16, ExIntNats-ExApEC mov t4 = ar.unat // captured Nats of s0-s3 ;;
.savepsp ar.lc, ExceptionFrameLength-ExApLC-STACK_SCRATCH_AREA st8 [t0] = t15 .savepsp @priunat, ExceptionFrameLength-ExIntNats-STACK_SCRATCH_AREA st8 [t1] = t4 // save Nats of s0-s3
PROLOGUE_END
// // For the call to SwapContext- // // s0 // Prcb address // s1 // old thread address // s2 // new thread address // pt0 = 1 //
mov s2 = a1 // s2 <- New Thread movl t0 = KiPcr + PcPrcb ;;
mov s1 = a0 // OldThread ld8 s0 = [t0] ;;
//// Swap context to the next thread.//
CAPSTART(KiSwapContext,SwapContext) br.call.sptk brp = SwapContext // call SwapContext(prcb, OldTh, NewTh) CAPEND(KiSwapContext) ;;
//// Deallocate exception/switch frame.//// N.B. SwapContext releases the dispatcher database lock.//// N.B. v0 contains the kernel APC pending state on return, ie, 0 if// no APC pending, 1 if APC pending. v0 will be forced to 0 if// the new IRQL doesn't allow APCs.//// N.B. KiRestoreExceptionFrame doesn't touch v0, t21 or t22.//
add rpT2 = ThWaitIrql, s2 // -> ThWaitIrql add rpT1 = ExApEC+SwExFrame+STACK_SCRATCH_AREA, sp add out0 = STACK_SCRATCH_AREA+SwExFrame, sp ;;
ld1 t21 = [rpT2] // t21 = original wait IRQL ld8 t22 = [rpT1] // t22 = PFS br.call.sptk brp = KiRestoreExceptionFrame ;;
mov brp = loc0 cmp.ne pNoAPC = zero, t21 // no APC if IRQL != 0 ;;
mov ar.unat = loc1 nop.f 0 mov ar.pfs = t22
.restore add sp = SwitchFrameLength, sp(pNoAPC) mov v0 = zero br.ret.sptk brp ;;
NESTED_EXIT(KiSwapContext) SBTTL("Swap Context to Next Thread")//++//--------------------------------------------------------------------// Routine://// SwapContext//// Routine Description://// This routine is called to swap context from one thread to the next.//// Arguments://// s0 - Address of Processor Control Block (PRCB).// s1 - Address of previous thread object.// s2 - Address of next thread object.//// Return value://// v0 - Kernel APC pending flag// s0 - Address of Processor Control Block (PRCB).// s1 - Address of previous thread object.// s2 - Address of current thread object.//// Note:// Kernel GP is not saved and restored across context switch//// !!WARNING!! - Thierry. 03/01/2000.// Be aware that this implementation is a result of performance analysis.// Please consider this when you are making changes...////--------------------------------------------------------------------//--
NESTED_ENTRY(SwapContext)
//// Register aliases//
rT1 = t1 // temp rT2 = t2 // temp rT3 = t3 // temp rNewproc = t4 // next process object rOldproc = t5 // previous process object rpThBSL = t6 // pointer to new thread backing store limit rpT1 = t7 // temp pointer rpT2 = t8 // temp pointer rpT3 = t9 // temp pointer rAr1 = t10 rAr2 = t11 rAr3 = t12 rAr4 = t13
rNewIKS = t14 // new initial kernel stack rNewKSL = t15 // new kernel stack limit rNewBSP = t16 // new thread BSP/BSPSTORE rOldBSP = t16 // old thread BSP rOldRNAT = t17 // old thread RNAT rNewRNAT = t17 // new thread RNAT rOldSbase = t18 // old thread kstack base
pUsTh = pt4 // is user thread? pLoop = pt4 // Need to loop waiting for context swap. pKrTh = pt5 // is user thread? pSave = pt7 // is high fp set dirty? pDiff = ps4 // if new and old process different pSame = ps5 // if new and old process same
//// Set new thread's state to running. Note this must be done// under the dispatcher lock so that KiSetPriorityThread sees// the correct state.//
PROLOGUE_BEGIN
#if !defined(NT_UP)
alloc rT2 = ar.pfs, 0, 0, 4, 0 mov rT1 = brp // move from brp takes 2 cycles ;;
add rpT2 = SwPFS+STACK_SCRATCH_AREA, sp add out0 = ThSwapBusy, s2 // Get address of new threads busy field ;;
.savesp ar.pfs, SwPFS+STACK_SCRATCH_AREA st8.nta [rpT2] = rT2, SwRp-SwPFS // save pfs ;;
.savesp brp, SwRp+STACK_SCRATCH_AREA st8.nta [rpT2] = rT1 // save return link
;;
// // Loop while the swap busy flag is set //loop1: ld1.acq out1 = [out0] ;;
cmp.ne pLoop = zero, out1 ;;
(pLoop) YIELD(pLoop) br.cond.spnt loop1
;;
mov out0 = ar.fpsr // move from ar.fpsr takes 12 cycles movl rpT1 = KiPcr+PcHighFpOwner // setup for prefetching ;;
{ .mmi lfetch [rpT1] cmp.ne pUsTh = zero, teb // test for ia32 save required // must not have a nop.f for next 10 cycles-- // Using temporarely the explicit templating // for the next cycles. add out1 = ThStackBase, s1 // move early to start access for rOldSbase{ .mmi add rpT1 = SwFPSR+STACK_SCRATCH_AREA, sp add rpT2 = SwPreds+STACK_SCRATCH_AREA, sp nop.i 0x0} ;;
{ .mmi ld8.nta rOldSbase = [out1] // speculative start early for ia32 saves lfetch.excl [rpT1] add out2 = ThNumber, s2 // setup for prefetching }{ .mmi mov.m ar.rsc = r0 // put RSE in lazy mode mov rOldBSP = ar.bsp // move from ar.bsp takes 12 cycles nop.i 0x0} ;;
{ .mmi lfetch [out2] nop.m 0x0 mov rT1 = pr // move from pr takes 2 cycles} ;;
{ .mmi flushrs mov rT3 = psr.um // move from psr.um takes 12 cycles nop.i 0x0} ;;
{ .mmi lfetch.excl [rpT2] mov.m rOldRNAT = ar.rnat // move from ar.rnat takes 5 cycles add out2 = @gprel(PPerfGlobalGroupMask), gp} ;;
{ .mli lfetch [out2] movl out3 = KiPcr + PcInterruptionCount // INTERRUPTION_LOGGING on or off, we are prefetching this line. // If any real performance problem is detected, we will undef these lines.} ;;
{ .mmi lfetch [out3] add rpT3 = SwRnat+STACK_SCRATCH_AREA, sp} ;;
#else // NT_UP
alloc rT2 = ar.pfs, 0, 0, 4, 0 cmp.ne pUsTh = zero, teb // test for ia32 save required ;;
mov.m ar.rsc = r0 // put RSE in lazy mode add out1 = ThStackBase, s1 // move early to start access for rOldSbase mov out0 = ar.fpsr // move from ar.fpsr takes 12 cycles // must not have a nop.f for next 10 cycles-- // Using temporarely the explicit templating // for the next cycles. ;;
{ .mmi ld8.nta rOldSbase = [out1] // speculative start early for ia32 saves mov rOldBSP = ar.bsp // move from ar.bsp takes 12 cycles add rpT1 = SwRp+STACK_SCRATCH_AREA, sp} ;;
flushrs mov rT3 = psr.um // move from psr.um takes 12 cycles add rpT2 = SwPFS+STACK_SCRATCH_AREA, sp ;;
mov.m rOldRNAT = ar.rnat // move from ar.rnat takes 5 cycles mov rT1 = brp // move from brp takes 2 cycles add rpT3 = ThState, s2 ;;
{ .mmi mov rAr1 = Running .savesp brp, SwRp+STACK_SCRATCH_AREA st8.nta [rpT1] = rT1, SwFPSR-SwRp // save return link nop.i 0x0 } ;;
{ .mii st1.nta [rpT3] = rAr1 // set thread state to Running mov rT1 = pr // move from pr takes 2 cycles nop.i 0x0 } ;;
{ .mii .savesp ar.pfs, SwPFS+STACK_SCRATCH_AREA st8.nta [rpT2] = rT2, SwPreds-SwPFS // save pfs add rpT3 = SwRnat+STACK_SCRATCH_AREA, sp nop.i 0x0 } ;;
#endif // NT_UP
{ .mmi st8.nta [rpT3] = rOldRNAT nop.m 0x0 nop.i 0x0 } st8 [rpT1] = out0, SwBsp-SwFPSR // save kernel FPSR st8 [rpT2] = rT1 // save preserved predicates ;;
st8.nta [rpT1] = rOldBSP add rpT3 = ThKernelBStore, s1 tbit.nz pSave = rT3, PSR_MFH // check mfh bit(pUsTh) br.call.spnt brp = SwapContextIA32Save ;;
st8.nta [rpT3] = rOldBSP(pSave) add out0 = -ThreadStateSaveAreaLength+TsHigherFPVolatile, rOldSbase(pSave) br.call.spnt brp = KiSaveHigherFPVolatileAtDispatchLevel ;;
PROLOGUE_END
//// ***** TBD ****** Save performance counters? (user vs. kernel)//
//// The following entry point is used to switch from the idle thread to// another thread.//
;;
ALTERNATE_ENTRY(SwapFromIdle)
alloc rT1 = ar.pfs, 2, 0, 2, 0
//// Check if we are tracing context swaps//
mov out0 = ATTEMPTED_SWITCH_FROM_DPC add rpT3 = @gprel(PPerfGlobalGroupMask), gp ;;
ld8.nta rpT3 = [rpT3] // get value of PperfGlobalGroupMask mov out1 = s2 // assign out1 to new ethread pointer add rT1 = PbDpcRoutineActive, s0 ;;
ld4 rT1 = [rT1] // Get the DPC active flag. add rpT2 = PERF_CONTEXTSWAP_OFFSET, rpT3 cmp.ne pt3 = zero, rpT3 // if it's non-zero, then trace on ;;
sync.i // Start sync.i well before the srlz.i(pt3) ld4.nta rpT2 = [rpT2] cmp.ne pt0 = zero, rT1 ;;
(pt3) and rpT2 = PERF_CONTEXTSWAP_FLAG, rpT2(pt0) br.call.spnt brp = KeBugCheck // Bugcheck if DPC active. ;;
(pt3) cmp.ne.unc pt4 = zero, rpT2 mov out0 = s1 // assign out0 to old ethread pointer(pt4) br.call.spnt brp = WmiTraceContextSwap // optimize for no tracing case ;;
//// Get address of old and new process objects.//
add rpT2 = ThApcState+AsProcess,s2 // -> new thread AsProcess add rpT1 = ThApcState+AsProcess,s1 // -> old thread AsProcess ;;
LDPTR (rOldproc, rpT1) // old process LDPTR (rNewproc, rpT2) // new process
#if !defined(NT_UP)
//// In MP system,// should a thread address is recycled and the thread is migrated to a// processor that holds the stale values in the high fp register set,// set KiPcr->HighFpOwner to zero (i.e. when pt4 is set to TRUE)//
add rpT1 = ThNumber, s2 movl rpT2 = KiPcr+PcHighFpOwner ;;
ld1 rT1 = [rpT1] ld8 rT2 = [rpT2], PcNumber-PcHighFpOwner ;;
ld1 rT3 = [rpT2], PcHighFpOwner-PcNumber cmp.eq pt3 = rT2, s2 ;;
(pt3) cmp.ne.unc pt4 = rT1, rT3 ;;
(pt4) st8 [rpT2] = zero
#endif // !defined(NT_UP)
;;
flushrs FAST_DISABLE_INTERRUPTS ;;
//// Thierry - 03/29/2000// It should be noticed that the performance analysis for SwapContext// was done with INTERRUPTION_LOGGING defined as 1.//
#if DBG // interruption logging is enabled in checked
#define INTERRUPTION_LOGGING 1
#endif // DBG
#if defined(INTERRUPTION_LOGGING)
// For Conditional Interrupt Logging#define ContextSwitchBit 63
.global KiVectorLogMask
mov rT3 = gp ;;
movl gp = _gp ;;
add rpT1 = @gprel(KiVectorLogMask), gp ;;
ld8 rT1 = [rpT1] mov gp = rT3 ;;
tbit.z pt4 = rT1, ContextSwitchBit (pt4) br.cond.sptk EndOfLogging0
movl rpT1 = KiPcr+PcInterruptionCount mov rT3 = MAX_NUMBER_OF_IHISTORY_RECORDS - 1 cmp.ne pDiff,pSame=rOldproc,rNewproc ;;
(pDiff) mov rT1 = 0x91 // process switch ld4.nt1 rT2 = [rpT1] // get current count ;;
(pSame) mov rT1 = 0x90 // thread switch add rpT3 = 1, rT2 // incr count and rT2 = rT3, rT2 // index of current entry add rpT2 = 0x1000-PcInterruptionCount, rpT1 // base of history ;;
st4.nta [rpT1] = rpT3 // save count shl rT2 = rT2, 5 // offset of current entry ;;
add rpT2 = rpT2, rT2 // address of current entry ;;
st8 [rpT2] = rT1, 8 // save switch type ;;
st8 [rpT2] = s2, 8 // save new thread pointer ;;
st8 [rpT2] = s1, 8 // save old thread ;;
st8 [rpT2] = sp // save old sp ;;
// For Conditional Interrupt LoggingEndOfLogging0:
#endif // INTERRUPTION_LOGGING
mov ar.rsc = r0 // put RSE in lazy mode add rpT1 = ThInitialStack, s2 add rpT2 = ThKernelStack, s1 ;;
//// Store the kernel stack pointer in the previous thread object,// load the new kernel stack pointer from the new thread object,// switch backing store pointers, select new process id and swap // to the new process.//
ld8.nta rNewIKS = [rpT1], ThKernelStack-ThInitialStack st8.nta [rpT2] = sp // save current sp ;;
ld8.nta sp = [rpT1], ThStackLimit-ThKernelStack movl rpT2 = KiPcr + PcInitialStack ;;
alloc rT1 = 0,0,0,0 // make current frame 0 size ld8.nta rNewKSL = [rpT1], ThInitialBStore-ThStackLimit ;;
loadrs // invalidate RSE and ALAT ld8.nta rT1 = [rpT1], ThBStoreLimit-ThInitialBStore ;;
ld8.nta rT2 = [rpT1], ThTeb-ThBStoreLimit st8 [rpT2] = rNewIKS, PcStackLimit-PcInitialStack add rpT3 = SwBsp+STACK_SCRATCH_AREA, sp ;;
// get debugger active state
ld8 rNewBSP = [rpT3], SwRnat-SwBsp st8 [rpT2] = rNewKSL, PcInitialBStore-PcStackLimit ;;
ld8 rNewRNAT = [rpT3] st8 [rpT2] = rT1, PcBStoreLimit-PcInitialBStore ;;
st8 [rpT2] = rT2, PcCurrentThread-PcBStoreLimit // load new teb ld8 teb = [rpT1], ThSpecialApcDisable-ThTeb // set new debugger active state invala
//// Setup PCR intial kernel BSP and BSTORE limit//
mov ar.bspstore = rNewBSP // load new bspstore cmp.ne pDiff,pSame=rOldproc,rNewproc // if ne, switch process ;;
ld2 rT2 = [rpT1], ThApcState+AsKernelApcPending-ThSpecialApcDisable // Get APC disable mov ar.rnat = rNewRNAT // load new RNATs ;;
mov ar.rsc = RSC_KERNEL // enable RSE cmp.eq pt0 = zero, rT2 // APC enabled if zero ;;
//// If the new process is not the same as the old process, then swap the// address space to the new process.//// N.B. The context swap lock cannot be dropped until all references to the// old process address space are complete. This includes any possible// TB Misses that could occur referencing the new address space while// still executing in the old address space.//// N.B. The process address space swap is executed with interrupts disabled.//
alloc rT1 = 0,4,2,0 add loc1 = PcApcInterrupt-PcCurrentThread, rpT2 mov loc0 = 0 // Initialize ApcPending to zero ;;
mov kteb = teb // update kernel TEB st1 [loc1] = r0 // Zero the old apc interrupt. STPTR (rpT2, s2) FAST_ENABLE_INTERRUPTS(pt0) ld1.acq loc0 = [rpT1] // load the ApcPending flag if enabled ;;
#ifdef _CAPKERN
(pDiff) movl out0 = @fptr(SwapContext)(pDiff) movl out1 = @fptr(KiSwapProcess)(pDiff) br.call.sptk.few b0=_CAP_Start_Profiling2;;
#endif
mov out0 = rNewproc // set address of new process mov out1 = rOldproc // set address of old process(pDiff) br.call.sptk brp = KxSwapProcess // call swap address space(NewProc, OldProc) ;;
#if !defined(NT_UP)
add out0 = ThSwapBusy, s1 // Get address of old thread busy flag cmp.ne pt0 = 0, loc0 ;;
#if DBG
ld1 rT2 = [out0] ;;
cmp.eq pt1 = rT2, r0 ;;
(pt1) break.i BREAKPOINT_STOP#endif
st1.rel [out0] = zero // clear swap busy flag in old thread ;;
#else
cmp.ne pt0 = 0, loc0 ;;
#endif // !defined(NT_UP)
#ifdef _CAPKERN
(pDiff) movl out0 = @fptr(SwapContext)(pDiff) br.call.sptk.few b0=_CAP_End_Profiling2;;
(pDiff) cmp.ne pt0 = 0, loc0 ;;
#endif
//// In new address space, if changed.//
(pt0) st1 [loc1] = loc0 // request APC interrupt. add rpT1 = PbContextSwitches, s0 add rpT2 = ThContextSwitches, s2 ;;
//// If the new thread has a kernel mode APC pending, then request an APC// interrupt.//
ld4 loc1 = [rpT1] ld4 loc2 = [rpT2] ;;
//// Increment context switch counters//
cmp.ne pUsTh, pKrTh = zero, teb add loc1 = loc1, zero, 1 add loc2 = loc2, zero, 1 ;;
st4 [rpT1] = loc1 // increment # of context switches
st4 [rpT2] = loc2 // increment # of context switches
add rpT1 = SwFPSR+STACK_SCRATCH_AREA, sp add rpT2 = SwPFS+STACK_SCRATCH_AREA, sp ;;
ld8 loc1 = [rpT1], SwRp-SwFPSR // restore brp and pfs ld8 loc2 = [rpT2], SwPreds-SwPFS ;;
ld8 rT3 = [rpT1] ld8 rT2 = [rpT2]
mov v0 = loc0 // set v0 = apc pending(pUsTh) br.call.spnt brp = SwapContextIA32Restore ;;
//// Note: at this point s0 = Prcb, s1 = previous thread, s2 = current thread//
mov ar.fpsr = loc1 mov ar.pfs = loc2 mov brp = rT3
mov pr = rT2 // Restore preserved preds
;;
srlz.i
br.ret.sptk brp
NESTED_EXIT(SwapContext)
//++//--------------------------------------------------------------------// Routine://// SwapContextIA32Save//// Routine Description://// This function saves the IA32 context on the kernel stack. // Called from SwapContext.//// Arguments://// rOldSbase : old thread kstack base.//// Return value://// None.//// Note://// SwapContext registers context.////--------------------------------------------------------------------//-- LEAF_ENTRY(SwapContextIA32Save)
mov rAr1 = ar21 // IA32 FP control register FCR ;;
mov rAr2 = ar24 // IA32 EFLAG register ;;
mov rAr3 = ar28 ;;
mov rAr4 = ar26 ;;
// // we may skip saving ar27 because it cannot be modified by user code // mov rT1 = ar30 ;;
mov rT3 = ar29 ;;
// these are separated out due to cache miss potential add rpT1 = -ThreadStateSaveAreaLength+TsAppRegisters+TsAr21, rOldSbase add rpT2 = -ThreadStateSaveAreaLength+TsAppRegisters+TsAr24, rOldSbase ;;
st8 [rpT1] = rAr1, TsAr28-TsAr21 st8 [rpT2] = rAr2, TsAr26-TsAr24 ;;
st8 [rpT1] = rAr3, TsAr29-TsAr28 st8 [rpT2] = rAr4, TsAr30-TsAr26 ;;
st8 [rpT2] = rT1 st8 [rpT1] = rT3
br.ret.sptk.few.clr brp LEAF_EXIT(SwapContextIA32Save)
//++//--------------------------------------------------------------------// Routine://// SwapContextIA32Restore//// Routine Description://// This function restores the IA32 registers context.// Called from SwapContext.//// Arguments://// s2 - Address of next thread object.//// Return value://// None.//// Note://// SwapContext registers context.////--------------------------------------------------------------------//-- LEAF_ENTRY(SwapContextIA32Restore)
add rpT1 = ThStackBase, s2 ;;
ld8.nta rpT1 = [rpT1] ;;
add rpT2 = -ThreadStateSaveAreaLength+TsAppRegisters+TsAr21, rpT1 add rpT3 = -ThreadStateSaveAreaLength+TsAppRegisters+TsAr24, rpT1 ;;
ld8.nta rAr1 = [rpT2], TsAr27-TsAr21 ld8.nta rAr2 = [rpT3], TsAr26-TsAr24 ;;
ld8.nta rAr4 = [rpT3], TsAr28-TsAr26 ;;
mov ar21 = rAr1 mov ar24 = rAr2
mov ar26 = rAr4
ld8.nta rAr1 = [rpT2], TsAr29-TsAr27 ld8.nta rAr2 = [rpT3], TsAr30-TsAr28 ;;
ld8.nta rAr3 = [rpT2] ld8.nta rAr4 = [rpT3] ;;
mov ar27 = rAr1 mov ar28 = rAr2
mov ar29 = rAr3 mov ar30 = rAr4
br.ret.sptk.few.clr brp LEAF_EXIT(SwapContextIA32Restore)
SBTTL("Swap Process")//++//--------------------------------------------------------------------//// VOID// KiSwapProcess (// IN PKPROCESS NewProcess,// IN PKPROCESS OldProcess// )//// Routine Description://// This function swaps the address space from one process to another by// assigning a new region id, if necessary, and loading the fixed entry// in the TB that maps the process page directory page. This routine follows// the PowerPC design for handling RID wrap.//// On entry/exit://// Interrupt enabled.//// Arguments://// NewProcess (a0) - Supplies a pointer to a control object of type process// which represents the new process that is switched to (32-bit address).//// OldProcess (a1) - Supplies a pointer to a control object of type process// which represents the old process that is switched from (32-bit address).//// Return Value://// None.////--------------------------------------------------------------------//-- NESTED_ENTRY(KiSwapProcess) NESTED_SETUP(2,3,3,0)
PROLOGUE_END
//// Register aliases//
rNewProc = a0 rOldProc = a1
rpCSLock = loc2
rpT1 = t0 rpT2 = t1 rProcSet = t2 rNewActive= t3 rOldActive= t4 rMasterSeq= t5 rNewSeq = t6 rOldPsrL = t7 rVa = t8 rPDE0 = t9 // PDE for page directory page 0 rVa2 = t10 rSessionBase = t11 rSessionInfo = t12 rT1 = t13 rT2 = t14
//
#if !defined(NT_UP)
movl rpT1 = KiPcr+PcPrcb ;;
ld8 rpT1 = [rpT1] ;;
br.sptk Ksp_Continue#endif // !defined(NT_UP)
;;
ALTERNATE_ENTRY(KxSwapProcess) NESTED_SETUP(2,3,3,0)
PROLOGUE_END//// Clear the processor set member number in the old process and set the// processor member number in the new process.//
Ksp_Continue:
#if !defined(NT_UP)
add rpT2 = PrActiveProcessors, rOldProc // -> old active processor set movl rpT1 = KiPcr + PcSetMember // -> processor set member ;;
ld8 rProcSet= [rpT1] // rProcSet = processor set member add rpT1 = PrActiveProcessors, rNewProc // -> new active processor set ;;
ksp_retry1:
ld8 rT1 = [rpT1] // rT1 = Old processor mask ;;
mov ar.ccv = rT1 // Load compare register or rNewActive = rT1,rProcSet // set processor member in new set ;;
cmpxchg8.rel out0 = [rpT1], rNewActive, ar.ccv // Attempt to set the bit ;;
cmp.ne pt0, pt1 = out0, rT1 // Did the swap suceed?(pt0) br.spnt.few ksp_retry1
ksp_retry2:
ld8 rT1 = [rpT2] // rT1 = Old processor mask ;;
mov ar.ccv = rT1 // Load compare register xor rOldActive = rT1,rProcSet // clear processor member in old set ;;
cmpxchg8.rel out0 = [rpT2], rOldActive, ar.ccv // Attempt to clear the bit ;;
cmp.ne pt0, pt1 = out0, rT1 // Did the swap suceed?(pt0) br.spnt.few ksp_retry2
#endif // !defined(NT_UP)
//// If the process sequence number matches the system sequence number, then// use the process RID. Otherwise, allocate a new process RID.//#ifdef _CAPKERN
;;
CAPSTART(KiSwapProcess,KiSyncNewRegionId)#endif
add rT2 = PrSessionMapInfo, rNewProc add out0 = PrProcessRegion, rNewProc ;;
ld8 out1 = [rT2] mov out2 = zero // We don't old the swap lock br.call.sptk brp = KiSyncNewRegionId CAPEND(KiSwapProcess) ;;
//// Switch address space to new process// v0 = rRid = new process rid//
fwb // hint to flush write buffers
FAST_DISABLE_INTERRUPTS
add rpT1 = PrDirectoryTableBase, rNewProc movl rVa = KiPcr+PcPdeUtbase add rpT2 = PrSessionParentBase, rNewProc movl rVa2 = KiPcr+PcPdeStbase ;;
ld8.nta rPDE0 = [rpT1] // rPDE0 = Page directory page 0 ld8.nta rSessionBase = [rpT2] ld8.nta rVa = [rVa] ld8.nta rVa2 = [rVa2] ;;
//// To access IFA, ITDR registers, PSR.ic bit must be 0. Otherwise,// it causes an illegal operation fault. While PSR.ic=0, any// interruption can not be afforded. Make sure there will be no// TLB miss and no interrupt coming in during this period.//
rsm 1 << PSR_IC // PSR.ic=0 ;;
srlz.d // must serialize mov rT1 = PAGE_SHIFT << IDTR_PS // load page size field for IDTR ;;
mov cr.itir = rT1 // set up IDTR for dirbase ptr.d rVa, rT1 // remove DTR for user space ;;
mov cr.ifa = rVa // set up IFA for dirbase vaddr mov rT2 = DTR_UTBASE_INDEX ;;
itr.d dtr[rT2] = rPDE0 // insert PDE0 to DTR ;;
ptr.d rVa2, rT1 // remove DTR for session ;; // to avoid a overlapping error
mov cr.ifa = rVa2 mov rT2 = DTR_STBASE_INDEX ;;
itr.d dtr[rT2] = rSessionBase // insert the root for session space ;;
ssm 1 << PSR_IC // PSR.ic=1 ;;
srlz.i // must I serialize
#if DBG
mov t0 = PbProcessorState+KpsSpecialRegisters+KsTrD0+(8*DTR_UTBASE_INDEX) movl t3 = KiPcr + PcPrcb ;;
ld8 t3 = [t3] mov t1 = PbProcessorState+KpsSpecialRegisters+KsTrD0+(8*DTR_STBASE_INDEX) ;;
add t0 = t3, t0 add t1 = t3, t1 ;;
st8 [t0] = rPDE0 st8 [t1] = rSessionBase ;;
#endif
FAST_ENABLE_INTERRUPTS
// // Now make sure branch history is enabled for non wow processes // and disabled for wow processes //
.global KiHistoryBufferLogMask
add t1 = @gprel(KiHistoryBufferLogMask), gp ;;
ld8 t1 = [t1] ;;
cmp.eq pt0 = t1, r0(pt0) br.cond.sptk SkipBranchHistory
mov t1 = 3 ;;
mov t2 = cpuid[t1] add t3 = PrWow64Process, rNewProc ;;
extr.u t2 = t2, 24, 8 ld4 t4 = [t3];
;;
cmp.ne pt1 = 7, t2 ;;
mov t1 = 675(pt1) br.dpnt SkipBranchHistory ;;
mov t2 = msr[t1] cmp.eq pt1,pt2 = zero, t4 // Wow64 is non-zero ;;
(pt1) mov t3 = 2 // Enable the HB for ia64 procs(pt2) mov t3 = 256 // Disable the HB for wow64 procs ;;
dep t2 = t3, t2, 0, 9 // Disable the HB for wow64 procs ;;
mov msr[t1] = t2;
;;
SkipBranchHistory:
#if !defined(NT_UP)
movl rpT1 = KiPcr+PcPrcb ;;
ld8 rpT1 = [rpT1] ;;
#endif // !defined(NT_UP)
NESTED_RETURN NESTED_EXIT(KiSwapProcess) SBTTL("Dispatch Interrupt")//++//--------------------------------------------------------------------// Routine://// KiDispatchInterrupt//// Routine Description://// This routine is entered as the result of a software interrupt generated// at DISPATCH_LEVEL. Its function is to process the Deferred Procedure Call// (DPC) list, and then perform a context switch if a new thread has been// selected for execution on the processor.//// This routine is entered at IRQL DISPATCH_LEVEL with the dispatcher// database unlocked. When a return to the caller finally occurs, the// IRQL remains at DISPATCH_LEVEL, and the dispatcher database is still// unlocked.//// N.B. On entry to this routine the volatile states (excluding high// floating point register set) have been saved.//// On entry://// sp - points to stack scratch area.//// Arguments://// None//// Return Value://// None.//--------------------------------------------------------------------//-- NESTED_ENTRY(KiDispatchInterrupt) NESTED_SETUP(0, 4, 2, 0) PROLOGUE_END
//// Register aliases//
rPrcb = loc2 rKerGP = loc3 rpT1 = t0 rpT2 = t1 rT1 = t2 rT2 = t3 rpDPLock = t4 // pointer to dispatcher lock
pNext = pt2 // next thread not null pNull = pt3 // no thread available pQEnd = pt6 // quantum end request pending pNoQEnd = pt7 // no quantum end request pending
//// Increment the dispatch interrupt count//
mov rKerGP = gp // save gp movl rPrcb = KiPcr + PcPrcb ;;
LDPTR (rPrcb, rPrcb) // rPrcb -> Prcb ;;
add rpT1 = PbDispatchInterruptCount, rPrcb ;;
ld4 rT1 = [rpT1] ;;
add rT1 = rT1, zero, 1 ;;
st4 [rpT1] = rT1
//// Process the deferred procedure call list.//
FAST_ENABLE_INTERRUPTS ;;
srlz.d
//// **** TBD ***** No stack switch as in alpha, mips...// Save current initial stack address and set new initial stack address.//
FAST_DISABLE_INTERRUPTS CAPSTART(KiDispatchInterrupt,KiRetireDpcList) mov out0 = rPrcb br.call.sptk brp = KiRetireDpcList CAPEND(KiDispatchInterrupt) ;;
//// Check to determine if quantum end has occured.//// N.B. If a new thread is selected as a result of processing a quantum// end request, then the new thread is returned with the dispatcher// database locked. Otherwise, NULL is returned with the dispatcher// database unlocked.//
FAST_ENABLE_INTERRUPTS add rpT1 = PbQuantumEnd, rPrcb add rpT2 = PbNextThread, rPrcb ;;
ld4 rT1 = [rpT1] // get quantum end indicator ;;
cmp4.ne pQEnd, pNoQEnd = rT1, zero // if zero, no quantum end reqs mov gp = rKerGP // restore gp ;;
(pQEnd) st4 [rpT1] = zero // clear quantum end indicator(pNoQEnd)ld8 rT1 = [rpT2] // rT1 = address of next thread object(pQEnd) br.cond.dpnt Kdi_QuantumEnd ;;
//// If no quantum end requests:// Check to determine if a new thread has been selected for execution on// this processor.//
cmp.eq pNull, pNext = rT1, zero // pNull => no thread selected mov ar.pfs = savedpfs // Prepare for return mov brp = savedbrp // or tail call ;;
(pNext) br.dpnt.many KxDispatchInterrupt // Jump to KxDispathInterrupt(pNull) br.ret.dptk.many.clr brp // Return to the caller
Kdi_QuantumEnd: CAPSTART(KiDispatchInterrupt,KiQuantumEnd) br.call.spnt brp = KiQuantumEnd // call KiQuantumEnd (C code) CAPEND(KiDispatchInterrupt) ;;
NESTED_RETURN NESTED_EXIT(KiDispatchInterrupt)
NESTED_ENTRY(KxDispatchInterrupt) PROLOGUE_BEGIN .regstk 0, 4, 2, 0 alloc t16 = ar.pfs, 0, 4, 2, 0 .save rp, loc0 mov loc0 = brp .fframe SwitchFrameLength add sp = -SwitchFrameLength, sp ;;
.save ar.unat, loc1 mov loc1 = ar.unat add t0 = ExFltS19+SwExFrame+STACK_SCRATCH_AREA, sp add t1 = ExFltS18+SwExFrame+STACK_SCRATCH_AREA, sp ;;
.save.gf 0x0, 0xC0000 stf.spill [t0] = fs19, ExFltS17-ExFltS19 stf.spill [t1] = fs18, ExFltS16-ExFltS18 ;;
.save.gf 0x0, 0x30000 stf.spill [t0] = fs17, ExFltS15-ExFltS17 stf.spill [t1] = fs16, ExFltS14-ExFltS16 mov t10 = bs4 ;;
.save.gf 0x0, 0xC000 stf.spill [t0] = fs15, ExFltS13-ExFltS15 stf.spill [t1] = fs14, ExFltS12-ExFltS14 mov t11 = bs3 ;;
.save.gf 0x0, 0x3000 stf.spill [t0] = fs13, ExFltS11-ExFltS13 stf.spill [t1] = fs12, ExFltS10-ExFltS12 mov t12 = bs2 ;;
.save.gf 0x0, 0xC00 stf.spill [t0] = fs11, ExFltS9-ExFltS11 stf.spill [t1] = fs10, ExFltS8-ExFltS10 mov t13 = bs1 ;;
.save.gf 0x0, 0x300 stf.spill [t0] = fs9, ExFltS7-ExFltS9 stf.spill [t1] = fs8, ExFltS6-ExFltS8 mov t14 = bs0 ;;
.save.gf 0x0, 0xC0 stf.spill [t0] = fs7, ExFltS5-ExFltS7 stf.spill [t1] = fs6, ExFltS4-ExFltS6 mov t15 = ar.lc ;;
.save.gf 0x0, 0x30 stf.spill [t0] = fs5, ExFltS3-ExFltS5 stf.spill [t1] = fs4, ExFltS2-ExFltS4 movl rPrcb = KiPcr + PcPrcb ;;
.save.f 0xC stf.spill [t0] = fs3, ExFltS1-ExFltS3 // save fs3 stf.spill [t1] = fs2, ExFltS0-ExFltS2 // save fs2 ;;
.save.f 0x3 stf.spill [t0] = fs1, ExBrS4-ExFltS1 // save fs1 stf.spill [t1] = fs0, ExBrS3-ExFltS0 // save fs0 ;;
.save.b 0x18 st8 [t0] = t10, ExBrS2-ExBrS4 // save bs4 st8 [t1] = t11, ExBrS1-ExBrS3 // save bs3 ;;
.save.b 0x6 st8 [t0] = t12, ExBrS0-ExBrS2 // save bs2 st8 [t1] = t13, ExIntS2-ExBrS1 // save bs1 ;;
.save.b 0x1 st8 [t0] = t14, ExIntS3-ExBrS0 // save bs0 ld8 rPrcb = [rPrcb] ;;
.save.gf 0xC, 0x0 .mem.offset 0,0 st8.spill [t0] = s3, ExIntS1-ExIntS3 // save s3 .mem.offset 8,0 st8.spill [t1] = s2, ExIntS0-ExIntS2 // save s2 ;;
.save.gf 0x3, 0x0 .mem.offset 0,0 st8.spill [t0] = s1, ExApLC-ExIntS1 // save s1 .mem.offset 8,0 st8.spill [t1] = s0, ExApEC-ExIntS0 // save s0 ;;
.savepsp ar.pfs, ExceptionFrameLength-ExApEC-STACK_SCRATCH_AREA st8 [t1] = t16, ExIntNats-ExApEC mov t4 = ar.unat // captured Nats of s0-s3 ;;
.savepsp ar.lc, ExceptionFrameLength-ExApLC-STACK_SCRATCH_AREA st8 [t0] = t15 .savepsp @priunat, ExceptionFrameLength-ExIntNats-STACK_SCRATCH_AREA st8 [t1] = t4 // save Nats of s0-s3
PROLOGUE_END
mov rT1 = SYNCH_LEVEL ;;
SET_IRQL (rT1)
movl rpT1 = KiPcr + PcCurrentThread mov s0 = rPrcb // setup call ;;
LDPTR (s1, rpT1) // current thread object ;;
add rpT1 = ThSwapBusy, s1 mov rT1 = 1 ;;
#if DBG
ld1 rT2 = [rpT1] ;;
cmp.ne pt0 = rT2, r0 ;;
(pt0) break.i BREAKPOINT_STOP#endif
st1.rel [rpT1] = rT1 // Set swap busy flag add out0 = PbPrcbLock, s0 // Get address of prcb lock br.call.sptk brp = KiAcquireSpinLock // Acquire PRCB lock ;;
//// Reread address of next thread object since it is possible for it to// change in a multiprocessor system.//
add rpT2 = PbNextThread, s0 // -> next thread add rpT1 = PbCurrentThread, s0 ;;
LDPTR (s2, rpT2) // next thread object mov rT1 = WrDispatchInt // wait reason is Dispatch Int. ;;
//// Reready current thread for execution and swap context to the selected// thread.//// Note: Lock the old thread so no idle processor// can switch to this processor before it is removed from the current// processor.//
STPTR (rpT2, zero) // clear addr of next thread add out0 = ThWaitReason, s1 // set wait reason ;;
st1 [out0] = rT1 add out1 = ThState, s2 mov out0 = Running ;;
STPTR (rpT1, s2) // set addr of current thread st1 [out1] = out0 // Set thread state to running
mov out0 = s1 mov out1 = s0 br.call.sptk brp = KiQueueReadyThread ;;
Kdi_Swap:
CAPSTART(KiDispatchInterrupt,SwapContext) // reenabled, works as stub br.call.sptk brp = SwapContext // call SwapContext(Prcb, OldTh, NewTh) CAPEND(KiDispatchInterrupt) ;;
//// Restore saved registers, and return.//
add out0 = STACK_SCRATCH_AREA+SwExFrame, sp br.call.sptk brp = KiRestoreExceptionFrame ;;
Kdi_Exit:
add rpT1 = ExApEC+SwExFrame+STACK_SCRATCH_AREA, sp ;;
ld8 rT1 = [rpT1] mov brp = loc0 ;;
mov ar.unat = loc1 mov ar.pfs = rT1 .restore add sp = SwitchFrameLength, sp br.ret.sptk brp
NESTED_EXIT(KxDispatchInterrupt)
|
