//++
//
// Copyright (c) 1989-2000 Microsoft Corporation
//
// Component Name:
//
// NT / KE
//
// Module Name:
//
// ctxswap.s
//
// Abstract:
//
// This module implements the IA64 Process and Thread Context Swaps.
//
// Author:
//
// David N. Cutler (davec) 5-Mar-1989
//
// Environment:
//
// Kernel mode only
//
// Revision History:
//
// Bernard Lint Jul-12-1995
//
// Initial IA64 version
//
//--
#include "ksia64.h"
.file "ctxswap.s"
.text
//
// Globals imported:
//
.global KiReadySummary
.global KiIdleSummary
.global KiDispatcherReadyListHead
.global KeTickCount
.global KiMasterSequence
.global KiMasterRid
.global PPerfGlobalGroupMask
PublicFunction(KiDeliverApc)
PublicFunction(KiSaveExceptionFrame)
PublicFunction(KiRestoreExceptionFrame)
PublicFunction(KiActivateWaiterQueue)
PublicFunction(KiReadyThread)
PublicFunction(KeFlushEntireTb)
PublicFunction(KiQuantumEnd)
PublicFunction(KiSyncNewRegionId)
PublicFunction(KiCheckForSoftwareInterrupt)
PublicFunction(KiSaveHigherFPVolatileAtDispatchLevel)
PublicFunction(KeAcquireQueuedSpinLockAtDpcLevel)
PublicFunction(KeReleaseQueuedSpinLockFromDpcLevel)
PublicFunction(KeTryToAcquireQueuedSpinLockRaiseToSynch)
PublicFunction(WmiTraceContextSwap)
#if DBG
PublicFunction(KeBugCheckEx)
#endif // DBG
SBTTL("Unlock Dispatcher Database")
//++
//--------------------------------------------------------------------
//
// VOID
// KiUnlockDispatcherDatabase (
// IN KIRQL OldIrql
// )
//
// Routine Description:
//
// This routine is entered at synchronization level with the dispatcher
// database locked. Its function is to either unlock the dispatcher
// database and return or initiate a context switch if another thread
// has been selected for execution.
//
// N.B. A context switch CANNOT be initiated if the previous IRQL
// is greater than or equal to DISPATCH_LEVEL.
//
// N.B. This routine is carefully written to be a leaf function. If,
// however, a context swap should be performed, the routine is
// switched to a nested fucntion.
//
// Arguments:
//
// OldIrql (a0) - Supplies the IRQL when the dispatcher database
// lock was acquired (in low order byte, not zero extended).
//
// Return Value:
//
// None.
//
//--------------------------------------------------------------------
//--
NESTED_ENTRY(KiUnlockDispatcherDatabase)
NESTED_SETUP(1,3,1,0)
//
// Register aliases
//
rDPC = loc2 // DPC active flag
rpT1 = t1 // temp pointer
rpT2 = t2 // temp pointer
rpT3 = t3 // temp pointer
rT1 = t5 // temp regs
rT2 = t6
rPrcb = t8 // PRCB pointer
pNotNl = pt2 // true if next thread not NULL
pIRQGE = pt3 // true if DISPATCH_LEVEL <= old irql
pIRQLT = pt4 // true if DISPATCH_LEVEL > old irql
pDPC = pt5 // true if DPC active
pNoAPC = pt2 // do not dispatch APC
pAPC = pt9
PROLOGUE_END
//
// Check if a thread has been scheduled to execute on the current processor
//
movl rPrcb = KiPcr + PcPrcb
;;
LDPTR (rPrcb, rPrcb) // rPrcb -> PRCB
;;
add rpT1 = PbNextThread, rPrcb // -> next thread
add rpT2 = PbDpcRoutineActive,rPrcb // -> DPC active flag
;;
LDPTR (v0, rpT1) // v0 = next thread
;;
cmp.ne pNotNl = zero, v0 // pNotNl = next thread is 0
zxt1 a0 = a0 // isolate old IRQL
;;
(pNotNl) cmp.leu.unc pIRQGE, pIRQLT = DISPATCH_LEVEL, a0
mov rDPC = 1 // speculate that DPC is active
(pIRQLT) br.spnt KxUnlockDispatcherDatabase
;;
//
// Case 1:
// Next thread is NULL:
// Release dispatcher database lock, restore IRQL to its previous level
// and return
//
//
// Case 2:
// A new thread has been selected to run on the current processor, but
// the new IRQL is not below dispatch level. Release the dispatcher
// lock and restore IRQL. If the current processor is
// not executing a DPC, then request a dispatch interrupt on the current
// processor.
//
// At this point pNotNl = 1 if thread not NULL, 0 if NULL
//
(pIRQGE) ld4 rDPC = [rpT2] // rDPC.4 = DPC active flag
#if !defined(NT_UP)
add out0 = (LockQueueDispatcherLock * 16) + PbLockQueue, rPrcb
br.call.sptk brp = KeReleaseQueuedSpinLockFromDpcLevel
#endif // !defined(NT_UP)
;;
LOWER_IRQL(a0)
cmp4.eq pDPC = rDPC, zero // pDPC = request DPC intr
REQUEST_DISPATCH_INT(pDPC) // request DPC interrupt
NESTED_RETURN
NESTED_EXIT(KiUnlockDispatcherDatabase)
//
// N.B. This routine is carefully written as a nested function.
// Control only reaches this routine from above.
//
// rPrcb contains the address of PRCB
// v0 contains the next thread
//
NESTED_ENTRY(KxUnlockDispatcherDatabase)
PROLOGUE_BEGIN
.regstk 1, 2, 1, 0
alloc t16 = ar.pfs, 1, 2, 1, 0
.save rp, loc0
mov loc0 = brp
.fframe SwitchFrameLength
add sp = -SwitchFrameLength, sp
;;
.save ar.unat, loc1
mov loc1 = ar.unat
add t0 = ExFltS19+SwExFrame+STACK_SCRATCH_AREA, sp
add t1 = ExFltS18+SwExFrame+STACK_SCRATCH_AREA, sp
;;
.save.gf 0x0, 0xC0000
stf.spill [t0] = fs19, ExFltS17-ExFltS19
stf.spill [t1] = fs18, ExFltS16-ExFltS18
;;
.save.gf 0x0, 0x30000
stf.spill [t0] = fs17, ExFltS15-ExFltS17
stf.spill [t1] = fs16, ExFltS14-ExFltS16
mov t10 = bs4
;;
.save.gf 0x0, 0xC000
stf.spill [t0] = fs15, ExFltS13-ExFltS15
stf.spill [t1] = fs14, ExFltS12-ExFltS14
mov t11 = bs3
;;
.save.gf 0x0, 0x3000
stf.spill [t0] = fs13, ExFltS11-ExFltS13
stf.spill [t1] = fs12, ExFltS10-ExFltS12
mov t12 = bs2
;;
.save.gf 0x0, 0xC00
stf.spill [t0] = fs11, ExFltS9-ExFltS11
stf.spill [t1] = fs10, ExFltS8-ExFltS10
mov t13 = bs1
;;
.save.gf 0x0, 0x300
stf.spill [t0] = fs9, ExFltS7-ExFltS9
stf.spill [t1] = fs8, ExFltS6-ExFltS8
mov t14 = bs0
;;
.save.gf 0x0, 0xC0
stf.spill [t0] = fs7, ExFltS5-ExFltS7
stf.spill [t1] = fs6, ExFltS4-ExFltS6
mov t15 = ar.lc
;;
.save.gf 0x0, 0x30
stf.spill [t0] = fs5, ExFltS3-ExFltS5
stf.spill [t1] = fs4, ExFltS2-ExFltS4
;;
.save.f 0xC
stf.spill [t0] = fs3, ExFltS1-ExFltS3 // save fs3
stf.spill [t1] = fs2, ExFltS0-ExFltS2 // save fs2
;;
.save.f 0x3
stf.spill [t0] = fs1, ExBrS4-ExFltS1 // save fs1
stf.spill [t1] = fs0, ExBrS3-ExFltS0 // save fs0
;;
.save.b 0x18
st8 [t0] = t10, ExBrS2-ExBrS4 // save bs4
st8 [t1] = t11, ExBrS1-ExBrS3 // save bs3
;;
.save.b 0x6
st8 [t0] = t12, ExBrS0-ExBrS2 // save bs2
st8 [t1] = t13, ExIntS2-ExBrS1 // save bs1
;;
.save.b 0x1
st8 [t0] = t14, ExIntS3-ExBrS0 // save bs0
movl t12 = KiPcr + PcCurrentThread
;;
.save.gf 0xC, 0x0
.mem.offset 0,0
st8.spill [t0] = s3, ExIntS1-ExIntS3 // save s3
.mem.offset 8,0
st8.spill [t1] = s2, ExIntS0-ExIntS2 // save s2
;;
.save.gf 0x3, 0x0
.mem.offset 0,0
st8.spill [t0] = s1, ExApLC-ExIntS1 // save s1
.mem.offset 8,0
st8.spill [t1] = s0, ExApEC-ExIntS0 // save s0
;;
.savepsp ar.pfs, ExceptionFrameLength-ExApEC-STACK_SCRATCH_AREA
st8 [t1] = t16, ExIntNats-ExApEC
mov t4 = ar.unat // captured Nats of s0-s3
mov s0 = rPrcb
LDPTR (s1, t12) // current thread
;;
.savepsp ar.lc, ExceptionFrameLength-ExApLC-STACK_SCRATCH_AREA
st8 [t0] = t15
.savepsp @priunat, ExceptionFrameLength-ExIntNats-STACK_SCRATCH_AREA
st8 [t1] = t4 // save Nats of s0-s3
mov s2 = v0
PROLOGUE_END
add rpT2 = PbNextThread, s0 // -> next thread
add out0 = ThWaitIrql, s1 // -> previous IRQL
;;
STPTRINC (rpT2, zero,PbCurrentThread-PbNextThread) // clear NextThread
st1 [out0] = a0, ThIdleSwapBlock-ThWaitIrql // save old IRQL
mov rpT3 = 1
;;
//
// Reready current thread for execution and swap context to the selected
// thread.
//
// Note: Set IdleSwapBlock in the current thread so no idle processor
// can switch to this processor before it is removed from the current
// processor.
STPTR (rpT2, s2) // set current thread object
st1 [out0] = rpT3, -ThIdleSwapBlock// out0 -> previous thread
br.call.sptk brp = KiReadyThread
;;
br.call.sptk brp = SwapContext
;;
//
// Lower IRQL, deallocate exception/switch frame.
//
// N.B. SwapContext releases the dispatcher database lock.
//
// N.B. v0 contains the kernel APC pending state on return.
//
// N.B. s2 contains the address of the new thread on return.
//
add rpT2 = ThWaitIrql, s2 // -> ThWaitIrql
cmp.ne pAPC, pNoAPC = zero, v0
;;
ld1 a0 = [rpT2] // a0 = original wait IRQL
;;
(pAPC) cmp.ne pNoAPC = zero, a0 // APC pending and IRQL == 0
(pNoAPC) br.spnt Kudd_Exit
;;
.regstk 1, 2, 3, 0
alloc t16 = ar.pfs, 1, 2, 3, 0
mov rT2 = APC_LEVEL
;;
SET_IRQL(rT2)
mov out0 = KernelMode
mov out1 = zero
mov out2 = zero
br.call.sptk brp = KiDeliverApc
;;
//
// Lower IRQL to wait level, set return status, restore registers, and return.
//
Kudd_Exit:
LOWER_IRQL(a0) // a0 = new irql
add out0 = STACK_SCRATCH_AREA+SwExFrame, sp
br.call.sptk brp = KiRestoreExceptionFrame
;;
add rpT1 = ExApEC+SwExFrame+STACK_SCRATCH_AREA, sp
;;
ld8 rT1 = [rpT1]
mov brp = loc0
;;
mov ar.unat = loc1
nop.f 0
mov ar.pfs = rT1
.restore
add sp = SwitchFrameLength, sp
nop.i 0
br.ret.sptk brp
;;
NESTED_EXIT(KxUnlockDispatcherDatabase)
SBTTL("Swap Thread")
//++
//--------------------------------------------------------------------
//
// BOOLEAN
// KiSwapContext (
// IN PKTHREAD Thread
// )
//
// Routine Description:
//
// This routine saves the non-volatile registers, marshals the
// arguments for SwapContext and calls SwapContext to perform
// the actual thread switch.
//
// Arguments:
//
// Thread - Supplies the address of the new thread.
//
// Return Value:
//
// If a kernel APC is pending, then a value of TRUE is returned.
// Otherwise, FALSE is returned.
//
// Notes:
//
// GP valid on entry -- GP is not switched, just use kernel GP
//--------------------------------------------------------------------
//--
NESTED_ENTRY(KiSwapContext)
//
// Register aliases
//
pNoAPC = pt2 // do not dispatch APC
rpT1 = t0 // temp pointer
rpT2 = t1 // temp pointer
rT1 = t10 // temp regs
PROLOGUE_BEGIN
.regstk 1, 2, 1, 0
alloc t16 = ar.pfs, 1, 2, 1, 0
.save rp, loc0
mov loc0 = brp
.fframe SwitchFrameLength
add sp = -SwitchFrameLength, sp
;;
.save ar.unat, loc1
mov loc1 = ar.unat
add t0 = ExFltS19+SwExFrame+STACK_SCRATCH_AREA, sp
add t1 = ExFltS18+SwExFrame+STACK_SCRATCH_AREA, sp
;;
.save.gf 0x0, 0xC0000
stf.spill [t0] = fs19, ExFltS17-ExFltS19
stf.spill [t1] = fs18, ExFltS16-ExFltS18
;;
.save.gf 0x0, 0x30000
stf.spill [t0] = fs17, ExFltS15-ExFltS17
stf.spill [t1] = fs16, ExFltS14-ExFltS16
mov t10 = bs4
;;
.save.gf 0x0, 0xC000
stf.spill [t0] = fs15, ExFltS13-ExFltS15
stf.spill [t1] = fs14, ExFltS12-ExFltS14
mov t11 = bs3
;;
.save.gf 0x0, 0x3000
stf.spill [t0] = fs13, ExFltS11-ExFltS13
stf.spill [t1] = fs12, ExFltS10-ExFltS12
mov t12 = bs2
;;
.save.gf 0x0, 0xC00
stf.spill [t0] = fs11, ExFltS9-ExFltS11
stf.spill [t1] = fs10, ExFltS8-ExFltS10
mov t13 = bs1
;;
.save.gf 0x0, 0x300
stf.spill [t0] = fs9, ExFltS7-ExFltS9
stf.spill [t1] = fs8, ExFltS6-ExFltS8
mov t14 = bs0
;;
.save.gf 0x0, 0xC0
stf.spill [t0] = fs7, ExFltS5-ExFltS7
stf.spill [t1] = fs6, ExFltS4-ExFltS6
mov t15 = ar.lc
;;
.save.gf 0x0, 0x30
stf.spill [t0] = fs5, ExFltS3-ExFltS5
stf.spill [t1] = fs4, ExFltS2-ExFltS4
;;
.save.f 0xC
stf.spill [t0] = fs3, ExFltS1-ExFltS3 // save fs3
stf.spill [t1] = fs2, ExFltS0-ExFltS2 // save fs2
;;
.save.f 0x3
stf.spill [t0] = fs1, ExBrS4-ExFltS1 // save fs1
stf.spill [t1] = fs0, ExBrS3-ExFltS0 // save fs0
;;
.save.b 0x18
st8 [t0] = t10, ExBrS2-ExBrS4 // save bs4
st8 [t1] = t11, ExBrS1-ExBrS3 // save bs3
;;
.save.b 0x6
st8 [t0] = t12, ExBrS0-ExBrS2 // save bs2
st8 [t1] = t13, ExIntS2-ExBrS1 // save bs1
;;
.save.b 0x1
st8 [t0] = t14, ExIntS3-ExBrS0 // save bs0
;;
.save.gf 0xC, 0x0
.mem.offset 0,0
st8.spill [t0] = s3, ExIntS1-ExIntS3 // save s3
.mem.offset 8,0
st8.spill [t1] = s2, ExIntS0-ExIntS2 // save s2
;;
.save.gf 0x3, 0x0
.mem.offset 0,0
st8.spill [t0] = s1, ExApLC-ExIntS1 // save s1
.mem.offset 8,0
st8.spill [t1] = s0, ExApEC-ExIntS0 // save s0
;;
.savepsp ar.pfs, ExceptionFrameLength-ExApEC-STACK_SCRATCH_AREA
st8 [t1] = t16, ExIntNats-ExApEC
mov t4 = ar.unat // captured Nats of s0-s3
;;
.savepsp ar.lc, ExceptionFrameLength-ExApLC-STACK_SCRATCH_AREA
st8 [t0] = t15
.savepsp @priunat, ExceptionFrameLength-ExIntNats-STACK_SCRATCH_AREA
st8 [t1] = t4 // save Nats of s0-s3
PROLOGUE_END
//
// For the call to SwapContext-
//
// s0 // Prcb address
// s1 // old thread address
// s2 // new thread address
// pt0 = 1
//
mov s2 = a0 // s2 <- New Thread
movl rpT1 = KiPcr + PcPrcb
;;
LDPTRINC (s0, rpT1, PcCurrentThread-PcPrcb)// s0 <- Prcb
;;
LDPTR (s1, rpT1) // s1 <- Old Thread
add rpT2 = PbCurrentThread, s0
;;
//
// Swap context to the next thread.
//
STPTR (rpT2, a0) // Set new thread current
cmp.eq pt0 = zero, zero // indicate lock context swap
br.call.sptk brp = SwapContext // call SwapContext(prcb, OldTh, NewTh)
;;
//
// Deallocate exception/switch frame.
//
// N.B. SwapContext releases the dispatcher database lock.
//
// N.B. v0 contains the kernel APC pending state on return, ie, 0 if
// no APC pending, 1 if APC pending. v0 will be forced to 0 if
// the new IRQL doesn't allow APCs.
//
// N.B. KiRestoreExceptionFrame doesn't touch v0, t21 or t22.
//
add rpT2 = ThWaitIrql, s2 // -> ThWaitIrql
add rpT1 = ExApEC+SwExFrame+STACK_SCRATCH_AREA, sp
add out0 = STACK_SCRATCH_AREA+SwExFrame, sp
;;
ld1 t21 = [rpT2] // t21 = original wait IRQL
ld8 t22 = [rpT1] // t22 = PFS
br.call.sptk brp = KiRestoreExceptionFrame
;;
mov brp = loc0
cmp.ne pNoAPC = zero, t21 // no APC if IRQL != 0
;;
mov ar.unat = loc1
nop.f 0
mov ar.pfs = t22
.restore
add sp = SwitchFrameLength, sp
(pNoAPC) mov v0 = zero
br.ret.sptk brp
;;
NESTED_EXIT(KiSwapContext)
SBTTL("Swap Context to Next Thread")
//++
//--------------------------------------------------------------------
// Routine:
//
// SwapContext
//
// Routine Description:
//
// This routine is called to swap context from one thread to the next.
//
// Arguments:
//
// s0 - Address of Processor Control Block (PRCB).
// s1 - Address of previous thread object.
// s2 - Address of next thread object.
//
// Return value:
//
// v0 - Kernel APC pending flag
// s0 - Address of Processor Control Block (PRCB).
// s1 - Address of previous thread object.
// s2 - Address of current thread object.
//
// Note:
// Kernel GP is not saved and restored across context switch
//
// !!WARNING!! - Thierry. 03/01/2000.
// Be aware that this implementation is a result of performance analysis.
// Please consider this when you are making changes...
//
//--------------------------------------------------------------------
//--
NESTED_ENTRY(SwapContext)
//
// Register aliases
//
rT1 = t1 // temp
rT2 = t2 // temp
rT3 = t3 // temp
rNewproc = t4 // next process object
rOldproc = t5 // previous process object
rpThBSL = t6 // pointer to new thread backing store limit
rpT1 = t7 // temp pointer
rpT2 = t8 // temp pointer
rpT3 = t9 // temp pointer
rAr1 = t10
rAr2 = t11
rAr3 = t12
rAr4 = t13
rNewIKS = t14 // new initial kernel stack
rNewKSL = t15 // new kernel stack limit
rNewBSP = t16 // new thread BSP/BSPSTORE
rOldBSP = t16 // old thread BSP
rOldRNAT = t17 // old thread RNAT
rNewRNAT = t17 // new thread RNAT
rOldSbase = t18 // old thread kstack base
pUsTh = pt4 // is user thread?
pKrTh = pt5 // is user thread?
pSave = pt7 // is high fp set dirty?
pDiff = ps4 // if new and old process different
pSame = ps5 // if new and old process same
//
// Set new thread's state to running. Note this must be done
// under the dispatcher lock so that KiSetPriorityThread sees
// the correct state.
//
PROLOGUE_BEGIN
#if !defined(NT_UP)
alloc rT2 = ar.pfs, 0, 0, 4, 0
mov rT1 = brp // move from brp takes 2 cycles
add rpT3 = ThState, s2
;;
lfetch.excl [rpT3]
mov rAr1 = Running
add rpT2 = SwPFS+STACK_SCRATCH_AREA, sp
;;
add out0 = (LockQueueContextSwapLock * 16) + PbLockQueue, s0
.savesp ar.pfs, SwPFS+STACK_SCRATCH_AREA
st8.nta [rpT2] = rT2, SwRp-SwPFS // save pfs
;;
.savesp brp, SwRp+STACK_SCRATCH_AREA
st8.nta [rpT2] = rT1 // save return link
st1.nta [rpT3] = rAr1 // set thread state to Running
br.call.sptk brp = KeAcquireQueuedSpinLockAtDpcLevel
;;
//
// Release DispatcherLock.
//
add out0 = (LockQueueDispatcherLock * 16) + PbLockQueue, s0
br.call.sptk brp = KeReleaseQueuedSpinLockFromDpcLevel
;;
mov out0 = ar.fpsr // move from ar.fpsr takes 12 cycles
movl rpT1 = KiPcr+PcHighFpOwner // setup for prefetching
;;
{ .mmi
lfetch [rpT1]
cmp.ne pUsTh = zero, teb // test for ia32 save required
// must not have a nop.f for next 10 cycles--
// Using temporarely the explicit templating
// for the next cycles.
add out1 = ThStackBase, s1 // move early to start access for rOldSbase
{ .mmi
add rpT1 = SwFPSR+STACK_SCRATCH_AREA, sp
add rpT2 = SwPreds+STACK_SCRATCH_AREA, sp
nop.i 0x0
}
;;
{ .mmi
ld8.nta rOldSbase = [out1] // speculative start early for ia32 saves
lfetch.excl [rpT1]
add out2 = ThNumber, s2 // setup for prefetching
}
{ .mmi
mov.m ar.rsc = r0 // put RSE in lazy mode
mov rOldBSP = ar.bsp // move from ar.bsp takes 12 cycles
nop.i 0x0
}
;;
{ .mmi
lfetch [out2]
nop.m 0x0
mov rT1 = pr // move from pr takes 2 cycles
}
;;
{ .mmi
flushrs
mov rT3 = psr.um // move from psr.um takes 12 cycles
nop.i 0x0
}
;;
{ .mmi
lfetch.excl [rpT2]
mov.m rOldRNAT = ar.rnat // move from ar.rnat takes 5 cycles
add out2 = @gprel(PPerfGlobalGroupMask), gp
}
;;
{ .mli
lfetch [out2]
movl out3 = KiPcr + PcInterruptionCount // INTERRUPTION_LOGGING on or off, we are prefetching this line.
// If any real performance problem is detected, we will undef these lines.
}
;;
{ .mmi
lfetch [out3]
add rpT3 = SwRnat+STACK_SCRATCH_AREA, sp
}
;;
#else // NT_UP
alloc rT2 = ar.pfs, 0, 0, 4, 0
cmp.ne pUsTh = zero, teb // test for ia32 save required
;;
mov.m ar.rsc = r0 // put RSE in lazy mode
add out1 = ThStackBase, s1 // move early to start access for rOldSbase
mov out0 = ar.fpsr // move from ar.fpsr takes 12 cycles
// must not have a nop.f for next 10 cycles--
// Using temporarely the explicit templating
// for the next cycles.
;;
{ .mmi
ld8.nta rOldSbase = [out1] // speculative start early for ia32 saves
mov rOldBSP = ar.bsp // move from ar.bsp takes 12 cycles
add rpT1 = SwRp+STACK_SCRATCH_AREA, sp
}
;;
flushrs
mov rT3 = psr.um // move from psr.um takes 12 cycles
add rpT2 = SwPFS+STACK_SCRATCH_AREA, sp
;;
mov.m rOldRNAT = ar.rnat // move from ar.rnat takes 5 cycles
mov rT1 = brp // move from brp takes 2 cycles
add rpT3 = ThState, s2
;;
{ .mmi
mov rAr1 = Running
.savesp brp, SwRp+STACK_SCRATCH_AREA
st8.nta [rpT1] = rT1, SwFPSR-SwRp // save return link
nop.i 0x0
}
;;
{ .mii
st1.nta [rpT3] = rAr1 // set thread state to Running
mov rT1 = pr // move from pr takes 2 cycles
nop.i 0x0
}
;;
{ .mii
.savesp ar.pfs, SwPFS+STACK_SCRATCH_AREA
st8.nta [rpT2] = rT2, SwPreds-SwPFS // save pfs
add rpT3 = SwRnat+STACK_SCRATCH_AREA, sp
nop.i 0x0
}
;;
#endif // NT_UP
{ .mmi
st8.nta [rpT3] = rOldRNAT
nop.m 0x0
nop.i 0x0
}
st8 [rpT1] = out0, SwBsp-SwFPSR // save kernel FPSR
st8 [rpT2] = rT1 // save preserved predicates
;;
st8.nta [rpT1] = rOldBSP
add rpT3 = ThKernelBStore, s1
tbit.nz pSave = rT3, PSR_MFH // check mfh bit
(pUsTh) br.call.spnt brp = SwapContextIA32Save
;;
st8.nta [rpT3] = rOldBSP
(pSave) add out0 = -ThreadStateSaveAreaLength+TsHigherFPVolatile, rOldSbase
(pSave) br.call.spnt brp = KiSaveHigherFPVolatileAtDispatchLevel
;;
//
// Acquire the context swap lock so the address space of the old process
// cannot be deleted and then release the dispatcher database lock.
//
// N.B. This lock is used to protect the address space until the context
// switch has sufficiently progressed to the point where the address
// space is no longer needed. This lock is also acquired by the reaper
// thread before it finishes thread termination.
//
PROLOGUE_END
//
// ***** TBD ****** Save performance counters? (user vs. kernel)
//
//
// Accumlate the total time spent in a thread.
//
#if defined(PERF_DATA)
**** TBD **** MIPS code
addu a0,sp,ExFltF20 // compute address of result
move a1,zero // set address of optional frequency
jal KeQueryPerformanceCounter // query performance counter
lw t0,ExFltF20(sp) // get current cycle count
lw t1,ExFltF20 + 4(sp) //
lw t2,PbStartCount(s0) // get starting cycle count
lw t3,PbStartCount + 4(s0) //
sw t0,PbStartCount(s0) // set starting cycle count
sw t1,PbStartCount + 4(s0) //
lw t4,EtPerformanceCountLow(s1) // get accumulated cycle count
lw t5,EtPerformanceCountHigh(s1) //
subu t6,t0,t2 // subtract low parts
subu t7,t1,t3 // subtract high parts
sltu v0,t0,t2 // generate borrow from high part
subu t7,t7,v0 // subtract borrow
addu t6,t6,t4 // add low parts
addu t7,t7,t5 // add high parts
sltu v0,t6,t4 // generate carry into high part
addu t7,t7,v0 // add carry
sw t6,EtPerformanceCountLow(s1) // set accumulated cycle count
sw t7,EtPerformanceCountHigh(s1) //
#endif // defined(PERF_DATA)
//
// The following entry point is used to switch from the idle thread to
// another thread.
//
;;
ALTERNATE_ENTRY(SwapFromIdle)
alloc rT1 = ar.pfs, 2, 0, 2, 0
//
// Check if we are tracing context swaps
//
mov out0 = s1 // assign out0 to old ethread pointer
add rpT3 = @gprel(PPerfGlobalGroupMask), gp
;;
ld8.nta rpT3 = [rpT3] // get value of PperfGlobalGroupMask
mov out1 = s2 // assign out1 to new ethread pointer
;;
add rpT2 = PERF_CONTEXTSWAP_OFFSET, rpT3
cmp.ne pt3 = zero, rpT3 // if it's non-zero, then trace on
;;
(pt3) ld4.nta rpT2 = [rpT2]
;;
(pt3) and rpT2 = PERF_CONTEXTSWAP_FLAG, rpT2
;;
(pt3) cmp.ne.unc pt4 = zero, rpT2
(pt4) br.call.spnt brp = WmiTraceContextSwap // optimize for no tracing case
;;
//
// Get address of old and new process objects.
//
add rpT2 = ThApcState+AsProcess,s2 // -> new thread AsProcess
add rpT1 = ThApcState+AsProcess,s1 // -> old thread AsProcess
;;
LDPTR (rOldproc, rpT1) // old process
LDPTR (rNewproc, rpT2) // new process
#if !defined(NT_UP)
//
// In MP system,
// should a thread address is recycled and the thread is migrated to a
// processor that holds the stale values in the high fp register set,
// set KiPcr->HighFpOwner to zero (i.e. when pt4 is set to TRUE)
//
add rpT1 = ThNumber, s2
movl rpT2 = KiPcr+PcHighFpOwner
;;
ld1 rT1 = [rpT1]
ld8 rT2 = [rpT2], PcNumber-PcHighFpOwner
add out0 = ThIdleSwapBlock, s1
;;
ld1 rT3 = [rpT2], PcHighFpOwner-PcNumber
st1 [out0] = zero // clear OldThread->IdleSwapBlock
cmp.eq pt3 = rT2, s2
;;
(pt3) cmp.ne.unc pt4 = rT1, rT3
;;
(pt4) st8 [rpT2] = zero
#endif // !defined(NT_UP)
;;
flushrs
FAST_DISABLE_INTERRUPTS
;;
//
// Thierry - 03/29/2000
// It should be noticed that the performance analysis for SwapContext
// was done with INTERRUPTION_LOGGING defined as 1.
//
#define INTERRUPTION_LOGGING 1
#if defined(INTERRUPTION_LOGGING)
// For Conditional Interrupt Logging
#define ContextSwitchBit 63
.global KiVectorLogMask
mov rT3 = gp
;;
movl gp = _gp
;;
add rpT1 = @gprel(KiVectorLogMask), gp
;;
ld8 rT1 = [rpT1]
mov gp = rT3
;;
tbit.z pt4 = rT1, ContextSwitchBit
(pt4) br.cond.sptk EndOfLogging0
movl rpT1 = KiPcr+PcInterruptionCount
mov rT3 = MAX_NUMBER_OF_IHISTORY_RECORDS - 1
cmp.ne pDiff,pSame=rOldproc,rNewproc
;;
(pDiff) mov rT1 = 0x91 // process switch
ld4.nt1 rT2 = [rpT1] // get current count
;;
(pSame) mov rT1 = 0x90 // thread switch
add rpT3 = 1, rT2 // incr count
and rT2 = rT3, rT2 // index of current entry
add rpT2 = 0x1000-PcInterruptionCount, rpT1 // base of history
;;
st4.nta [rpT1] = rpT3 // save count
shl rT2 = rT2, 5 // offset of current entry
;;
add rpT2 = rpT2, rT2 // address of current entry
;;
st8 [rpT2] = rT1, 8 // save switch type
;;
st8 [rpT2] = s2, 8 // save new thread pointer
;;
st8 [rpT2] = s1, 8 // save old thread
;;
st8 [rpT2] = sp // save old sp
;;
// For Conditional Interrupt Logging
EndOfLogging0:
#endif // INTERRUPTION_LOGGING
mov ar.rsc = r0 // put RSE in lazy mode
add rpT1 = ThInitialStack, s2
add rpT2 = ThKernelStack, s1
;;
//
// Store the kernel stack pointer in the previous thread object,
// load the new kernel stack pointer from the new thread object,
// switch backing store pointers, select new process id and swap
// to the new process.
//
ld8.nta rNewIKS = [rpT1], ThKernelStack-ThInitialStack
st8.nta [rpT2] = sp // save current sp
;;
ld8.nta sp = [rpT1], ThStackLimit-ThKernelStack
movl rpT2 = KiPcr + PcInitialStack
;;
alloc rT1 = 0,0,0,0 // make current frame 0 size
ld8.nta rNewKSL = [rpT1], ThInitialBStore-ThStackLimit
;;
loadrs // invalidate RSE and ALAT
ld8.nta rT1 = [rpT1], ThBStoreLimit-ThInitialBStore
;;
ld8.nta rT2 = [rpT1], ThDebugActive-ThBStoreLimit
st8 [rpT2] = rNewIKS, PcStackLimit-PcInitialStack
;;
// get debugger active state
ld1.nta rT3 = [rpT1], ThTeb-ThDebugActive
st8 [rpT2] = rNewKSL, PcInitialBStore-PcStackLimit
add rpT3 = SwBsp+STACK_SCRATCH_AREA, sp
;;
ld8 rNewBSP = [rpT3], SwRnat-SwBsp
st8 [rpT2] = rT1, PcBStoreLimit-PcInitialBStore
;;
ld8 rNewRNAT = [rpT3]
st8 [rpT2] = rT2, PcDebugActive-PcBStoreLimit
;;
// load new teb
ld8 teb = [rpT1], ThApcState+AsKernelApcPending-ThTeb
// set new debugger active state
st1 [rpT2] = rT3, PcCurrentThread-PcDebugActive
invala
//
// Setup PCR intial kernel BSP and BSTORE limit
//
mov ar.bspstore = rNewBSP // load new bspstore
cmp.ne pDiff,pSame=rOldproc,rNewproc // if ne, switch process
;;
mov ar.rnat = rNewRNAT // load new RNATs
;;
mov ar.rsc = RSC_KERNEL // enable RSE
;;
//
// If the new process is not the same as the old process, then swap the
// address space to the new process.
//
// N.B. The context swap lock cannot be dropped until all references to the
// old process address space are complete. This includes any possible
// TB Misses that could occur referencing the new address space while
// still executing in the old address space.
//
// N.B. The process address space swap is executed with interrupts disabled.
//
alloc rT1 = 0,4,2,0
STPTR (rpT2, s2)
;;
mov kteb = teb // update kernel TEB
FAST_ENABLE_INTERRUPTS
ld1 loc0 = [rpT1] // load the ApcPending flag
#if !defined(NT_UP)
//
// Release the context swap lock
// N.B. ContextSwapLock is always released in KxSwapProcess, if called
//
add out0 = (LockQueueContextSwapLock * 16) + PbLockQueue, s0
add loc1 = PcApcInterrupt-PcCurrentThread, rpT2
(pSame) br.call.sptk brp = KeReleaseQueuedSpinLockFromDpcLevel
;;
#else // !defined(NT_UP)
add loc1 = PcApcInterrupt-PcCurrentThread, rpT2
;;
#endif // !defined(NT_UP)
mov out0 = rNewproc // set address of new process
mov out1 = rOldproc // set address of old process
(pDiff) br.call.sptk brp = KxSwapProcess // call swap address space(NewProc, OldProc)
;;
//
// In new address space, if changed.
//
st1 [loc1] = loc0 // request (or clear) APC pend.
add rpT1 = PbContextSwitches, s0
add rpT2 = ThContextSwitches, s2
;;
//
// If the new thread has a kernel mode APC pending, then request an APC
// interrupt.
//
ld4 loc1 = [rpT1]
ld4 loc2 = [rpT2]
;;
//
// Increment context switch counters
//
cmp.ne pUsTh, pKrTh = zero, teb
add loc1 = loc1, zero, 1
add loc2 = loc2, zero, 1
;;
st4 [rpT1] = loc1 // increment # of context switches
st4 [rpT2] = loc2 // increment # of context switches
add rpT1 = SwFPSR+STACK_SCRATCH_AREA, sp
add rpT2 = SwPFS+STACK_SCRATCH_AREA, sp
;;
ld8 loc1 = [rpT1], SwRp-SwFPSR // restore brp and pfs
ld8 loc2 = [rpT2], SwPreds-SwPFS
;;
ld8 rT3 = [rpT1]
ld8 rT2 = [rpT2]
mov v0 = loc0 // set v0 = apc pending
(pUsTh) br.call.spnt brp = SwapContextIA32Restore
;;
//
// Note: at this point s0 = Prcb, s1 = previous thread, s2 = current thread
//
mov ar.fpsr = loc1
mov ar.pfs = loc2
mov brp = rT3
mov pr = rT2 // Restore preserved preds
#if 0
//
// Thierry 03/22/2000:
//
// The following memory synchronization of the local processor
// I-cache and D-cache because of I-stream modifications is not
// required if the modifying code is written following the NT
// Core Team specifications:
// - [Allocate VA]
// - Modify the code
// - Call FlushIntructionCache()
// -> calls KiSweepIcache[Range]()
// - Execute the code.
//
// The removal of this instruction eliminates a "> 100 cycle" stall.
//
sync.i
#endif // 0
;;
srlz.i
br.ret.sptk brp
NESTED_EXIT(SwapContext)
//++
//--------------------------------------------------------------------
// Routine:
//
// SwapContextIA32Save
//
// Routine Description:
//
// This function saves the IA32 context on the kernel stack.
// Called from SwapContext.
//
// Arguments:
//
// rOldSbase : old thread kstack base.
//
// Return value:
//
// None.
//
// Note:
//
// SwapContext registers context.
//
//--------------------------------------------------------------------
//--
LEAF_ENTRY(SwapContextIA32Save)
mov rAr1 = ar21 // IA32 FP control register FCR
;;
mov rAr2 = ar24 // IA32 EFLAG register
;;
mov rAr3 = ar25
;;
mov rAr4 = ar26
;;
//
// we may skip saving ar27 because it cannot be modified by user code
//
mov rT1 = ar30
;;
mov rT2 = ar28
;;
mov rT3 = ar29
;;
// these are separated out due to cache miss potential
add rpT1 = -ThreadStateSaveAreaLength+TsAppRegisters+TsAr21, rOldSbase
add rpT2 = -ThreadStateSaveAreaLength+TsAppRegisters+TsAr24, rOldSbase
;;
st8 [rpT1] = rAr1, TsAr25-TsAr21
st8 [rpT2] = rAr2, TsAr26-TsAr24
;;
st8 [rpT1] = rAr3, TsAr29-TsAr25
st8 [rpT2] = rAr4, TsAr28-TsAr26
;;
st8 [rpT2] = rT2, TsAr30-TsAr28
;;
st8 [rpT2] = rT1
st8 [rpT1] = rT3
br.ret.sptk.few.clr brp
LEAF_EXIT(SwapContextIA32Save)
//++
//--------------------------------------------------------------------
// Routine:
//
// SwapContextIA32Restore
//
// Routine Description:
//
// This function restores the IA32 registers context.
// Called from SwapContext.
//
// Arguments:
//
// s2 - Address of next thread object.
//
// Return value:
//
// None.
//
// Note:
//
// SwapContext registers context.
//
//--------------------------------------------------------------------
//--
LEAF_ENTRY(SwapContextIA32Restore)
add rpT1 = ThStackBase, s2
;;
ld8.nta rpT1 = [rpT1]
;;
add rpT2 = -ThreadStateSaveAreaLength+TsAppRegisters+TsAr21, rpT1
add rpT3 = -ThreadStateSaveAreaLength+TsAppRegisters+TsAr24, rpT1
;;
ld8.nta rAr1 = [rpT2], TsAr25-TsAr21
ld8.nta rAr2 = [rpT3], TsAr26-TsAr24
;;
ld8.nta rAr3 = [rpT2], TsAr27-TsAr25
ld8.nta rAr4 = [rpT3], TsAr28-TsAr26
;;
mov ar21 = rAr1
mov ar24 = rAr2
mov ar25 = rAr3
mov ar26 = rAr4
ld8.nta rAr1 = [rpT2], TsAr29-TsAr27
ld8.nta rAr2 = [rpT3], TsAr30-TsAr28
;;
ld8.nta rAr3 = [rpT2]
ld8.nta rAr4 = [rpT3]
;;
mov ar27 = rAr1
mov ar28 = rAr2
mov ar29 = rAr3
mov ar30 = rAr4
br.ret.sptk.few.clr brp
LEAF_EXIT(SwapContextIA32Restore)
SBTTL("Swap Process")
//++
//--------------------------------------------------------------------
//
// VOID
// KiSwapProcess (
// IN PKPROCESS NewProcess,
// IN PKPROCESS OldProcess
// )
//
// Routine Description:
//
// This function swaps the address space from one process to another by
// assigning a new region id, if necessary, and loading the fixed entry
// in the TB that maps the process page directory page. This routine follows
// the PowerPC design for handling RID wrap.
//
// On entry/exit:
//
// Interrupt enabled.
//
// Arguments:
//
// NewProcess (a0) - Supplies a pointer to a control object of type process
// which represents the new process that is switched to (32-bit address).
//
// OldProcess (a1) - Supplies a pointer to a control object of type process
// which represents the old process that is switched from (32-bit address).
//
// Return Value:
//
// None.
//
//--------------------------------------------------------------------
//--
NESTED_ENTRY(KiSwapProcess)
NESTED_SETUP(2,3,3,0)
PROLOGUE_END
//
// Register aliases
//
rNewProc = a0
rOldProc = a1
rpCSLock = loc2
rpT1 = t0
rpT2 = t1
rProcSet = t2
rNewActive= t3
rOldActive= t4
rMasterSeq= t5
rNewSeq = t6
rOldPsrL = t7
rVa = t8
rPDE0 = t9 // PDE for page directory page 0
rVa2 = t10
rSessionBase = t11
rSessionInfo = t12
rT1 = t13
rT2 = t14
//
// KiSwapProcess must get the context swap lock
// KxSwapProcess is called from SwapContext with the lock held
//
#if !defined(NT_UP)
movl rpT1 = KiPcr+PcPrcb
;;
ld8 rpT1 = [rpT1]
;;
add out0 = (LockQueueContextSwapLock * 16) + PbLockQueue, rpT1
br.call.sptk brp = KeAcquireQueuedSpinLockAtDpcLevel
;;
br.sptk Ksp_Continue
#endif // !defined(NT_UP)
;;
ALTERNATE_ENTRY(KxSwapProcess)
NESTED_SETUP(2,3,3,0)
PROLOGUE_END
//
// Clear the processor set member number in the old process and set the
// processor member number in the new process.
//
Ksp_Continue:
#if !defined(NT_UP)
add rpT2 = PrActiveProcessors, rOldProc // -> old active processor set
movl rpT1 = KiPcr + PcSetMember // -> processor set member
;;
ld4 rProcSet= [rpT1] // rProcSet.4 = processor set member
add rpT1 = PrActiveProcessors, rNewProc // -> new active processor set
;;
ld4 rNewActive = [rpT1] // rNewActive.4 = new active processor set
ld4 rOldActive = [rpT2] // rOldActive.4 = old active processor set
;;
or rNewActive = rNewActive,rProcSet // set processor member in new set
xor rOldActive = rOldActive,rProcSet // clear processor member in old set
;;
st4 [rpT1] = rNewActive // set new active processor set
st4 [rpT2] = rOldActive // set old active processor set
#endif // !defined(NT_UP)
//
// If the process sequence number matches the system sequence number, then
// use the process RID. Otherwise, allocate a new process RID.
//
// N.B. KiMasterRid, KiMasterSequence are changed only when holding the
// KiContextSwapLock.
//
add rT2 = PrSessionMapInfo, rNewProc
add out0 = PrProcessRegion, rNewProc
;;
ld8 out1 = [rT2]
br.call.sptk brp = KiSyncNewRegionId
;;
//
// Switch address space to new process
// v0 = rRid = new process rid
//
fwb // hint to flush write buffers
FAST_DISABLE_INTERRUPTS
add rpT1 = PrDirectoryTableBase, rNewProc
movl rVa = KiPcr+PcPdeUtbase
add rpT2 = PrSessionParentBase, rNewProc
movl rVa2 = KiPcr+PcPdeStbase
;;
ld8.nta rPDE0 = [rpT1] // rPDE0 = Page directory page 0
ld8.nta rSessionBase = [rpT2]
ld8.nta rVa = [rVa]
ld8.nta rVa2 = [rVa2]
;;
//
// To access IFA, ITDR registers, PSR.ic bit must be 0. Otherwise,
// it causes an illegal operation fault. While PSR.ic=0, any
// interruption can not be afforded. Make sure there will be no
// TLB miss and no interrupt coming in during this period.
//
rsm 1 << PSR_IC // PSR.ic=0
;;
srlz.d // must serialize
mov rT1 = PAGE_SHIFT << IDTR_PS // load page size field for IDTR
;;
mov cr.itir = rT1 // set up IDTR for dirbase
ptr.d rVa, rT1 // remove DTR for user space
;;
mov cr.ifa = rVa // set up IFA for dirbase vaddr
mov rT2 = DTR_UTBASE_INDEX
;;
itr.d dtr[rT2] = rPDE0 // insert PDE0 to DTR
;;
ptr.d rVa2, rT1 // remove DTR for session
;; // to avoid a overlapping error
mov cr.ifa = rVa2
mov rT2 = DTR_STBASE_INDEX
;;
itr.d dtr[rT2] = rSessionBase // insert the root for session space
;;
ssm 1 << PSR_IC // PSR.ic=1
;;
srlz.i // must I serialize
#if DBG
mov t0 = PbProcessorState+KpsSpecialRegisters+KsTrD0+(8*DTR_UTBASE_INDEX)
movl t3 = KiPcr + PcPrcb
;;
ld8 t3 = [t3]
mov t1 = PbProcessorState+KpsSpecialRegisters+KsTrD0+(8*DTR_STBASE_INDEX)
;;
add t0 = t3, t0
add t1 = t3, t1
;;
st8 [t0] = rPDE0
st8 [t1] = rSessionBase
;;
#endif
FAST_ENABLE_INTERRUPTS
//
// Now make sure branch history is enabled for non wow processes
// and disabled for wow processes
//
add t1 = @gprel(KiVectorLogMask), gp
;;
ld8 t1 = [t1]
;;
cmp.eq pt0 = t1, r0
(pt0) br.cond.sptk SkipBranchHistory
mov t1 = 3
;;
mov t2 = cpuid[t1]
add t3 = PrWow64Process, rNewProc
;;
extr.u t2 = t2, 24, 8
ld4 t4 = [t3];
;;
cmp.ne pt1 = 7, t2
;;
mov t1 = 675
(pt1) br.dpnt SkipBranchHistory
;;
mov t2 = msr[t1]
cmp.eq pt1,pt2 = zero, t4 // Wow64 is non-zero
;;
(pt1) mov t3 = 2 // Enable the HB for ia64 procs
(pt2) mov t3 = 256 // Disable the HB for wow64 procs
;;
dep t2 = t3, t2, 0, 9 // Disable the HB for wow64 procs
;;
mov msr[t1] = t2;
;;
SkipBranchHistory:
#if !defined(NT_UP)
//
// Can now release the context swap lock
//
movl rpT1 = KiPcr+PcPrcb
;;
ld8 rpT1 = [rpT1]
;;
add out0 = (LockQueueContextSwapLock * 16) + PbLockQueue, rpT1
br.call.sptk brp = KeReleaseQueuedSpinLockFromDpcLevel
;;
#endif // !defined(NT_UP)
NESTED_RETURN
NESTED_EXIT(KiSwapProcess)
SBTTL("Retire Deferred Procedure Call List")
//++
// Routine:
//
// VOID
// KiRetireDpcList (
// PKPRCB Prcb,
// )
//
// Routine Description:
//
// This routine is called to retire the specified deferred procedure
// call list. DPC routines are called using the idle thread (current)
// stack.
//
// N.B. Interrupts must be disabled on entry to this routine. Control is returned
// to the caller with the same conditions true.
//
// Arguments:
//
// a0 - Address of the current PRCB.
//
// Return value:
//
// None.
//
//--
NESTED_ENTRY(KiRetireDpcList)
NESTED_SETUP(1,2,4,0)
PROLOGUE_END
Krdl_Restart:
add t0 = PbDpcQueueDepth, a0
add t1 = PbDpcRoutineActive, a0
add t2 = PbDpcLock, a0
;;
ld4 t4 = [t0]
add t3 = PbDpcListHead+LsFlink, a0
;;
Krdl_Restart2:
cmp4.eq pt1 = zero, t4
st4 [t1] = t4
(pt1) br.spnt Krdl_Exit
;;
#if !defined(NT_UP)
ACQUIRE_SPINLOCK(t2, a0, Krdl_20)
#endif // !defined(NT_UP)
ld4 t4 = [t0]
LDPTR (t5, t3) // -> first DPC entry
;;
cmp4.eq pt1, pt2 = zero, t4
;;
(pt2) add t10 = LsFlink, t5
(pt2) add out0 = -DpDpcListEntry, t5
(pt1) br.spnt Krdl_Unlock
;;
LDPTR (t6, t10)
add t11 = DpDeferredRoutine, out0
add t12 = DpSystemArgument1, out0
;;
//
// Setup call to DPC routine
//
// arguments are:
// dpc object address (out0)
// deferred context (out1)
// system argument 1 (out2)
// system argument 2 (out3)
//
// N.B. the arguments must be loaded from the DPC object BEFORE
// the inserted flag is cleared to prevent the object being
// overwritten before its time.
//
ld8.nt1 t13 = [t11], DpDeferredContext-DpDeferredRoutine
ld8.nt1 out2 = [t12], DpSystemArgument2-DpSystemArgument1
;;
ld8.nt1 out1 = [t11], DpLock-DpDeferredContext
ld8.nt1 out3 = [t12]
add t4 = -1, t4
STPTRINC (t3, t6, -LsFlink)
ld8.nt1 t14 = [t13], 8
add t15 = LsBlink, t6
;;
ld8.nt1 gp = [t13]
STPTR (t15, t3)
STPTR (t11, zero)
st4 [t0] = t4
#if !defined(NT_UP)
RELEASE_SPINLOCK(t2) // set spin lock not owned
#endif //!defined(NT_UP)
FAST_ENABLE_INTERRUPTS
mov bt0 = t14
br.call.sptk.few.clr brp = bt0 // call DPC routine
;;
//
// Check to determine if any more DPCs are available to process.
//
FAST_DISABLE_INTERRUPTS
br Krdl_Restart
;;
//
// The DPC list became empty while we were acquiring the DPC queue lock.
// Clear DPC routine active. The race condition mentioned above doesn't
// exist here because we hold the DPC queue lock.
//
Krdl_Unlock:
#if !defined(NT_UP)
add t2 = PbDpcLock, a0
;;
RELEASE_SPINLOCK(t2)
#endif // !defined(NT_UP)
Krdl_Exit:
add t0 = PbDpcQueueDepth, a0
add t1 = PbDpcRoutineActive, a0
add out0 = PbDpcInterruptRequested, a0
;;
st4.nta [t1] = zero
st4.rel.nta [out0] = zero
add t2 = PbDpcLock, a0
ld4 t4 = [t0]
add t3 = PbDpcListHead+LsFlink, a0
;;
cmp4.eq pt1, pt2 = zero, t4
(pt2) br.spnt Krdl_Restart2
;;
NESTED_RETURN
NESTED_EXIT(KiRetireDpcList)
SBTTL("Dispatch Interrupt")
//++
//--------------------------------------------------------------------
// Routine:
//
// KiDispatchInterrupt
//
// Routine Description:
//
// This routine is entered as the result of a software interrupt generated
// at DISPATCH_LEVEL. Its function is to process the Deferred Procedure Call
// (DPC) list, and then perform a context switch if a new thread has been
// selected for execution on the processor.
//
// This routine is entered at IRQL DISPATCH_LEVEL with the dispatcher
// database unlocked. When a return to the caller finally occurs, the
// IRQL remains at DISPATCH_LEVEL, and the dispatcher database is still
// unlocked.
//
// N.B. On entry to this routine the volatile states (excluding high
// floating point register set) have been saved.
//
// On entry:
//
// sp - points to stack scratch area.
//
// Arguments:
//
// None
//
// Return Value:
//
// None.
//--------------------------------------------------------------------
//--
NESTED_ENTRY(KiDispatchInterrupt)
PROLOGUE_BEGIN
.regstk 0, 4, 2, 0
alloc t16 = ar.pfs, 0, 4, 2, 0
.save rp, loc0
mov loc0 = brp
.fframe SwitchFrameLength
add sp = -SwitchFrameLength, sp
;;
.save ar.unat, loc1
mov loc1 = ar.unat
add t0 = ExFltS19+SwExFrame+STACK_SCRATCH_AREA, sp
add t1 = ExFltS18+SwExFrame+STACK_SCRATCH_AREA, sp
;;
.save.gf 0x0, 0xC0000
stf.spill [t0] = fs19, ExFltS17-ExFltS19
stf.spill [t1] = fs18, ExFltS16-ExFltS18
;;
.save.gf 0x0, 0x30000
stf.spill [t0] = fs17, ExFltS15-ExFltS17
stf.spill [t1] = fs16, ExFltS14-ExFltS16
mov t10 = bs4
;;
.save.gf 0x0, 0xC000
stf.spill [t0] = fs15, ExFltS13-ExFltS15
stf.spill [t1] = fs14, ExFltS12-ExFltS14
mov t11 = bs3
;;
.save.gf 0x0, 0x3000
stf.spill [t0] = fs13, ExFltS11-ExFltS13
stf.spill [t1] = fs12, ExFltS10-ExFltS12
mov t12 = bs2
;;
.save.gf 0x0, 0xC00
stf.spill [t0] = fs11, ExFltS9-ExFltS11
stf.spill [t1] = fs10, ExFltS8-ExFltS10
mov t13 = bs1
;;
.save.gf 0x0, 0x300
stf.spill [t0] = fs9, ExFltS7-ExFltS9
stf.spill [t1] = fs8, ExFltS6-ExFltS8
mov t14 = bs0
;;
.save.gf 0x0, 0xC0
stf.spill [t0] = fs7, ExFltS5-ExFltS7
stf.spill [t1] = fs6, ExFltS4-ExFltS6
mov t15 = ar.lc
;;
.save.gf 0x0, 0x30
stf.spill [t0] = fs5, ExFltS3-ExFltS5
stf.spill [t1] = fs4, ExFltS2-ExFltS4
;;
.save.f 0xC
stf.spill [t0] = fs3, ExFltS1-ExFltS3 // save fs3
stf.spill [t1] = fs2, ExFltS0-ExFltS2 // save fs2
;;
.save.f 0x3
stf.spill [t0] = fs1, ExBrS4-ExFltS1 // save fs1
stf.spill [t1] = fs0, ExBrS3-ExFltS0 // save fs0
;;
.save.b 0x18
st8 [t0] = t10, ExBrS2-ExBrS4 // save bs4
st8 [t1] = t11, ExBrS1-ExBrS3 // save bs3
;;
.save.b 0x6
st8 [t0] = t12, ExBrS0-ExBrS2 // save bs2
st8 [t1] = t13, ExIntS2-ExBrS1 // save bs1
;;
.save.b 0x1
st8 [t0] = t14, ExIntS3-ExBrS0 // save bs0
;;
.save.gf 0xC, 0x0
.mem.offset 0,0
st8.spill [t0] = s3, ExIntS1-ExIntS3 // save s3
.mem.offset 8,0
st8.spill [t1] = s2, ExIntS0-ExIntS2 // save s2
;;
.save.gf 0x3, 0x0
.mem.offset 0,0
st8.spill [t0] = s1, ExApLC-ExIntS1 // save s1
.mem.offset 8,0
st8.spill [t1] = s0, ExApEC-ExIntS0 // save s0
;;
.savepsp ar.pfs, ExceptionFrameLength-ExApEC-STACK_SCRATCH_AREA
st8 [t1] = t16, ExIntNats-ExApEC
mov t4 = ar.unat // captured Nats of s0-s3
;;
.savepsp ar.lc, ExceptionFrameLength-ExApLC-STACK_SCRATCH_AREA
st8 [t0] = t15
.savepsp @priunat, ExceptionFrameLength-ExIntNats-STACK_SCRATCH_AREA
st8 [t1] = t4 // save Nats of s0-s3
PROLOGUE_END
//
// Register aliases
//
rPrcb = loc2
rKerGP = loc3
rpT1 = t0
rpT2 = t1
rT1 = t2
rT2 = t3
rpDPLock = t4 // pointer to dispatcher lock
pNoTh = pt1 // No next thread to run
pNext = pt2 // next thread not null
pNull = pt3 // no thread available
pOwned = pt4 // dispatcher lock already owned
pNotOwned = pt5
pQEnd = pt6 // quantum end request pending
pNoQEnd = pt7 // no quantum end request pending
//
// Increment the dispatch interrupt count
//
mov rKerGP = gp // save gp
movl rPrcb = KiPcr + PcPrcb
;;
LDPTR (rPrcb, rPrcb) // rPrcb -> Prcb
;;
add rpT1 = PbDispatchInterruptCount, rPrcb
;;
ld4 rT1 = [rpT1]
;;
add rT1 = rT1, zero, 1
;;
st4 [rpT1] = rT1
// **** TBD **** use alpha optimization to first check Dpc Q depth
//
// Process the DPC list
//
Kdi_PollDpcList:
//
// Process the deferred procedure call list.
//
FAST_ENABLE_INTERRUPTS
;;
srlz.d
//
// **** TBD ***** No stack switch as in alpha, mips...
// Save current initial stack address and set new initial stack address.
//
FAST_DISABLE_INTERRUPTS
mov out0 = rPrcb
br.call.sptk brp = KiRetireDpcList
;;
//
// Check to determine if quantum end has occured.
//
// N.B. If a new thread is selected as a result of processing a quantum
// end request, then the new thread is returned with the dispatcher
// database locked. Otherwise, NULL is returned with the dispatcher
// database unlocked.
//
FAST_ENABLE_INTERRUPTS
add rpT1 = PbQuantumEnd, rPrcb
;;
ld4 rT1 = [rpT1] // get quantum end indicator
;;
cmp4.ne pQEnd, pNoQEnd = rT1, zero // if zero, no quantum end reqs
mov gp = rKerGP // restore gp
;;
(pQEnd) st4 [rpT1] = zero // clear quantum end indicator
(pNoQEnd) br.cond.sptk Kdi_NoQuantumEnd
(pQEnd) br.call.spnt brp = KiQuantumEnd // call KiQuantumEnd (C code)
;;
cmp4.eq pNoTh, pNext = v0, zero // pNoTh = no next thread
(pNoTh) br.dpnt Kdi_Exit // br to exit if no next thread
(pNext) br.dpnt Kdi_Swap // br to swap to next thread
//
// If no quantum end requests:
// Check to determine if a new thread has been selected for execution on
// this processor.
//
Kdi_NoQuantumEnd:
add rpT2 = PbNextThread, rPrcb
;;
LDPTR (rT1, rpT2) // rT1 = address of next thread object
;;
cmp.eq pNull = rT1, zero // pNull => no thread selected
(pNull) br.dpnt Kdi_Exit // exit if no thread selected
#if !defined(NT_UP)
//
// try to acquire the dispatcher database lock.
//
mov out0 = LockQueueDispatcherLock
movl out1 = KiPcr+PcSystemReserved+8
br.call.sptk brp = KeTryToAcquireQueuedSpinLockRaiseToSynch
;;
cmp.ne pOwned, pNotOwned = TRUE, v0 // pOwned = 1 if not free
(pOwned) br.dpnt Kdi_PollDpcList // br out if owned
;;
#else
mov rT1 = SYNCH_LEVEL
;;
SET_IRQL (rT1)
#endif // !defined(NT_UP)
//
// Reread address of next thread object since it is possible for it to
// change in a multiprocessor system.
//
Kdi_Swap:
add rpT2 = PbNextThread, rPrcb // -> next thread
movl rpT1 = KiPcr + PcCurrentThread
;;
LDPTR (s1, rpT1) // current thread object
LDPTR (s2, rpT2) // next thread object
add rpT1 = PbCurrentThread, rPrcb
;;
//
// Reready current thread for execution and swap context to the selected
// thread.
//
// Note: Set IdleSwapBlock in the current thread so no idle processor
// can switch to this processor before it is removed from the current
// processor.
//
STPTR (rpT2, zero) // clear addr of next thread
add out0 = ThIdleSwapBlock, s1 // block swap from idle
mov rT1 = 1
;;
STPTR (rpT1, s2) // set addr of current thread
st1 [out0] = rT1, -ThIdleSwapBlock// set addr of previous thread
br.call.sptk brp = KiReadyThread // call KiReadyThread(OldTh)
;;
mov s0 = rPrcb // setup call
cmp.ne pt0 = zero, zero // no need to lock context swap
br.call.sptk brp = SwapContext // call SwapContext(Prcb, OldTh, NewTh)
;;
//
// Restore saved registers, and return.
//
add out0 = STACK_SCRATCH_AREA+SwExFrame, sp
br.call.sptk brp = KiRestoreExceptionFrame
;;
Kdi_Exit:
add rpT1 = ExApEC+SwExFrame+STACK_SCRATCH_AREA, sp
;;
ld8 rT1 = [rpT1]
mov brp = loc0
;;
mov ar.unat = loc1
mov ar.pfs = rT1
.restore
add sp = SwitchFrameLength, sp
br.ret.sptk brp
NESTED_EXIT(KiDispatchInterrupt)