You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1490 lines
35 KiB
1490 lines
35 KiB
//++
|
|
//
|
|
// Module Name:
|
|
//
|
|
// spinlock.s
|
|
//
|
|
// Abstract:
|
|
//
|
|
// This module implements the routines for acquiring and releasing
|
|
// spin locks.
|
|
//
|
|
// Author:
|
|
//
|
|
// William K. Cheung (wcheung) 29-Sep-1995
|
|
//
|
|
// Environment:
|
|
//
|
|
// Kernel mode only.
|
|
//
|
|
// Revision History:
|
|
//
|
|
// 31-Dec-1998 wc Updated to use xchg8
|
|
//
|
|
// 07-Jul-1997 bl Updated to EAS2.3
|
|
//
|
|
// 08-Feb-1996 Updated to EAS2.1
|
|
//
|
|
//--
|
|
|
|
#include "ksia64.h"
|
|
#include "icecap.h"
|
|
|
|
|
|
.file "spinlock.s"
|
|
|
|
//
|
|
// Define LOG2(x) for those values whose bit numbers are needed in
|
|
// order to test a single bit with the tbit instruction.
|
|
//
|
|
|
|
#define _LOG2_0x1 0
|
|
#define _LOG2_0x2 1
|
|
#define _LOG2_x(n) _LOG2_##n
|
|
#define LOG2(n) _LOG2_x(n)
|
|
|
|
//
|
|
// Globals
|
|
//
|
|
|
|
PublicFunction(KiCheckForSoftwareInterrupt)
|
|
|
|
|
|
//++
|
|
//
|
|
// VOID
|
|
// KiAcquireSpinLock (
|
|
// IN PKSPIN_LOCK SpinLock
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// This function acquires a kernel spin lock.
|
|
//
|
|
// N.B. This function assumes that the current IRQL is set properly.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// SpinLock (a0) - Supplies a pointer to a kernel spin lock.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// None.
|
|
//
|
|
//--
|
|
|
|
LEAF_ENTRY(KiAcquireSpinLock)
|
|
|
|
ALTERNATE_ENTRY(KeAcquireSpinLockAtDpcLevel)
|
|
|
|
#if !defined(NT_UP)
|
|
|
|
#ifndef CAPKERN_SYNCH_POINTS
|
|
ACQUIRE_SPINLOCK(a0,a0,Kiasl10)
|
|
#else
|
|
CAP_ACQUIRE_SPINLOCK(a0,a0,Kiasl10,t0,t1,t2,t3)
|
|
#endif
|
|
|
|
#endif // !defined(NT_UP)
|
|
|
|
LEAF_RETURN
|
|
|
|
LEAF_EXIT(KiAcquireSpinLock)
|
|
|
|
//++
|
|
//
|
|
// BOOLEAN
|
|
// KeTryToAcquireSpinLockAtDpcLevel (
|
|
// IN PKSPIN_LOCK SpinLock
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// This function attempts to acquires the specified kernel spinlock. If
|
|
// the spinlock can be acquired, then TRUE is returned. Otherwise, FALSE
|
|
// is returned.
|
|
//
|
|
// N.B. This function assumes that the current IRQL is set properly.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// SpinLock (a0) - Supplies a pointer to a kernel spin lock.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// If the spin lock is acquired, then a value of TRUE is returned.
|
|
// Otherwise, a value of FALSE is returned.
|
|
//
|
|
// N.B. The caller KeTryToAcquireSpinLock implicitly depends on the
|
|
// contents of predicate registers pt1 & pt2.
|
|
//
|
|
//--
|
|
|
|
LEAF_ENTRY(KeTryToAcquireSpinLockAtDpcLevel)
|
|
|
|
#if !defined(NT_UP)
|
|
|
|
#ifdef CAPKERN_SYNCH_POINTS
|
|
CAPSPINLOG1INT(a0, 8, t0, t1, t2, t3, pt0)
|
|
#endif
|
|
|
|
xchg8 t0 = [a0], a0
|
|
;;
|
|
cmp.ne pt0 = t0, zero // if ne, lock acq failed
|
|
mov v0 = TRUE // acquire assumed succeed
|
|
;;
|
|
(pt0) YIELD
|
|
(pt0) mov v0 = FALSE // return FALSE
|
|
|
|
#ifdef CAPKERN_SYNCH_POINTS
|
|
(pt0) br.cond.spnt kttasladlSkipLog
|
|
CAPSPINLOG1INT(a0, 1, t0, t1, t2, t3, pt0)
|
|
kttasladlSkipLog:
|
|
#endif
|
|
|
|
|
|
#else
|
|
mov v0 = TRUE
|
|
#endif
|
|
|
|
LEAF_RETURN
|
|
LEAF_EXIT(KeTryToAcquireSpinLockAtDpcLevel)
|
|
|
|
|
|
//++
|
|
//
|
|
// VOID
|
|
// KeAcquireSpinLock (
|
|
// IN PKSPIN_LOCK SpinLock
|
|
// OUT PKIRQL OldIrql
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// This function raises the current IRQL to DISPATCH_LEVEL and acquires
|
|
// the specified executive spinlock.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// SpinLock (a0) - Supplies a pointer to a executive spinlock.
|
|
//
|
|
// OldIrql (a1) - Supplies a pointer to a variable that receives the
|
|
// the previous IRQL value.
|
|
//
|
|
// N.B. The Old IRQL MUST be stored after the lock is acquired.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// None.
|
|
//
|
|
//--
|
|
|
|
LEAF_ENTRY(KeAcquireSpinLock)
|
|
|
|
//
|
|
// Get original IRQL, raise IRQL to DISPATCH_LEVEL
|
|
// and then acquire the specified spinlock.
|
|
//
|
|
|
|
mov t0 = DISPATCH_LEVEL
|
|
SWAP_IRQL(t0)
|
|
|
|
#if !defined(NT_UP)
|
|
|
|
#ifndef CAPKERN_SYNCH_POINTS
|
|
ACQUIRE_SPINLOCK(a0,a0,Kasl10)
|
|
#else
|
|
CAP_ACQUIRE_SPINLOCK(a0,a0,Kasl10,t1,t2,t3,t4)
|
|
#endif
|
|
|
|
#endif // !defined(NT_UP)
|
|
|
|
st1 [a1] = v0 // save old IRQL
|
|
LEAF_RETURN
|
|
|
|
LEAF_EXIT(KeAcquireSpinLock)
|
|
|
|
|
|
|
|
//++
|
|
//
|
|
// KIRQL
|
|
// KeAcquireSpinLockRaiseToSynch (
|
|
// IN PKSPIN_LOCK SpinLock
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// This function raises the current IRQL to synchronization level and
|
|
// acquires the specified spinlock.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// SpinLock (a0) - Supplies a pointer to the spinlock that is to be
|
|
// acquired.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// The previous IRQL is returned as the function value.
|
|
//
|
|
//--
|
|
|
|
LEAF_ENTRY(KeAcquireSpinLockRaiseToSynch)
|
|
|
|
//
|
|
// Register aliases
|
|
//
|
|
|
|
pHeld = pt0
|
|
pFree = pt1
|
|
|
|
mov t1 = SYNCH_LEVEL
|
|
|
|
#if !defined(NT_UP)
|
|
|
|
#ifdef CAPKERN_SYNCH_POINTS
|
|
CAPSPINLOG1INT(a0,1,t3,t4,t5,t6,pt2)
|
|
mov t2 = zero
|
|
#endif
|
|
|
|
|
|
GET_IRQL (v0)
|
|
|
|
KaslrtsRetry:
|
|
|
|
SET_IRQL (t1)
|
|
xchg8 t0 = [a0], a0
|
|
;;
|
|
cmp.eq pFree, pHeld = 0, t0
|
|
;;
|
|
PSET_IRQL (pHeld, v0)
|
|
|
|
#ifdef CAPKERN_SYNCH_POINTS
|
|
|
|
(pHeld) br.cond.dpnt KaslrtsSkipCollLog
|
|
cmp.eq pt2 = t2, zero
|
|
(pt2) br.cond.sptk KaslrtsSkipCollLog
|
|
CAPSPINLOG2INT(t2,a0,2,t3,t4,t5,t6,pt2)
|
|
|
|
KaslrtsSkipCollLog:
|
|
|
|
#endif //CAPKERN_SYNCH_POINTS
|
|
|
|
(pFree) LEAF_RETURN
|
|
;;
|
|
|
|
KaslrtsLoop:
|
|
|
|
YIELD
|
|
|
|
#ifdef CAPKERN_SYNCH_POINTS
|
|
add t2 = 1, t2
|
|
#endif
|
|
cmp.eq pFree, pHeld = 0, t0
|
|
ld8.nt1 t0 = [a0]
|
|
(pFree) br.cond.dpnt KaslrtsRetry
|
|
(pHeld) br.cond.dptk KaslrtsLoop
|
|
|
|
|
|
#else
|
|
|
|
SWAP_IRQL (t1) // Raise IRQL
|
|
LEAF_RETURN
|
|
|
|
#endif // !defined(NT_UP)
|
|
|
|
LEAF_EXIT(KeAcquireSpinLockRaiseToSynch)
|
|
|
|
|
|
//++
|
|
//
|
|
// KIRQL
|
|
// KeAcquireSpinLockRaiseToDpc (
|
|
// IN PKSPIN_LOCK SpinLock
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// This function raises the current IRQL to dispatcher level and acquires
|
|
// the specified spinlock.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// SpinLock (a0) - Supplies a pointer to the spinlock that is to be
|
|
// acquired.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// The previous IRQL is returned as the function value.
|
|
//
|
|
//--
|
|
|
|
|
|
LEAF_ENTRY(KeAcquireSpinLockRaiseToDpc)
|
|
|
|
mov t2 = DISPATCH_LEVEL
|
|
;;
|
|
SWAP_IRQL (t2)
|
|
|
|
#if !defined(NT_UP)
|
|
|
|
cmp.eq pt0, pt1 = zero, zero
|
|
cmp.eq pt2, pt3 = zero, zero
|
|
|
|
#ifdef CAPKERN_SYNCH_POINTS
|
|
CAPSPINLOG1INT(a0, 1, t3, t4, t5, t6, pt4)
|
|
mov t1 = zero
|
|
#endif
|
|
;;
|
|
|
|
Kaslrtp10:
|
|
.pred.rel "mutex",pt0,pt1
|
|
(pt0) xchg8 t0 = [a0], a0
|
|
(pt1) ld8.nt1 t0 = [a0]
|
|
|
|
;;
|
|
(pt0) cmp.ne pt2, pt3 = zero, t0
|
|
cmp.eq pt0, pt1 = zero, t0
|
|
;;
|
|
(pt1) YIELD
|
|
#ifdef CAPKERN_SYNCH_POINTS
|
|
(pt2) add t1 = 1, t1
|
|
#endif
|
|
(pt2) br.dpnt Kaslrtp10
|
|
|
|
#ifdef CAPKERN_SYNCH_POINTS
|
|
cmp.eq pt1 = t1, zero
|
|
(pt1) br.sptk KaslrtpSkipCollLog
|
|
CAPSPINLOG2INT(t1,a0,2,t3,t4,t5,t6,pt1)
|
|
KaslrtpSkipCollLog:
|
|
#endif
|
|
|
|
(pt3) br.ret.dptk brp
|
|
|
|
#else
|
|
|
|
LEAF_RETURN
|
|
|
|
#endif // !defined(NT_UP)
|
|
|
|
LEAF_EXIT(KeAcquireSpinLockRaiseToDpc)
|
|
|
|
|
|
|
|
//++
|
|
//
|
|
// VOID
|
|
// KiReleaseSpinLock (
|
|
// IN PKSPIN_LOCK SpinLock
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// This function releases a kernel spin lock.
|
|
//
|
|
// N.B. This function assumes that the current IRQL is set properly.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// SpinLock (a0) - Supplies a pointer to an executive spin lock.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// None.
|
|
//
|
|
//--
|
|
|
|
LEAF_ENTRY(KiReleaseSpinLock)
|
|
ALTERNATE_ENTRY(KeReleaseSpinLockFromDpcLevel)
|
|
|
|
#if !defined(NT_UP)
|
|
st8.rel [a0] = zero // set spin lock not owned
|
|
|
|
#ifdef CAPKERN_SYNCH_POINTS
|
|
CAPSPINLOG1INT(a0, 7, t0, t1, t2, t3, pt0)
|
|
#endif
|
|
|
|
#endif
|
|
|
|
LEAF_RETURN
|
|
LEAF_EXIT(KiReleaseSpinLock)
|
|
|
|
|
|
//++
|
|
//
|
|
// VOID
|
|
// KeReleaseSpinLock (
|
|
// IN PKSPIN_LOCK SpinLock
|
|
// IN KIRQL OldIrql
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// This function releases an executive spin lock and lowers the IRQL
|
|
// to its previous value. Called at DPC_LEVEL.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// SpinLock (a0) - Supplies a pointer to an executive spin lock.
|
|
//
|
|
// OldIrql (a1) - Supplies the previous IRQL value.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// None.
|
|
//
|
|
//--
|
|
|
|
|
|
LEAF_ENTRY(KeReleaseSpinLock)
|
|
|
|
zxt1 a1 = a1
|
|
|
|
#if !defined(NT_UP)
|
|
st8.rel [a0] = zero // set spinlock not owned
|
|
|
|
#ifdef CAPKERN_SYNCH_POINTS
|
|
CAPSPINLOG1INT(a0, 7, t0, t1, t2, t3, pt0)
|
|
#endif
|
|
|
|
#endif
|
|
|
|
;;
|
|
LEAF_LOWER_IRQL_AND_RETURN(a1) // Lower IRQL and return
|
|
|
|
LEAF_EXIT(KeReleaseSpinLock)
|
|
|
|
//++
|
|
//
|
|
// BOOLEAN
|
|
// KeTryToAcquireSpinLock (
|
|
// IN PKSPIN_LOCK SpinLock
|
|
// OUT PKIRQL OldIrql
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// This function raises the current IRQL to DISPATCH_LEVEL and attempts
|
|
// to acquires the specified executive spinlock. If the spinlock can be
|
|
// acquired, then TRUE is returned. Otherwise, the IRQL is restored to
|
|
// its previous value and FALSE is returned. Called at IRQL <= DISPATCH_LEVEL.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// SpinLock (a0) - Supplies a pointer to a executive spinlock.
|
|
//
|
|
// OldIrql (a1) - Supplies a pointer to a variable that receives the
|
|
// the previous IRQL value.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// If the spin lock is acquired, then a value of TRUE is returned.
|
|
// Otherwise, a value of FALSE is returned.
|
|
//
|
|
// N.B. This routine assumes KeTryToAcquireSpinLockAtDpcLevel pt1 & pt2 will
|
|
// be set to reflect the result of the attempt to acquire the spinlock.
|
|
//
|
|
//--
|
|
|
|
LEAF_ENTRY(KeTryToAcquireSpinLock)
|
|
|
|
#ifdef CAPKERN_SYNCH_POINTS
|
|
CAPSPINLOG1INT(a0, 8, t0, t1, t2, t3, pt2)
|
|
#endif
|
|
|
|
rOldIrql = t2
|
|
|
|
//
|
|
// Raise IRQL to DISPATCH_LEVEL and try to acquire the specified spinlock.
|
|
// Return FALSE if failed; otherwise, return TRUE.
|
|
//
|
|
|
|
GET_IRQL (rOldIrql) // get original IRQL
|
|
mov t0 = DISPATCH_LEVEL;;
|
|
SET_IRQL (t0) // raise to dispatch level
|
|
|
|
#if !defined(NT_UP)
|
|
|
|
xchg8 t0 = [a0], a0
|
|
;;
|
|
cmp.ne pt2 = t0, zero // if ne, lock acq failed
|
|
;;
|
|
|
|
|
|
//
|
|
// If successfully acquired, pt1 is set to TRUE while pt2 is set to FALSE.
|
|
// Otherwise, pt2 is set to TRUE while pt1 is set to FALSE.
|
|
//
|
|
|
|
(pt2) YIELD
|
|
(pt2) mov v0 = FALSE // return FALSE
|
|
PSET_IRQL (pt2, rOldIrql) // restore old IRQL
|
|
|
|
#ifdef CAPKERN_SYNCH_POINTS
|
|
|
|
(pt2) br.cond.dpnt KttasSkipLog
|
|
CAPSPINLOG1INT(a0, 1, t0, t1, t2, t3, pt0)
|
|
|
|
KttasSkipLog:
|
|
#endif
|
|
|
|
(pt2) LEAF_RETURN
|
|
;;
|
|
|
|
#endif // !defined(NT_UP)
|
|
|
|
st1 [a1] = rOldIrql // save old IRQL
|
|
mov v0 = TRUE // successfully acquired
|
|
LEAF_RETURN
|
|
|
|
LEAF_EXIT(KeTryToAcquireSpinLock)
|
|
|
|
//++
|
|
//
|
|
// BOOLEAN
|
|
// KeTestSpinLock (
|
|
// IN PKSPIN_LOCK SpinLock
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// This function tests a kernel spin lock. If the spinlock is
|
|
// busy, FALSE is returned. If not, TRUE is returned. The spinlock
|
|
// is never acquired. This is provided to allow code to spin at low
|
|
// IRQL, only raising the IRQL when there is a reasonable hope of
|
|
// acquiring the lock.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// SpinLock (a0) - Supplies a pointer to a kernel spin lock.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// TRUE - Spinlock appears available
|
|
// FALSE - SpinLock is busy
|
|
//--
|
|
|
|
LEAF_ENTRY(KeTestSpinLock)
|
|
|
|
ld8.nt1 t0 = [a0]
|
|
;;
|
|
cmp.ne pt0 = 0, t0
|
|
mov v0 = TRUE // default TRUE
|
|
;;
|
|
|
|
(pt0) YIELD
|
|
(pt0) mov v0 = FALSE // if t0 != 0 return FALSE
|
|
LEAF_RETURN
|
|
|
|
LEAF_EXIT(KeTestSpinLock)
|
|
|
|
SBTTL("Acquire Queued SpinLock and Raise IRQL")
|
|
//++
|
|
//
|
|
// VOID
|
|
// KeAcquireInStackQueuedSpinLock (
|
|
// IN PKSPIN_LOCK SpinLock,
|
|
// IN PKLOCK_QUEUE_HANDLE LockHandle
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// This function raises the current IRQL to dispatch level and
|
|
// acquires the specified queued spinlock.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// SpinLock (a0) - Supplies a pointer to a spin lock.
|
|
//
|
|
// LockHandle (a1) - Supplies a pointer to a lock handle.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// None.
|
|
//
|
|
//--
|
|
LEAF_ENTRY(KeAcquireInStackQueuedSpinLockRaiseToSynch)
|
|
|
|
add t5 = LqhNext, a1
|
|
mov t1 = SYNCH_LEVEL
|
|
br.sptk Kaisqsl10
|
|
;;
|
|
|
|
|
|
ALTERNATE_ENTRY(KeAcquireInStackQueuedSpinLock)
|
|
|
|
add t5 = LqhNext, a1
|
|
mov t1 = DISPATCH_LEVEL
|
|
;;
|
|
|
|
Kaisqsl10:
|
|
#if !defined(NT_UP)
|
|
|
|
st8 [t5] = zero // set next link to NULL
|
|
add t4 = LqhLock, a1
|
|
;;
|
|
|
|
st8.rel [t4] = a0 // set spin lock address
|
|
|
|
#endif // !defined(NT_UP)
|
|
|
|
SWAP_IRQL (t1) // old IRQL in register v0
|
|
add t0 = LqhOldIrql, a1
|
|
;;
|
|
|
|
st1 [t0] = v0 // save old IRQL
|
|
|
|
#if !defined(NT_UP)
|
|
|
|
//
|
|
// Finish in common code. The following register values
|
|
// are assumed in that code.
|
|
//
|
|
// t4 = &LockEntry->ActualLock
|
|
// t5 = LockEntry
|
|
// a0 = *(&LockEntry->ActualLock)
|
|
//
|
|
// Note: LqhNext == 0, otherwise we would have to trim t5 here.
|
|
//
|
|
|
|
br KxqAcquireQueuedSpinLock // finish in common code
|
|
|
|
#else
|
|
|
|
br.ret.sptk brp
|
|
|
|
#endif !defined(NT_UP)
|
|
|
|
LEAF_EXIT(KeAcquireInStackQueuedSpinLock)
|
|
|
|
|
|
SBTTL("Acquire Queued SpinLock and Raise IRQL")
|
|
//++
|
|
//
|
|
// KIRQL
|
|
// KeAcquireQueuedSpinLock (
|
|
// IN KSPIN_LOCK_QUEUE_NUMBER Number
|
|
// )
|
|
//
|
|
// KIRQL
|
|
// KeAcquireQueuedSpinLockRaiseToSynch (
|
|
// IN KSPIN_LOCK_QUEUE_NUMBER Number
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// This function raises the current IRQL to synchronization level and
|
|
// acquires the specified queued spinlock.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// Number (a0) - Supplies the queued spinlock number.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// The previous IRQL is returned as the function value.
|
|
//
|
|
//--
|
|
|
|
LEAF_ENTRY(KeAcquireQueuedSpinLock)
|
|
|
|
add t0 = a0, a0
|
|
mov t1 = DISPATCH_LEVEL
|
|
br Kaqsl10
|
|
;;
|
|
|
|
ALTERNATE_ENTRY(KeAcquireQueuedSpinLockRaiseToSynch)
|
|
|
|
add t0 = a0, a0
|
|
mov t1 = SYNCH_LEVEL
|
|
;;
|
|
|
|
Kaqsl10:
|
|
|
|
SWAP_IRQL (t1) // old IRQL in register v0
|
|
|
|
#if !defined(NT_UP)
|
|
movl t2 = KiPcr+PcPrcb
|
|
;;
|
|
ld8 t2 = [t2]
|
|
;;
|
|
shladd t3 = t0, 3, t2 // get associated spinlock addr
|
|
;;
|
|
add t5 = PbLockQueue, t3
|
|
;;
|
|
|
|
add t4 = LqLock, t5
|
|
;;
|
|
ld8 t6 = [t4]
|
|
mov t11 = 0x7
|
|
;;
|
|
|
|
andcm a0 = t6, t11 // mask the lower 3 bits
|
|
;;
|
|
|
|
ALTERNATE_ENTRY(KxqAcquireQueuedSpinLock)
|
|
|
|
#ifdef CAPKERN_SYNCH_POINTS
|
|
CAPSPINLOG1INT(a0,1,t6,t7,t8,t9,pt1)
|
|
mov t12 = zero
|
|
#endif
|
|
|
|
mf // Do a memory fence to ensure the write of LqhNext
|
|
// occurs before the xchg8 on lock address.
|
|
|
|
//
|
|
// t4 = &LockEntry->ActualLock
|
|
// t5 = LockEntry
|
|
// a0 = *(&LockEntry->ActualLock)
|
|
//
|
|
|
|
xchg8 t7 = [a0], t5
|
|
;;
|
|
cmp.ne pt0, pt1 = r0, t7 // if ne, lock already owned
|
|
;;
|
|
(pt1) or t8 = LOCK_QUEUE_OWNER, a0
|
|
(pt0) or t8 = LOCK_QUEUE_WAIT, a0
|
|
;;
|
|
|
|
st8.rel [t4] = t8
|
|
add t9 = LqNext, t7
|
|
(pt1) br.ret.sptk brp
|
|
;;
|
|
|
|
//
|
|
// The lock is already held by another processor. Set the wait
|
|
// bit in this processor's Lock Queue entry, then set the next
|
|
// field in the Lock Queue entry of the last processor to attempt
|
|
// to acquire the lock (this is the address returned by the xchg
|
|
// above) to point to THIS processor's lock queue entry.
|
|
//
|
|
|
|
st8.rel [t9] = t5
|
|
|
|
Kaqsl20:
|
|
#ifdef CAPKERN_SYNCH_POINTS
|
|
add t12 = 1, t12
|
|
#endif
|
|
YIELD
|
|
ld8 t10 = [t4]
|
|
;;
|
|
tbit.z pt1, pt0 = t10, LOG2(LOCK_QUEUE_WAIT)
|
|
(pt0) br.dptk.few Kaqsl20
|
|
#ifdef CAPKERN_SYNCH_POINTS
|
|
CAPSPINLOG2INT(t12,a0,4,t6,t7,t8,t9,pt2)
|
|
#endif
|
|
(pt1) br.ret.dpnt brp // if zero, lock acquired
|
|
|
|
#else
|
|
br.ret.sptk brp
|
|
#endif // !defined(NT_UP)
|
|
;;
|
|
|
|
LEAF_EXIT(KeAcquireQueuedSpinLock)
|
|
|
|
SBTTL("Release Queued SpinLock and Lower IRQL")
|
|
//++
|
|
//
|
|
// VOID
|
|
// KeReleaseInStackQueuedSpinLock (
|
|
// IN PKLOCK_QUEUE_HANDLE LockHandle
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// This function releases a queued spinlock and lowers the IRQL to its
|
|
// previous value.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// LockHandle (a0) - Supplies a pointer to a lock handle.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// None.
|
|
//
|
|
//--
|
|
|
|
LEAF_ENTRY(KeReleaseInStackQueuedSpinLock)
|
|
|
|
alloc t22 = ar.pfs, 2, 2, 2, 0
|
|
add t9 = LqhOldIrql, a0
|
|
add t5 = LqhNext, a0 // set address of lock queue
|
|
;;
|
|
|
|
ld1.nt1 a1 = [t9] // get old IRQL
|
|
br KxqReleaseQueuedSpinLock // finish in common code
|
|
|
|
LEAF_EXIT(KeReleaseInStackQueuedSpinLock)
|
|
|
|
|
|
SBTTL("Release Queued SpinLock and Lower IRQL")
|
|
//++
|
|
//
|
|
// VOID
|
|
// KeReleaseQueuedSpinLock (
|
|
// IN KSPIN_LOCK_QUEUE_NUMBER Number,
|
|
// IN KIRQL OldIrql
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// This function releases a queued spinlock and lowers the IRQL to its
|
|
// previous value.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// Number (a0) - Supplies the queued spinlock number.
|
|
//
|
|
// OldIrql (a1) - Supplies the previous IRQL value.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// None.
|
|
//
|
|
//--
|
|
|
|
|
|
LEAF_ENTRY(KeReleaseQueuedSpinLock)
|
|
PROLOGUE_BEGIN
|
|
|
|
#if !defined(NT_UP)
|
|
movl v0 = KiPcr+PcPrcb
|
|
;;
|
|
|
|
ld8 v0 = [v0]
|
|
add t0 = a0, a0
|
|
;;
|
|
shladd t1 = t0, 3, v0
|
|
;;
|
|
|
|
add t5 = PbLockQueue, t1
|
|
;;
|
|
#endif // !defined(NT_UP)
|
|
|
|
ALTERNATE_ENTRY(KxqReleaseQueuedSpinLock)
|
|
|
|
#if !defined(NT_UP)
|
|
add v0 = LqNext, t5
|
|
add t2 = LqLock, t5
|
|
;;
|
|
|
|
ld8.acq t4 = [t2]
|
|
mov ar.ccv = t5
|
|
|
|
ld8 t3 = [v0]
|
|
;;
|
|
#ifdef CAPKERN_SYNCH_POINTS
|
|
and t6 = ~7, t4
|
|
CAPSPINLOG1INT(t6,7,t7,t8,t9,t10,pt0)
|
|
mov ar.ccv = t5 /* CAPSPINLOG1INT mutates ar.ccv */
|
|
mov t9 = zero
|
|
#endif
|
|
and t4 = ~LOCK_QUEUE_OWNER, t4 // clear lock owner bit
|
|
;;
|
|
add t6 = LqLock, t3
|
|
|
|
st8.rel [t2] = t4
|
|
cmp.ne pt0, pt1 = r0, t3 // if ne, another processor waiting
|
|
(pt0) br.sptk.few Krqsl30
|
|
|
|
ld8 t7 = [t4]
|
|
;;
|
|
cmp.ne pt2 = t5, t7
|
|
(pt2) br.spnt.few Krqsl20
|
|
|
|
cmpxchg8.rel t8 = [t4], r0, ar.ccv
|
|
;;
|
|
cmp.ne pt0, pt1 = t8, t5 // if ne, another processor waiting
|
|
(pt0) br.spnt.few Krqsl20
|
|
;;
|
|
|
|
Krqsl10:
|
|
|
|
#ifdef CAPKERN_SYNCH_POINTS
|
|
cmp.eq pt2 = t9, zero
|
|
(pt2) br.sptk KrqslSkipCollLog
|
|
and t6 = ~7, t4
|
|
CAPSPINLOG2INT(t9,t6,9,t7,t8,t10,t11,pt2)
|
|
KrqslSkipCollLog:
|
|
#endif
|
|
|
|
#endif // !defined(NT_UP)
|
|
|
|
LEAF_LOWER_IRQL_AND_RETURN(a1) // lower IRQL to previous level
|
|
;;
|
|
|
|
#if !defined(NT_UP)
|
|
|
|
//
|
|
// Another processor has inserted its lock queue entry in the lock queue,
|
|
// but has not yet written its lock queue entry address in the current
|
|
// processor's next link. Spin until the lock queue address is written.
|
|
//
|
|
|
|
Krqsl20:
|
|
YIELD
|
|
#ifdef CAPKERN_SYNCH_POINTS
|
|
and t9 = 1, t9
|
|
#endif
|
|
ld8 t3 = [v0] // get next lock queue entry addr
|
|
;;
|
|
cmp.eq pt0 = r0, t3 // if eq, addr not written yet
|
|
add t6 = LqLock, t3
|
|
(pt0) br.sptk Krqsl20 // try again
|
|
;;
|
|
|
|
//
|
|
// Grant the next process in the lock queue ownership of the spinlock.
|
|
// (Turn off the WAIT bit and on the OWNER bit in the next entries lock
|
|
// field).
|
|
//
|
|
|
|
Krqsl30:
|
|
|
|
ld8.nt1 t2 = [t6] // get spinlock addr and lock bit
|
|
;;
|
|
st8 [v0] = r0 // clear next lock queue entry addr
|
|
;;
|
|
xor t2 = (LOCK_QUEUE_OWNER|LOCK_QUEUE_WAIT), t2
|
|
;;
|
|
st8.rel [t6] = t2
|
|
br Krqsl10
|
|
;;
|
|
|
|
#endif // !defined(NT_UP)
|
|
|
|
LEAF_EXIT(KeReleaseQueuedSpinLock)
|
|
|
|
|
|
|
|
SBTTL("Try to Acquire Queued SpinLock and Raise IRQL")
|
|
//++
|
|
//
|
|
// LOGICAL
|
|
// KeTryToAcquireQueuedSpinLock (
|
|
// IN KSPIN_LOCK_QUEUE_NUMBER Number
|
|
// OUT PKIRQL OldIrql
|
|
// )
|
|
//
|
|
// LOGICAL
|
|
// KeTryToAcquireQueuedSpinLockRaiseToSynch (
|
|
// IN KSPIN_LOCK_QUEUE_NUMBER Number
|
|
// OUT PKIRQL OldIrql
|
|
// )
|
|
//
|
|
// LOGICAL
|
|
// KeTryToAcquireQueuedSpinLockAtRaisedIrql (
|
|
// IN PKSPIN_LOCK_QUEUE LockQueue
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// This function raises the current IRQL to synchronization level and
|
|
// attempts to acquire the specified queued spinlock. If the spinlock
|
|
// cannot be acquired, then IRQL is restored and FALSE is returned as
|
|
// the function value. Otherwise, TRUE is returned as the function
|
|
// value.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// Number (a0) - Supplies the queued spinlock number.
|
|
//
|
|
// OldIrql (a1) - Supplies a pointer to a variable that receives the
|
|
// the previous IRQL value.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// If the spin lock is acquired, then a value of TRUE is returned.
|
|
// Otherwise, a value of FALSE is returned.
|
|
//
|
|
//--
|
|
|
|
LEAF_ENTRY(KeTryToAcquireQueuedSpinLock)
|
|
|
|
movl t2 = KiPcr+PcPrcb
|
|
|
|
mov t1 = DISPATCH_LEVEL
|
|
br Kttaqsl10
|
|
;;
|
|
|
|
ALTERNATE_ENTRY(KeTryToAcquireQueuedSpinLockRaiseToSynch)
|
|
|
|
mov t1 = SYNCH_LEVEL
|
|
movl t2 = KiPcr+PcPrcb
|
|
;;
|
|
|
|
Kttaqsl10:
|
|
|
|
#if !defined(NT_UP)
|
|
|
|
rsm 1 << PSR_I // disable interrupts
|
|
ld8 t2 = [t2] // get PRCB address
|
|
;;
|
|
shladd t3 = a0, 4, t2 // get address of lock queue entry
|
|
;;
|
|
add t5 = PbLockQueue, t3
|
|
add t6 = LqLock+PbLockQueue, t3
|
|
;;
|
|
|
|
ld8 t4 = [t6] // get associated spinlock addr
|
|
mov ar.ccv = r0 // cmpxchg oldvalue must be 0
|
|
mov t11 = 0x7
|
|
;;
|
|
andcm t12 = t4, t11
|
|
;;
|
|
|
|
#ifdef CAPKERN_SYNCH_POINTS
|
|
CAPSPINLOG1INT(t12,8,t7,t8,t9,t10,pt0)
|
|
mov ar.ccv = zero /* CAPSPINLOG1INT mutates ar.ccv */
|
|
#endif
|
|
|
|
//
|
|
// Try to acquire the specified spinlock.
|
|
//
|
|
// N.B. A noninterlocked test is done before the interlocked attempt. This
|
|
// allows spinning without interlocked cycles.
|
|
//
|
|
|
|
ld8 t8 = [t12] // get current lock value
|
|
;;
|
|
cmp.ne pt0, pt1 = r0, t8 // if ne, lock owned
|
|
(pt0) br.spnt.few Kttaqs20
|
|
;;
|
|
|
|
cmpxchg8.acq t8 = [t12], t5, ar.ccv // try to acquire the lock
|
|
;;
|
|
|
|
cmp.ne pt0, pt1 = r0, t8 // if ne, lock owned
|
|
or t4 = LOCK_QUEUE_OWNER, t4// set lock owner bit
|
|
(pt0) br.spnt.few Kttaqs20
|
|
;;
|
|
|
|
st8 [t6] = t4
|
|
|
|
#endif
|
|
|
|
SWAP_IRQL(t1)
|
|
|
|
#if !defined(NT_UP)
|
|
|
|
ssm 1 << PSR_I // enable interrupts
|
|
|
|
#ifdef CAPKERN_SYNCH_POINTS
|
|
CAPSPINLOG1INT(t12,1,t7,t8,t9,t10,pt0)
|
|
#endif
|
|
|
|
#endif
|
|
|
|
st1 [a1] = v0 // save old IRQL value
|
|
mov v0 = TRUE // set return value to TRUE
|
|
LEAF_RETURN
|
|
;;
|
|
|
|
#if !defined(NT_UP)
|
|
|
|
//
|
|
// The attempt to acquire the specified spin lock failed. Lower IRQL to its
|
|
// previous value and return FALSE.
|
|
//
|
|
|
|
Kttaqs20:
|
|
ssm 1 << PSR_I // enable interrupts
|
|
mov v0 = FALSE // set return value to FALSE
|
|
YIELD
|
|
LEAF_RETURN
|
|
|
|
#endif
|
|
|
|
LEAF_EXIT(KeTryToAcquireQueuedSpinLock)
|
|
|
|
SBTTL("Try to Acquire Queued SpinLock without raising IRQL")
|
|
//++
|
|
//
|
|
// LOGICAL
|
|
// KeTryToAcquireQueuedSpinLockAtRaisedIrql (
|
|
// IN PKSPIN_LOCK_QUEUE LockQueue
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// This function attempts to acquire the specified queued spinlock.
|
|
// If the spinlock cannot be acquired, then FALSE is returned as
|
|
// the function value. Otherwise, TRUE is returned as the function
|
|
// value.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// LockQueue (a0) - Supplies the address of the queued spinlock.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// If the spin lock is acquired, then a value of TRUE is returned.
|
|
// Otherwise, a value of FALSE is returned.
|
|
//
|
|
//--
|
|
|
|
LEAF_ENTRY(KeTryToAcquireQueuedSpinLockAtRaisedIrql)
|
|
|
|
#if !defined(NT_UP)
|
|
|
|
add t6 = LqLock, a0
|
|
;;
|
|
|
|
ld8 t4 = [t6] // get associated spinlock addr
|
|
mov ar.ccv = r0 // cmpxchg oldvalue must be 0
|
|
mov t11 = 0x7
|
|
;;
|
|
andcm t12 = t4, t11
|
|
;;
|
|
|
|
#ifdef CAPKERN_SYNCH_POINTS
|
|
CAPSPINLOG1INT(t12,8,t7,t8,t9,t10,pt0)
|
|
mov ar.ccv = zero /*CAPSPINLOG1INT mutates ar.ccv*/
|
|
#endif
|
|
|
|
//
|
|
// Try to acquire the specified spinlock.
|
|
//
|
|
// N.B. A noninterlocked test is done before the interlocked attempt. This
|
|
// allows spinning without interlocked cycles.
|
|
//
|
|
|
|
ld8 t8 = [t12] // get current lock value
|
|
mov v0 = FALSE // assume failure
|
|
;;
|
|
cmp.ne pt0, pt1 = r0, t8 // if ne, lock owned
|
|
(pt0) br.ret.spnt.few.clr brp // if owned, return failure
|
|
;;
|
|
|
|
cmpxchg8.acq t8 = [t12], a0, ar.ccv // try to acquire the lock
|
|
;;
|
|
|
|
cmp.ne pt0, pt1 = r0, t8 // if ne, lock owned
|
|
or t4 = LOCK_QUEUE_OWNER, t4// set lock owner bit
|
|
;;
|
|
(pt0) YIELD
|
|
(pt0) br.ret.spnt.few.clr brp // if owned, return failure
|
|
;;
|
|
|
|
st8 [t6] = t4
|
|
|
|
#ifdef CAPKERN_SYNCH_POINTS
|
|
CAPSPINLOG1INT(t12,1,t7,t8,t9,t10,pt0)
|
|
#endif
|
|
|
|
#endif
|
|
|
|
mov v0 = TRUE // set return value to TRUE
|
|
LEAF_RETURN
|
|
;;
|
|
|
|
LEAF_EXIT(KeTryToAcquireQueuedSpinLockAtRaisedIrql)
|
|
|
|
|
|
SBTTL("Acquire Queued SpinLock at Current IRQL")
|
|
|
|
//++
|
|
// VOID
|
|
// KeAcquireInStackQueuedSpinLockAtDpcLevel (
|
|
// IN PKSPIN_LOCK SpinLock,
|
|
// IN PKLOCK_QUEUE_HANDLE LockHandle
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// This function acquires the specified queued spinlock at the current
|
|
// IRQL.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// SpinLock (a0) - Supplies the address of a spin lock.
|
|
//
|
|
// LockHandle (a1) - Supplies the address of an in stack lock handle.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// None.
|
|
//
|
|
//--
|
|
|
|
LEAF_ENTRY(KeAcquireInStackQueuedSpinLockAtDpcLevel)
|
|
|
|
#if !defined(NT_UP)
|
|
|
|
add t0 = LqhNext, a1
|
|
add t1 = LqhLock, a1
|
|
;;
|
|
st8 [t0] = r0
|
|
st8 [t1] = a0
|
|
add a0 = LqhNext, a1
|
|
;;
|
|
|
|
#endif // !defined(NT_UP)
|
|
|
|
//++
|
|
//
|
|
// VOID
|
|
// KeAcquireQueuedSpinLockAtDpcLevel (
|
|
// IN PKSPIN_LOCK_QUEUE LockQueue
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// This function acquires the specified queued spinlock at the current
|
|
// IRQL.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// LockQueue (a0) - Supplies the address of the lock queue entry.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// None.
|
|
//
|
|
//--
|
|
|
|
ALTERNATE_ENTRY(KeAcquireQueuedSpinLockAtDpcLevel)
|
|
|
|
#if !defined(NT_UP)
|
|
add t0 = LqLock, a0
|
|
add t1 = LqNext, a0
|
|
mov t11 = 0x7
|
|
;;
|
|
|
|
ld8 t4 = [t0]
|
|
;;
|
|
mf
|
|
andcm t12 = t4, t11 // mask the lower 3 bits
|
|
;;
|
|
|
|
#ifdef CAPKERN_SYNCH_POINTS
|
|
CAPSPINLOG1INT(t12,1,t5,t6,t7,t8,pt0)
|
|
mov t6 = zero
|
|
#endif
|
|
|
|
xchg8 t3 = [t12], a0
|
|
or t5 = LOCK_QUEUE_OWNER, t4 // set lock owner bit
|
|
;;
|
|
|
|
cmp.ne pt0, pt1 = r0, t3 // if ne, lock already owned
|
|
add t2 = LqNext, t3
|
|
;;
|
|
(pt0) or t5 = LOCK_QUEUE_WAIT, t4
|
|
;;
|
|
|
|
st8.rel [t0] = t5
|
|
(pt0) st8.rel [t2] = a0 // set addr of lock queue entry
|
|
(pt1) br.ret.sptk brp
|
|
;;
|
|
|
|
//
|
|
// The lock is owned by another processor. Set the lock bit in the current
|
|
// processor lock queue entry, set the next link in the previous lock queue
|
|
// entry, and spin on the current processor's lock bit.
|
|
//
|
|
|
|
Kiaqsl10:
|
|
|
|
#ifdef CAPKERN_SYNCH_POINTS
|
|
add t6 = 1, t6
|
|
#endif
|
|
YIELD
|
|
ld8 t4 = [t0] // get lock addr and lock wait bit
|
|
;;
|
|
tbit.z pt1, pt0 = t4, LOG2(LOCK_QUEUE_WAIT)
|
|
(pt0) br.dptk.few Kiaqsl10
|
|
#ifdef CAPKERN_SYNCH_POINTS
|
|
CAPSPINLOG2INT(t6,t12,4,t7,t8,t9,t10,pt2)
|
|
#endif
|
|
(pt1) br.ret.dpnt brp // if zero, lock acquired
|
|
|
|
#else
|
|
br.ret.sptk brp
|
|
#endif // !defined(NT_UP)
|
|
;;
|
|
|
|
LEAF_EXIT(KeAcquireInStackQueuedSpinLockAtDpcLevel)
|
|
|
|
|
|
SBTTL("Release Queued SpinLock at Current IRQL")
|
|
|
|
//++
|
|
//
|
|
// VOID
|
|
// KeReleaseInStackQueuedSpinLockFromDpcLevel (
|
|
// IN PKLOCK_QUEUE_HANDLE LockHandle
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// This function releases a queued spinlock and preserves the current
|
|
// IRQL.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// LockHandle (a0) - Supplies the address of a lock handle.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// None.
|
|
//
|
|
//--
|
|
|
|
LEAF_ENTRY(KeReleaseInStackQueuedSpinLockFromDpcLevel)
|
|
|
|
#if !defined(NT_UP)
|
|
add a0 = LqhNext, a0
|
|
;;
|
|
#endif // !defined(NT_UP)
|
|
|
|
//++
|
|
//
|
|
// VOID
|
|
// KiReleaseQueuedSpinLock (
|
|
// IN PKSPIN_LOCK_QUEUE LockQueue
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// This function releases a queued spinlock and preserves the current
|
|
// IRQL.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// LockQueue (a0) - Supplies the address of the lock queue entry.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// None.
|
|
//
|
|
//--
|
|
|
|
ALTERNATE_ENTRY(KeReleaseQueuedSpinLockFromDpcLevel)
|
|
|
|
#if !defined(NT_UP)
|
|
mov ar.ccv = a0
|
|
add t0 = LqNext, a0
|
|
add t1 = LqLock, a0
|
|
;;
|
|
|
|
ld8 t3 = [t0] // get next lock queue entry addr
|
|
ld8 t4 = [t1] // get associate spin lock addr
|
|
;;
|
|
|
|
#ifdef CAPKERN_SYNCH_POINTS
|
|
and t6 = ~7, t4
|
|
CAPSPINLOG1INT(t6,7,t7,t8,t9,t10,pt0)
|
|
mov ar.ccv = a0 /* CAPSPINLOG1INT mutates ar.ccv */
|
|
mov t9 = zero
|
|
#endif
|
|
|
|
and t4 = ~LOCK_QUEUE_OWNER, t4 // clear lock owner bit
|
|
;;
|
|
|
|
st8.rel [t1] = t4
|
|
cmp.ne pt0, pt1 = r0, t3 // if ne, another processor waiting
|
|
(pt0) br.spnt.few Kirqsl30
|
|
|
|
KiIrqsl10:
|
|
|
|
ld8.nt1 t3 = [t4] // get current lock ownership
|
|
;;
|
|
cmp.ne pt0, pt1 = a0, t3 // if ne, another processor waiting
|
|
(pt0) br.spnt.few Kirqsl20
|
|
;;
|
|
|
|
cmpxchg8.rel t3 = [t4], r0, ar.ccv
|
|
;;
|
|
cmp.ne pt0, pt1 = a0, t3 // if ne, try again
|
|
(pt1) br.ret.sptk brp
|
|
;;
|
|
|
|
//
|
|
// Another processor has inserted its lock queue entry in the lock queue,
|
|
// but has not yet written its lock queue entry address in the current
|
|
// processor's next link. Spin until the lock queue address is written.
|
|
//
|
|
|
|
Kirqsl20:
|
|
YIELD
|
|
#ifdef CAPKERN_SYNCH_POINTS
|
|
add t9 = 1, t9
|
|
#endif
|
|
ld8 t3 = [t0] // get next lock queue entry addr
|
|
;;
|
|
cmp.eq pt0 = r0, t3 // if eq, addr not written yet
|
|
(pt0) br.sptk Kirqsl20 // try again
|
|
|
|
Kirqsl30:
|
|
|
|
add t6 = LqLock, t3
|
|
;;
|
|
ld8.nt1 t2 = [t6] // get spinlock addr and lock bit
|
|
st8 [t0] = r0 // clear next lock queue entry addr
|
|
;;
|
|
// Set owner bit, clear wait bit.
|
|
xor t2 = (LOCK_QUEUE_OWNER|LOCK_QUEUE_WAIT), t2
|
|
;;
|
|
st8.rel [t6] = t2
|
|
|
|
#ifdef CAPKERN_SYNCH_POINTS
|
|
cmp.eq pt2 = t9, zero
|
|
(pt2) br.sptk KirqslSkipCollLog
|
|
and t6 = ~7, t4
|
|
CAPSPINLOG2INT(t9,t6,9,t7,t8,t10,t11,pt2)
|
|
KirqslSkipCollLog:
|
|
#endif
|
|
|
|
#endif // !defined(NT_UP)
|
|
|
|
br.ret.sptk brp
|
|
;;
|
|
|
|
LEAF_EXIT(KeReleaseInStackQueuedSpinLockFromDpcLevel)
|
|
|
|
|
|
//++
|
|
//
|
|
// VOID
|
|
// KiLowerIrqlSoftwareInterruptPending(
|
|
// IN TEMP_REG NewIrql
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// This function is entered directly from a LEAF function that is
|
|
// lowering IRQL before it exits when there is a software interrupt
|
|
// pending that will fire as a result of lowering IRQL.
|
|
//
|
|
// In this special case, we need to promote to a nested entry in
|
|
// order to process the simulated interrupt.
|
|
//
|
|
// Return is directly to the caller of the leaf function.
|
|
//
|
|
// This routine is entered with interrupts disabled, this is a
|
|
// side effect of the code that branched here needing interrupts
|
|
// disabled while checking and lowering.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// NewIrql - Because we are branched to from a leaf routine,
|
|
// the argument must be passed in non-windowed
|
|
// register t22 (r31).
|
|
//
|
|
// Return Value:
|
|
//
|
|
// None.
|
|
//
|
|
//--
|
|
NESTED_ENTRY(KiLowerIrqlSoftwareInterruptPending)
|
|
NESTED_SETUP(0,3,1,0)
|
|
PROLOGUE_END
|
|
mov out0 = t22
|
|
ssm 1 << PSR_I
|
|
;;
|
|
br.call.sptk brp = KiCheckForSoftwareInterrupt;;
|
|
|
|
NESTED_RETURN
|
|
NESTED_EXIT(KiLowerIrqlSoftwareInterruptPending)
|
|
|