//++
//
// Module Name:
//
//    spinlock.s
//
// Abstract:
//
//    This module implements the routines for acquiring and releasing
//    spin locks.
//
// Author:
//
//    William K. Cheung (wcheung) 29-Sep-1995
//
// Environment:
//
//    Kernel mode only.
//
// Revision History:
//
//    31-Dec-1998  wc   Updated to use xchg8
//
//    07-Jul-1997  bl   Updated to EAS2.3
//
//    08-Feb-1996       Updated to EAS2.1
//
//--

#include "ksia64.h"
#include "icecap.h"


        .file       "spinlock.s"

//
// Define LOG2(x) for those values whose bit numbers are needed in
// order to test a single bit with the tbit instruction.
//

#define _LOG2_0x1   0
#define _LOG2_0x2   1
#define _LOG2_x(n)  _LOG2_##n
#define LOG2(n)     _LOG2_x(n)

//
// Globals
//

        PublicFunction(KiCheckForSoftwareInterrupt)


//++
//
// VOID
// KiAcquireSpinLock (
//    IN PKSPIN_LOCK SpinLock
//    )
//
// Routine Description:
//
//    This function acquires a kernel spin lock.
//
//    N.B. This function assumes that the current IRQL is set properly.
//
// Arguments:
//
//    SpinLock (a0) - Supplies a pointer to a kernel spin lock.
//
// Return Value:
//
//    None.
//
//--

        LEAF_ENTRY(KiAcquireSpinLock)

        ALTERNATE_ENTRY(KeAcquireSpinLockAtDpcLevel)

#if !defined(NT_UP)

#ifndef CAPKERN_SYNCH_POINTS
        ACQUIRE_SPINLOCK(a0,a0,Kiasl10)
#else
        CAP_ACQUIRE_SPINLOCK(a0,a0,Kiasl10,t0,t1,t2,t3)
#endif

#endif // !defined(NT_UP)

        LEAF_RETURN

        LEAF_EXIT(KiAcquireSpinLock)

//++
//
// BOOLEAN
// KeTryToAcquireSpinLockAtDpcLevel (
//    IN PKSPIN_LOCK SpinLock
//    )
//
// Routine Description:
//
//    This function attempts to acquires the specified kernel spinlock. If
//    the spinlock can be acquired, then TRUE is returned. Otherwise, FALSE
//    is returned.
//
//    N.B. This function assumes that the current IRQL is set properly.
//
// Arguments:
//
//    SpinLock (a0) - Supplies a pointer to a kernel spin lock.
//
// Return Value:
//
//    If the spin lock is acquired, then a value of TRUE is returned.
//    Otherwise, a value of FALSE is returned.
//
// N.B. The caller KeTryToAcquireSpinLock implicitly depends on the
//      contents of predicate registers pt1 & pt2.
//
//--

        LEAF_ENTRY(KeTryToAcquireSpinLockAtDpcLevel)

#if !defined(NT_UP)

#ifdef CAPKERN_SYNCH_POINTS
        CAPSPINLOG1INT(a0, 8, t0, t1, t2, t3, pt0)
#endif

        xchg8       t0 = [a0], a0
        ;;
        cmp.ne      pt0 = t0, zero              // if ne, lock acq failed
        mov         v0 = TRUE                   // acquire assumed succeed
        ;;
  (pt0) YIELD
  (pt0) mov         v0 = FALSE                  // return FALSE

#ifdef CAPKERN_SYNCH_POINTS
  (pt0) br.cond.spnt kttasladlSkipLog
        CAPSPINLOG1INT(a0, 1, t0, t1, t2, t3, pt0)
kttasladlSkipLog:
#endif


#else
        mov         v0 = TRUE
#endif

        LEAF_RETURN
        LEAF_EXIT(KeTryToAcquireSpinLockAtDpcLevel)


//++
//
// VOID
// KeAcquireSpinLock (
//    IN PKSPIN_LOCK SpinLock
//    OUT PKIRQL OldIrql
//    )
//
// Routine Description:
//
//    This function raises the current IRQL to DISPATCH_LEVEL and acquires
//    the specified executive spinlock.
//
// Arguments:
//
//    SpinLock (a0) - Supplies a pointer to a executive spinlock.
//
//    OldIrql  (a1) - Supplies a pointer to a variable that receives the
//        the previous IRQL value.
//
//    N.B. The Old IRQL MUST be stored after the lock is acquired.
//
// Return Value:
//
//    None.
//
//--

        LEAF_ENTRY(KeAcquireSpinLock)

//
// Get original IRQL, raise IRQL to DISPATCH_LEVEL
// and then acquire the specified spinlock.
//

        mov         t0 = DISPATCH_LEVEL
        SWAP_IRQL(t0)

#if !defined(NT_UP)

#ifndef CAPKERN_SYNCH_POINTS
        ACQUIRE_SPINLOCK(a0,a0,Kasl10)
#else
        CAP_ACQUIRE_SPINLOCK(a0,a0,Kasl10,t1,t2,t3,t4)
#endif

#endif  // !defined(NT_UP)

        st1         [a1] = v0                   // save old IRQL
        LEAF_RETURN

        LEAF_EXIT(KeAcquireSpinLock)



//++
//
// KIRQL
// KeAcquireSpinLockRaiseToSynch (
//    IN PKSPIN_LOCK SpinLock
//    )
//
// Routine Description:
//
//    This function raises the current IRQL to synchronization level and
//    acquires the specified spinlock.
//
// Arguments:
//
//    SpinLock (a0) - Supplies a pointer to the spinlock that is to be
//        acquired.
//
// Return Value:
//
//    The previous IRQL is returned as the function value.
//
//--

        LEAF_ENTRY(KeAcquireSpinLockRaiseToSynch)

//
// Register aliases
//

        pHeld       = pt0
        pFree       = pt1

        mov         t1 = SYNCH_LEVEL

#if !defined(NT_UP)

#ifdef CAPKERN_SYNCH_POINTS
        CAPSPINLOG1INT(a0,1,t3,t4,t5,t6,pt2)
        mov         t2 = zero
#endif


        GET_IRQL    (v0)

KaslrtsRetry:

        SET_IRQL    (t1)
        xchg8       t0 = [a0], a0
        ;;
        cmp.eq      pFree, pHeld = 0, t0
        ;;
        PSET_IRQL   (pHeld, v0)

#ifdef CAPKERN_SYNCH_POINTS

(pHeld) br.cond.dpnt KaslrtsSkipCollLog
        cmp.eq pt2 = t2, zero
(pt2)   br.cond.sptk  KaslrtsSkipCollLog
        CAPSPINLOG2INT(t2,a0,2,t3,t4,t5,t6,pt2)

KaslrtsSkipCollLog:

#endif //CAPKERN_SYNCH_POINTS

(pFree) LEAF_RETURN
        ;;

KaslrtsLoop:

        YIELD

#ifdef CAPKERN_SYNCH_POINTS
        add t2 = 1, t2
#endif
        cmp.eq      pFree, pHeld = 0, t0
        ld8.nt1     t0 = [a0]
(pFree) br.cond.dpnt  KaslrtsRetry
(pHeld) br.cond.dptk  KaslrtsLoop


#else

        SWAP_IRQL   (t1)                        // Raise IRQL
        LEAF_RETURN

#endif // !defined(NT_UP)

        LEAF_EXIT(KeAcquireSpinLockRaiseToSynch)


//++
//
// KIRQL
// KeAcquireSpinLockRaiseToDpc (
//    IN PKSPIN_LOCK SpinLock
//    )
//
// Routine Description:
//
//    This function raises the current IRQL to dispatcher level and acquires
//    the specified spinlock.
//
// Arguments:
//
//    SpinLock (a0) - Supplies a pointer to the spinlock that is to be
//        acquired.
//
// Return Value:
//
//    The previous IRQL is returned as the function value.
//
//--


        LEAF_ENTRY(KeAcquireSpinLockRaiseToDpc)

        mov         t2 = DISPATCH_LEVEL
        ;;
        SWAP_IRQL   (t2)

#if !defined(NT_UP)

        cmp.eq      pt0, pt1 = zero, zero
        cmp.eq      pt2, pt3 = zero, zero

#ifdef CAPKERN_SYNCH_POINTS
        CAPSPINLOG1INT(a0, 1, t3, t4, t5, t6, pt4)
        mov t1 = zero
#endif
        ;;

Kaslrtp10:
.pred.rel "mutex",pt0,pt1
(pt0)   xchg8       t0 = [a0], a0
(pt1)   ld8.nt1     t0 = [a0]

        ;;
(pt0)   cmp.ne      pt2, pt3 = zero, t0
        cmp.eq      pt0, pt1 = zero, t0
        ;;
(pt1)   YIELD
#ifdef CAPKERN_SYNCH_POINTS
(pt2)   add t1 = 1, t1
#endif
(pt2)   br.dpnt     Kaslrtp10

#ifdef CAPKERN_SYNCH_POINTS
        cmp.eq      pt1 = t1, zero
(pt1)   br.sptk     KaslrtpSkipCollLog
        CAPSPINLOG2INT(t1,a0,2,t3,t4,t5,t6,pt1)
KaslrtpSkipCollLog:
#endif

(pt3)   br.ret.dptk brp

#else

        LEAF_RETURN

#endif // !defined(NT_UP)

        LEAF_EXIT(KeAcquireSpinLockRaiseToDpc)



//++
//
// VOID
// KiReleaseSpinLock (
//    IN PKSPIN_LOCK SpinLock
//    )
//
// Routine Description:
//
//    This function releases a kernel spin lock.
//
//    N.B. This function assumes that the current IRQL is set properly.
//
// Arguments:
//
//    SpinLock (a0) - Supplies a pointer to an executive spin lock.
//
// Return Value:
//
//    None.
//
//--

        LEAF_ENTRY(KiReleaseSpinLock)
        ALTERNATE_ENTRY(KeReleaseSpinLockFromDpcLevel)

#if !defined(NT_UP)
        st8.rel     [a0] = zero             // set spin lock not owned

#ifdef CAPKERN_SYNCH_POINTS
        CAPSPINLOG1INT(a0, 7, t0, t1, t2, t3, pt0)
#endif

#endif

        LEAF_RETURN
        LEAF_EXIT(KiReleaseSpinLock)


//++
//
// VOID
// KeReleaseSpinLock (
//    IN PKSPIN_LOCK SpinLock
//    IN KIRQL OldIrql
//    )
//
// Routine Description:
//
//    This function releases an executive spin lock and lowers the IRQL
//    to its previous value.  Called at DPC_LEVEL.
//
// Arguments:
//
//    SpinLock (a0) - Supplies a pointer to an executive spin lock.
//
//    OldIrql (a1) - Supplies the previous IRQL value.
//
// Return Value:
//
//    None.
//
//--


        LEAF_ENTRY(KeReleaseSpinLock)

        zxt1        a1 = a1

#if !defined(NT_UP)
        st8.rel     [a0] = zero                 // set spinlock not owned

#ifdef CAPKERN_SYNCH_POINTS
        CAPSPINLOG1INT(a0, 7, t0, t1, t2, t3, pt0)
#endif

#endif

        ;;
        LEAF_LOWER_IRQL_AND_RETURN(a1)          // Lower IRQL and return

        LEAF_EXIT(KeReleaseSpinLock)

//++
//
// BOOLEAN
// KeTryToAcquireSpinLock (
//    IN PKSPIN_LOCK SpinLock
//    OUT PKIRQL OldIrql
//    )
//
// Routine Description:
//
//    This function raises the current IRQL to DISPATCH_LEVEL and attempts
//    to acquires the specified executive spinlock. If the spinlock can be
//    acquired, then TRUE is returned. Otherwise, the IRQL is restored to
//    its previous value and FALSE is returned. Called at IRQL <= DISPATCH_LEVEL.
//
// Arguments:
//
//    SpinLock (a0) - Supplies a pointer to a executive spinlock.
//
//    OldIrql  (a1) - Supplies a pointer to a variable that receives the
//        the previous IRQL value.
//
// Return Value:
//
//    If the spin lock is acquired, then a value of TRUE is returned.
//    Otherwise, a value of FALSE is returned.
//
// N.B. This routine assumes KeTryToAcquireSpinLockAtDpcLevel pt1 & pt2 will
//      be set to reflect the result of the attempt to acquire the spinlock.
//
//--

        LEAF_ENTRY(KeTryToAcquireSpinLock)

#ifdef CAPKERN_SYNCH_POINTS
        CAPSPINLOG1INT(a0, 8, t0, t1, t2, t3, pt2)
#endif

        rOldIrql    = t2

//
// Raise IRQL to DISPATCH_LEVEL and try to acquire the specified spinlock.
// Return FALSE if failed; otherwise, return TRUE.
//

        GET_IRQL    (rOldIrql)                  // get original IRQL
        mov         t0 = DISPATCH_LEVEL;;
        SET_IRQL    (t0)                        // raise to dispatch level

#if !defined(NT_UP)

        xchg8       t0 = [a0], a0
        ;;
        cmp.ne      pt2 = t0, zero              // if ne, lock acq failed
        ;;


//
// If successfully acquired, pt1 is set to TRUE while pt2 is set to FALSE.
// Otherwise, pt2 is set to TRUE while pt1 is set to FALSE.
//

  (pt2) YIELD
  (pt2) mov         v0 = FALSE                  // return FALSE
        PSET_IRQL   (pt2, rOldIrql)             // restore old IRQL

#ifdef CAPKERN_SYNCH_POINTS

  (pt2) br.cond.dpnt KttasSkipLog
  CAPSPINLOG1INT(a0, 1, t0, t1, t2, t3, pt0)

KttasSkipLog:
#endif

  (pt2) LEAF_RETURN
        ;;

#endif // !defined(NT_UP)

        st1         [a1] = rOldIrql             // save old IRQL
        mov         v0 = TRUE                   // successfully acquired
        LEAF_RETURN

        LEAF_EXIT(KeTryToAcquireSpinLock)

//++
//
// BOOLEAN
// KeTestSpinLock (
//    IN PKSPIN_LOCK SpinLock
//    )
//
// Routine Description:
//
//    This function tests a kernel spin lock.  If the spinlock is
//    busy, FALSE is returned.  If not, TRUE is returned.  The spinlock
//    is never acquired.  This is provided to allow code to spin at low
//    IRQL, only raising the IRQL when there is a reasonable hope of
//    acquiring the lock.
//
// Arguments:
//
//    SpinLock (a0) - Supplies a pointer to a kernel spin lock.
//
// Return Value:
//
//    TRUE  - Spinlock appears available
//    FALSE - SpinLock is busy
//--

        LEAF_ENTRY(KeTestSpinLock)

        ld8.nt1    t0 = [a0]
        ;;
        cmp.ne     pt0 = 0, t0
        mov        v0 = TRUE                        // default TRUE
        ;;

 (pt0)  YIELD
 (pt0)  mov        v0 = FALSE                       // if t0 != 0 return FALSE
        LEAF_RETURN

        LEAF_EXIT(KeTestSpinLock)

        SBTTL("Acquire Queued SpinLock and Raise IRQL")
//++
//
// VOID
// KeAcquireInStackQueuedSpinLock (
//    IN PKSPIN_LOCK SpinLock,
//    IN PKLOCK_QUEUE_HANDLE LockHandle
//    )
//
// Routine Description:
//
//    This function raises the current IRQL to dispatch level and
//    acquires the specified queued spinlock.
//
// Arguments:
//
//    SpinLock (a0) - Supplies a pointer to a spin lock.
//
//    LockHandle (a1) - Supplies a pointer to a lock handle.
//
// Return Value:
//
//    None.
//
//--
        LEAF_ENTRY(KeAcquireInStackQueuedSpinLockRaiseToSynch)

        add        t5 = LqhNext, a1
        mov        t1 = SYNCH_LEVEL
        br.sptk    Kaisqsl10
        ;;


        ALTERNATE_ENTRY(KeAcquireInStackQueuedSpinLock)

        add        t5 = LqhNext, a1
        mov        t1 = DISPATCH_LEVEL
        ;;

Kaisqsl10:
#if !defined(NT_UP)

        st8        [t5] = zero               // set next link to NULL
        add        t4 = LqhLock, a1
        ;;

        st8.rel    [t4] = a0                 // set spin lock address

#endif // !defined(NT_UP)

        SWAP_IRQL (t1)                       // old IRQL in register v0
        add        t0 = LqhOldIrql, a1
        ;;

        st1        [t0] = v0                 // save old IRQL

#if !defined(NT_UP)

        //
        // Finish in common code.   The following register values
        // are assumed in that code.
        //
        // t4 = &LockEntry->ActualLock
        // t5 = LockEntry
        // a0 = *(&LockEntry->ActualLock)
        //
        // Note: LqhNext == 0, otherwise we would have to trim t5 here.
        //

        br         KxqAcquireQueuedSpinLock  // finish in common code

#else

        br.ret.sptk brp

#endif !defined(NT_UP)

        LEAF_EXIT(KeAcquireInStackQueuedSpinLock)


        SBTTL("Acquire Queued SpinLock and Raise IRQL")
//++
//
// KIRQL
// KeAcquireQueuedSpinLock (
//    IN KSPIN_LOCK_QUEUE_NUMBER Number
//    )
//
// KIRQL
// KeAcquireQueuedSpinLockRaiseToSynch (
//    IN KSPIN_LOCK_QUEUE_NUMBER Number
//    )
//
// Routine Description:
//
//    This function raises the current IRQL to synchronization level and
//    acquires the specified queued spinlock.
//
// Arguments:
//
//    Number (a0) - Supplies the queued spinlock number.
//
// Return Value:
//
//    The previous IRQL is returned as the function value.
//
//--

        LEAF_ENTRY(KeAcquireQueuedSpinLock)

        add        t0 = a0, a0
        mov        t1 = DISPATCH_LEVEL
        br         Kaqsl10
        ;;

        ALTERNATE_ENTRY(KeAcquireQueuedSpinLockRaiseToSynch)

        add        t0 = a0, a0
        mov        t1 = SYNCH_LEVEL
        ;;

Kaqsl10:

        SWAP_IRQL (t1)                      // old IRQL in register v0

#if !defined(NT_UP)
        movl       t2 = KiPcr+PcPrcb
        ;;
        ld8        t2 = [t2]
        ;;
        shladd     t3 = t0, 3, t2           // get associated spinlock addr
        ;;
        add        t5 = PbLockQueue, t3
        ;;

        add        t4 = LqLock, t5
        ;;
        ld8        t6 = [t4]
        mov        t11 = 0x7
        ;;

        andcm      a0 = t6, t11             // mask the lower 3 bits
        ;;

        ALTERNATE_ENTRY(KxqAcquireQueuedSpinLock)

#ifdef CAPKERN_SYNCH_POINTS        
        CAPSPINLOG1INT(a0,1,t6,t7,t8,t9,pt1)
        mov     t12 = zero
#endif

        mf         // Do a memory fence to ensure the write of LqhNext 
                   // occurs before the xchg8 on lock address.
                   
        //
        // t4 = &LockEntry->ActualLock
        // t5 = LockEntry
        // a0 = *(&LockEntry->ActualLock)
        //

        xchg8      t7 = [a0], t5
        ;;
        cmp.ne     pt0, pt1 = r0, t7        // if ne, lock already owned
        ;;
 (pt1)  or         t8 = LOCK_QUEUE_OWNER, a0
 (pt0)  or         t8 = LOCK_QUEUE_WAIT, a0
        ;;

        st8.rel    [t4] = t8
        add        t9 = LqNext, t7
 (pt1)  br.ret.sptk brp
        ;;

        //
        // The lock is already held by another processor.  Set the wait
        // bit in this processor's Lock Queue entry, then set the next
        // field in the Lock Queue entry of the last processor to attempt
        // to acquire the lock (this is the address returned by the xchg
        // above) to point to THIS processor's lock queue entry.
        //

        st8.rel    [t9] = t5

Kaqsl20:
#ifdef CAPKERN_SYNCH_POINTS
        add t12 = 1, t12
#endif
        YIELD        
        ld8        t10 = [t4]
        ;;
        tbit.z     pt1, pt0 = t10, LOG2(LOCK_QUEUE_WAIT)
 (pt0)  br.dptk.few Kaqsl20
#ifdef CAPKERN_SYNCH_POINTS
        CAPSPINLOG2INT(t12,a0,4,t6,t7,t8,t9,pt2)
#endif
 (pt1)  br.ret.dpnt brp                    // if zero, lock acquired

#else
        br.ret.sptk brp
#endif // !defined(NT_UP)
        ;;

        LEAF_EXIT(KeAcquireQueuedSpinLock)

        SBTTL("Release Queued SpinLock and Lower IRQL")
//++
//
// VOID
// KeReleaseInStackQueuedSpinLock (
//    IN PKLOCK_QUEUE_HANDLE LockHandle
//    )
//
// Routine Description:
//
//    This function releases a queued spinlock and lowers the IRQL to its
//    previous value.
//
// Arguments:
//
//    LockHandle (a0) - Supplies a pointer to a lock handle.
//
// Return Value:
//
//    None.
//
//--

        LEAF_ENTRY(KeReleaseInStackQueuedSpinLock)

        alloc      t22 = ar.pfs, 2, 2, 2, 0
        add        t9 = LqhOldIrql, a0
        add        t5 = LqhNext, a0              // set address of lock queue
        ;;

        ld1.nt1    a1 = [t9]                     // get old IRQL
        br         KxqReleaseQueuedSpinLock      // finish in common code

        LEAF_EXIT(KeReleaseInStackQueuedSpinLock)


        SBTTL("Release Queued SpinLock and Lower IRQL")
//++
//
// VOID
// KeReleaseQueuedSpinLock (
//    IN KSPIN_LOCK_QUEUE_NUMBER Number,
//    IN KIRQL OldIrql
//    )
//
// Routine Description:
//
//    This function releases a queued spinlock and lowers the IRQL to its
//    previous value.
//
// Arguments:
//
//    Number (a0) - Supplies the queued spinlock number.
//
//    OldIrql (a1) - Supplies the previous IRQL value.
//
// Return Value:
//
//    None.
//
//--


        LEAF_ENTRY(KeReleaseQueuedSpinLock)
        PROLOGUE_BEGIN

#if !defined(NT_UP)
        movl        v0 = KiPcr+PcPrcb
        ;;

        ld8         v0 = [v0]
        add         t0 = a0, a0
        ;;
        shladd      t1 = t0, 3, v0
        ;;

        add         t5 = PbLockQueue, t1
        ;;
#endif // !defined(NT_UP)

        ALTERNATE_ENTRY(KxqReleaseQueuedSpinLock)

#if !defined(NT_UP)
        add         v0 = LqNext, t5
        add         t2 = LqLock, t5
        ;;

        ld8.acq     t4 = [t2]
        mov         ar.ccv = t5

        ld8         t3 = [v0]
        ;;
#ifdef CAPKERN_SYNCH_POINTS
        and         t6 = ~7, t4
        CAPSPINLOG1INT(t6,7,t7,t8,t9,t10,pt0)
        mov ar.ccv = t5                         /* CAPSPINLOG1INT mutates ar.ccv */
        mov t9     = zero
#endif
        and         t4 = ~LOCK_QUEUE_OWNER, t4   // clear lock owner bit
        ;;
        add         t6 = LqLock, t3

        st8.rel     [t2] = t4
        cmp.ne      pt0, pt1 = r0, t3       // if ne, another processor waiting
(pt0)   br.sptk.few Krqsl30

        ld8         t7 = [t4]
        ;;
        cmp.ne      pt2 = t5, t7
(pt2)   br.spnt.few Krqsl20

        cmpxchg8.rel t8 = [t4], r0, ar.ccv
        ;;
        cmp.ne      pt0, pt1 = t8, t5       // if ne, another processor waiting
(pt0)   br.spnt.few Krqsl20
        ;;

Krqsl10:

#ifdef CAPKERN_SYNCH_POINTS
        cmp.eq      pt2 = t9, zero
(pt2)   br.sptk     KrqslSkipCollLog
        and         t6 = ~7, t4
        CAPSPINLOG2INT(t9,t6,9,t7,t8,t10,t11,pt2)
KrqslSkipCollLog:
#endif

#endif // !defined(NT_UP)

        LEAF_LOWER_IRQL_AND_RETURN(a1)      // lower IRQL to previous level
        ;;

#if !defined(NT_UP)

//
// Another processor has inserted its lock queue entry in the lock queue,
// but has not yet written its lock queue entry address in the current
// processor's next link. Spin until the lock queue address is written.
//

Krqsl20:
        YIELD
#ifdef CAPKERN_SYNCH_POINTS
        and         t9 = 1, t9
#endif
        ld8         t3 = [v0]               // get next lock queue entry addr
        ;;
        cmp.eq      pt0 = r0, t3            // if eq, addr not written yet
        add         t6 = LqLock, t3
 (pt0)  br.sptk     Krqsl20                 // try again
        ;;

//
// Grant the next process in the lock queue ownership of the spinlock.
// (Turn off the WAIT bit and on the OWNER bit in the next entries lock
// field).
//

Krqsl30:

        ld8.nt1     t2 = [t6]               // get spinlock addr and lock bit
        ;;
        st8         [v0] = r0               // clear next lock queue entry addr
        ;;
        xor         t2 = (LOCK_QUEUE_OWNER|LOCK_QUEUE_WAIT), t2
        ;;
        st8.rel     [t6] = t2
        br          Krqsl10
        ;;

#endif // !defined(NT_UP)

        LEAF_EXIT(KeReleaseQueuedSpinLock)



        SBTTL("Try to Acquire Queued SpinLock and Raise IRQL")
//++
//
// LOGICAL
// KeTryToAcquireQueuedSpinLock (
//    IN KSPIN_LOCK_QUEUE_NUMBER Number
//    OUT PKIRQL OldIrql
//    )
//
// LOGICAL
// KeTryToAcquireQueuedSpinLockRaiseToSynch (
//    IN KSPIN_LOCK_QUEUE_NUMBER Number
//    OUT PKIRQL OldIrql
//    )
//
// LOGICAL
// KeTryToAcquireQueuedSpinLockAtRaisedIrql (
//    IN PKSPIN_LOCK_QUEUE LockQueue
//    )
//
// Routine Description:
//
//    This function raises the current IRQL to synchronization level and
//    attempts to acquire the specified queued spinlock. If the spinlock
//    cannot be acquired, then IRQL is restored and FALSE is returned as
//    the function value. Otherwise, TRUE is returned as the function
//    value.
//
// Arguments:
//
//    Number (a0) - Supplies the queued spinlock number.
//
//    OldIrql  (a1) - Supplies a pointer to a variable that receives the
//        the previous IRQL value.
//
// Return Value:
//
//    If the spin lock is acquired, then a value of TRUE is returned.
//    Otherwise, a value of FALSE is returned.
//
//--

        LEAF_ENTRY(KeTryToAcquireQueuedSpinLock)

        movl       t2 = KiPcr+PcPrcb

        mov        t1 = DISPATCH_LEVEL
        br         Kttaqsl10
        ;;

        ALTERNATE_ENTRY(KeTryToAcquireQueuedSpinLockRaiseToSynch)

        mov        t1 = SYNCH_LEVEL
        movl       t2 = KiPcr+PcPrcb
        ;;

Kttaqsl10:

#if !defined(NT_UP)

        rsm        1 << PSR_I               // disable interrupts
        ld8        t2 = [t2]                // get PRCB address
        ;;
        shladd     t3 = a0, 4, t2           // get address of lock queue entry
        ;;
        add        t5 = PbLockQueue, t3
        add        t6 = LqLock+PbLockQueue, t3
        ;;

        ld8        t4 = [t6]                // get associated spinlock addr
        mov        ar.ccv = r0              // cmpxchg oldvalue must be 0
        mov        t11 = 0x7
        ;;
        andcm      t12 = t4, t11
        ;;

#ifdef CAPKERN_SYNCH_POINTS        
        CAPSPINLOG1INT(t12,8,t7,t8,t9,t10,pt0)
        mov        ar.ccv = zero                  /* CAPSPINLOG1INT mutates ar.ccv */
#endif

//
// Try to acquire the specified spinlock.
//
// N.B. A noninterlocked test is done before the interlocked attempt. This
//      allows spinning without interlocked cycles.
//

        ld8        t8 = [t12]               // get current lock value
        ;;
        cmp.ne     pt0, pt1 = r0, t8        // if ne, lock owned
 (pt0)  br.spnt.few Kttaqs20
        ;;

        cmpxchg8.acq t8 = [t12], t5, ar.ccv // try to acquire the lock
        ;;

        cmp.ne     pt0, pt1 = r0, t8        // if ne, lock owned
        or         t4 = LOCK_QUEUE_OWNER, t4// set lock owner bit
 (pt0)  br.spnt.few Kttaqs20
        ;;

        st8        [t6] = t4

#endif

        SWAP_IRQL(t1)

#if !defined(NT_UP)

        ssm         1 << PSR_I              // enable interrupts

#ifdef CAPKERN_SYNCH_POINTS        
        CAPSPINLOG1INT(t12,1,t7,t8,t9,t10,pt0)
#endif

#endif

        st1        [a1] = v0                // save old IRQL value
        mov        v0 = TRUE                // set return value to TRUE
        LEAF_RETURN
        ;;

#if !defined(NT_UP)

//
// The attempt to acquire the specified spin lock failed. Lower IRQL to its
// previous value and return FALSE.
//

Kttaqs20:
        ssm         1 << PSR_I              // enable interrupts
        mov         v0 = FALSE              // set return value to FALSE
        YIELD
        LEAF_RETURN

#endif

        LEAF_EXIT(KeTryToAcquireQueuedSpinLock)

        SBTTL("Try to Acquire Queued SpinLock without raising IRQL")
//++
//
// LOGICAL
// KeTryToAcquireQueuedSpinLockAtRaisedIrql (
//    IN PKSPIN_LOCK_QUEUE LockQueue
//    )
//
// Routine Description:
//
//    This function attempts to acquire the specified queued spinlock.
//    If the spinlock cannot be acquired, then FALSE is returned as
//    the function value. Otherwise, TRUE is returned as the function
//    value.
//
// Arguments:
//
//    LockQueue (a0) - Supplies the address of the queued spinlock.
//
// Return Value:
//
//    If the spin lock is acquired, then a value of TRUE is returned.
//    Otherwise, a value of FALSE is returned.
//
//--

        LEAF_ENTRY(KeTryToAcquireQueuedSpinLockAtRaisedIrql)

#if !defined(NT_UP)

        add        t6 = LqLock, a0
        ;;

        ld8        t4 = [t6]                // get associated spinlock addr
        mov        ar.ccv = r0              // cmpxchg oldvalue must be 0
        mov        t11 = 0x7
        ;;
        andcm      t12 = t4, t11
        ;;

#ifdef CAPKERN_SYNCH_POINTS        
        CAPSPINLOG1INT(t12,8,t7,t8,t9,t10,pt0)
        mov        ar.ccv = zero                  /*CAPSPINLOG1INT mutates ar.ccv*/
#endif

//
// Try to acquire the specified spinlock.
//
// N.B. A noninterlocked test is done before the interlocked attempt. This
//      allows spinning without interlocked cycles.
//

        ld8        t8 = [t12]               // get current lock value
        mov        v0 = FALSE               // assume failure
        ;;
        cmp.ne     pt0, pt1 = r0, t8        // if ne, lock owned
 (pt0)  br.ret.spnt.few.clr brp             // if owned, return failure
        ;;

        cmpxchg8.acq t8 = [t12], a0, ar.ccv // try to acquire the lock
        ;;

        cmp.ne     pt0, pt1 = r0, t8        // if ne, lock owned
        or         t4 = LOCK_QUEUE_OWNER, t4// set lock owner bit
        ;;
 (pt0)  YIELD        
 (pt0)  br.ret.spnt.few.clr brp             // if owned, return failure
        ;;

        st8        [t6] = t4

#ifdef CAPKERN_SYNCH_POINTS        
        CAPSPINLOG1INT(t12,1,t7,t8,t9,t10,pt0)
#endif

#endif

        mov        v0 = TRUE                // set return value to TRUE
        LEAF_RETURN
        ;;

        LEAF_EXIT(KeTryToAcquireQueuedSpinLockAtRaisedIrql)


        SBTTL("Acquire Queued SpinLock at Current IRQL")

//++
// VOID
// KeAcquireInStackQueuedSpinLockAtDpcLevel (
//    IN PKSPIN_LOCK SpinLock,
//    IN PKLOCK_QUEUE_HANDLE LockHandle
//    )
//
// Routine Description:
//
//    This function acquires the specified queued spinlock at the current
//    IRQL.
//
// Arguments:
//
//    SpinLock (a0) - Supplies the address of a spin lock.
//
//    LockHandle (a1) - Supplies the address of an in stack lock handle.
//
// Return Value:
//
//    None.
//
//--

        LEAF_ENTRY(KeAcquireInStackQueuedSpinLockAtDpcLevel)

#if !defined(NT_UP)

        add         t0 = LqhNext, a1
        add         t1 = LqhLock, a1
        ;;
        st8         [t0] = r0
        st8         [t1] = a0
        add         a0 = LqhNext, a1
        ;;

#endif // !defined(NT_UP)

//++
//
// VOID
// KeAcquireQueuedSpinLockAtDpcLevel (
//    IN PKSPIN_LOCK_QUEUE LockQueue
//    )
//
// Routine Description:
//
//    This function acquires the specified queued spinlock at the current
//    IRQL.
//
// Arguments:
//
//    LockQueue (a0) - Supplies the address of the lock queue entry.
//
// Return Value:
//
//    None.
//
//--

        ALTERNATE_ENTRY(KeAcquireQueuedSpinLockAtDpcLevel)

#if !defined(NT_UP)
        add         t0 = LqLock, a0
        add         t1 = LqNext, a0
        mov         t11 = 0x7
        ;;

        ld8         t4 = [t0]
        ;;
        mf
        andcm       t12 = t4, t11             // mask the lower 3 bits
        ;;

#ifdef CAPKERN_SYNCH_POINTS        
        CAPSPINLOG1INT(t12,1,t5,t6,t7,t8,pt0)
        mov     t6 = zero
#endif

        xchg8       t3 = [t12], a0
        or          t5 = LOCK_QUEUE_OWNER, t4 // set lock owner bit
        ;;

        cmp.ne      pt0, pt1 = r0, t3         // if ne, lock already owned
        add         t2 = LqNext, t3
        ;;
 (pt0)  or          t5 = LOCK_QUEUE_WAIT, t4
        ;;

        st8.rel     [t0] = t5
 (pt0)  st8.rel     [t2] = a0                 // set addr of lock queue entry
 (pt1)  br.ret.sptk brp
        ;;

//
// The lock is owned by another processor. Set the lock bit in the current
// processor lock queue entry, set the next link in the previous lock queue
// entry, and spin on the current processor's lock bit.
//

Kiaqsl10:

#ifdef CAPKERN_SYNCH_POINTS
        add t6 = 1, t6
#endif
        YIELD
        ld8         t4 = [t0]                 // get lock addr and lock wait bit
        ;;
        tbit.z      pt1, pt0 = t4, LOG2(LOCK_QUEUE_WAIT)
 (pt0)  br.dptk.few Kiaqsl10
#ifdef CAPKERN_SYNCH_POINTS
        CAPSPINLOG2INT(t6,t12,4,t7,t8,t9,t10,pt2)
#endif
 (pt1)  br.ret.dpnt brp                       // if zero, lock acquired

#else
        br.ret.sptk brp
#endif // !defined(NT_UP)
        ;;

        LEAF_EXIT(KeAcquireInStackQueuedSpinLockAtDpcLevel)


        SBTTL("Release Queued SpinLock at Current IRQL")

//++
//
// VOID
// KeReleaseInStackQueuedSpinLockFromDpcLevel (
//    IN PKLOCK_QUEUE_HANDLE LockHandle
//    )
//
// Routine Description:
//
//    This function releases a queued spinlock and preserves the current
//    IRQL.
//
// Arguments:
//
//    LockHandle (a0) - Supplies the address of a lock handle.
//
// Return Value:
//
//    None.
//
//--

        LEAF_ENTRY(KeReleaseInStackQueuedSpinLockFromDpcLevel)

#if !defined(NT_UP)
        add         a0 = LqhNext, a0
        ;;
#endif // !defined(NT_UP)

//++
//
// VOID
// KiReleaseQueuedSpinLock (
//    IN PKSPIN_LOCK_QUEUE LockQueue
//    )
//
// Routine Description:
//
//    This function releases a queued spinlock and preserves the current
//    IRQL.
//
// Arguments:
//
//    LockQueue (a0) - Supplies the address of the lock queue entry.
//
// Return Value:
//
//    None.
//
//--

        ALTERNATE_ENTRY(KeReleaseQueuedSpinLockFromDpcLevel)

#if !defined(NT_UP)
        mov         ar.ccv = a0
        add         t0 = LqNext, a0
        add         t1 = LqLock, a0
        ;;

        ld8         t3 = [t0]                  // get next lock queue entry addr
        ld8         t4 = [t1]                  // get associate spin lock addr
        ;;

#ifdef CAPKERN_SYNCH_POINTS
        and         t6 = ~7, t4
        CAPSPINLOG1INT(t6,7,t7,t8,t9,t10,pt0)
        mov         ar.ccv = a0                   /* CAPSPINLOG1INT mutates ar.ccv */
        mov         t9     = zero
#endif

        and         t4 = ~LOCK_QUEUE_OWNER, t4 // clear lock owner bit
        ;;

        st8.rel     [t1] = t4
        cmp.ne      pt0, pt1 = r0, t3    // if ne, another processor waiting
 (pt0)  br.spnt.few Kirqsl30

KiIrqsl10:

        ld8.nt1     t3 = [t4]            // get current lock ownership
        ;;
        cmp.ne      pt0, pt1 = a0, t3    // if ne, another processor waiting
 (pt0)  br.spnt.few Kirqsl20
        ;;

        cmpxchg8.rel t3 = [t4], r0, ar.ccv
        ;;
        cmp.ne      pt0, pt1 = a0, t3    // if ne, try again
 (pt1)  br.ret.sptk brp
        ;;

//
// Another processor has inserted its lock queue entry in the lock queue,
// but has not yet written its lock queue entry address in the current
// processor's next link. Spin until the lock queue address is written.
//

Kirqsl20:
        YIELD
#ifdef CAPKERN_SYNCH_POINTS
        add         t9 = 1, t9
#endif
        ld8         t3 = [t0]               // get next lock queue entry addr
        ;;
        cmp.eq      pt0 = r0, t3            // if eq, addr not written yet
 (pt0)  br.sptk     Kirqsl20                // try again

Kirqsl30:

        add         t6 = LqLock, t3
        ;;
        ld8.nt1     t2 = [t6]               // get spinlock addr and lock bit
        st8         [t0] = r0               // clear next lock queue entry addr
        ;;
        // Set owner bit, clear wait bit.
        xor         t2 = (LOCK_QUEUE_OWNER|LOCK_QUEUE_WAIT), t2
        ;;
        st8.rel     [t6] = t2

#ifdef CAPKERN_SYNCH_POINTS
        cmp.eq      pt2 = t9, zero
(pt2)   br.sptk     KirqslSkipCollLog
        and         t6 = ~7, t4
        CAPSPINLOG2INT(t9,t6,9,t7,t8,t10,t11,pt2)
KirqslSkipCollLog:
#endif

#endif // !defined(NT_UP)

        br.ret.sptk brp
        ;;

        LEAF_EXIT(KeReleaseInStackQueuedSpinLockFromDpcLevel)


//++
//
// VOID
// KiLowerIrqlSoftwareInterruptPending(
//   IN TEMP_REG NewIrql
//   )
//
// Routine Description:
//
//   This function is entered directly from a LEAF function that is
//   lowering IRQL before it exits when there is a software interrupt
//   pending that will fire as a result of lowering IRQL.
//
//   In this special case, we need to promote to a nested entry in
//   order to process the simulated interrupt.
//
//   Return is directly to the caller of the leaf function.
//
//   This routine is entered with interrupts disabled, this is a
//   side effect of the code that branched here needing interrupts
//   disabled while checking and lowering.
//
// Arguments:
//
//   NewIrql    - Because we are branched to from a leaf routine,
//                the argument must be passed in non-windowed
//                register t22 (r31).
//
// Return Value:
//
//   None.
//
//--
        NESTED_ENTRY(KiLowerIrqlSoftwareInterruptPending)
        NESTED_SETUP(0,3,1,0)
        PROLOGUE_END
        mov         out0 = t22
        ssm         1 << PSR_I
        ;;
        br.call.sptk brp = KiCheckForSoftwareInterrupt;;

        NESTED_RETURN
        NESTED_EXIT(KiLowerIrqlSoftwareInterruptPending)