|
|
// TITLE("Spin Locks") //++ // // Copyright (c) 1990 Microsoft Corporation // Copyright (c) 1992 Digital Equipment Corporation // // Module Name: // // spinlock.s // // Abstract: // // This module implements the routines for acquiring and releasing // spin locks. // // Author: // // David N. Cutler (davec) 23-Mar-1990 // Joe Notarangelo 06-Apr-1992 // // Environment: // // Kernel mode only. // // Revision History: // //--
#include "ksalpha.h"
//++ // // VOID // KeInitializeSpinLock ( // IN PKSPIN_LOCK SpinLock // ) // // Routine Description: // // This function initializes an executive spin lock. // // Argument: // // SpinLock (a0) - Supplies a pointer to the executive spin lock. // // Return Value: // // None. // //--
LEAF_ENTRY( KeInitializeSpinLock )
STP zero, 0(a0) // set spin lock not owned ret zero, (ra) // return
.end KeInitializeSpinLock
//++ // // VOID // KeAcquireSpinLock ( // IN PKSPIN_LOCK SpinLock // OUT PKIRQL OldIrql // ) // // Routine Description: // // This function raises the current IRQL to DISPATCH_LEVEL and acquires // the specified executive spinlock. // // Arguments: // // SpinLock (a0) - Supplies a pointer to a executive spinlock. // // OldIrql (a1) - Supplies a pointer to a variable that receives the // the previous IRQL value. // // Return Value: // // None. // //--
LEAF_ENTRY(KeAcquireSpinLock)
// // Raise IRQL to DISPATCH_LEVEL and acquire the specified spinlock. // // N.B. The raise IRQL code is duplicated here is avoid any extra overhead // since this is such a common operation. // // N.B. The previous IRQL must not be stored until the lock is owned. // // N.B. The longword surrounding the previous IRQL must not be read // until the lock is owned. //
bis a0, zero, t5 // t5 = address of spin lock ldil a0, DISPATCH_LEVEL // set new IRQL bis a1, zero, t0 // t0 = a1, a1 may be destroyed
SWAP_IRQL // swap irql, on return v0 = old irql
// // Acquire the specified spinlock. // // N.B. code below branches forward if spinlock fails intentionally // because branch forwards are predicted to miss //
#if !defined(NT_UP)
10: LDP_L t3, 0(t5) // get current lock value bis t5, zero, t4 // set ownership value bne t3, 15f // if ne => lock owned STP_C t4, 0(t5) // set lock owned beq t4, 15f // if eq => stx_c failed mb // synchronize memory access
#endif
// // Save the old Irql at the address saved by the caller. // Insure that the old Irql is updated with longword granularity. //
ldq_u t1, 0(t0) // read quadword surrounding KIRQL bic t0, 3, t2 // get address of containing longword mskbl t1, t0, t1 // clear KIRQL byte in quadword insbl v0, t0, v0 // get new KIRQL to correct byte bis t1, v0, t1 // merge KIRQL into quadword extll t1, t2, t1 // get longword containg KIRQL stl t1, 0(t2) // store containing longword ret zero, (ra) // return
// // Attempt to acquire spinlock failed. //
#if !defined(NT_UP)
15: LDP t3, 0(t5) // get current lock value beq t3, 10b // retry acquire lock if unowned br zero, 15b // loop in cache until lock free
#endif
.end KeAcquireSpinLock
SBTTL("Acquire SpinLock and Raise to Synch") //++ // // KIRQL // KeAcquireSpinLockRaiseToSynch ( // IN PKSPIN_LOCK SpinLock // ) // // Routine Description: // // This function raises the current IRQL to synchronization level and // acquires the specified spinlock. // // Arguments: // // SpinLock (a0) - Supplies a pointer to the spinlock that is to be // acquired. // // Return Value: // // The previous IRQL is returned as the function value. // //--
LEAF_ENTRY(KeAcquireSpinLockRaiseToSynch)
#if !defined(NT_UP)
bis a0, zero, t5 // save spinlock address ldl a0, KiSynchIrql // get synch level IRQL
// // Raise IRQL and attempt to acquire the specified spinlock. //
10: SWAP_IRQL // raise IRQL to synch level
LDP_L t3, 0(t5) // get current lock value bis t5, zero, t4 // set ownership value bne t3, 25f // if ne, lock owned STP_C t4, 0(t5) // set lock owned beq t4, 25f // if eq, conditional store failed mb // synchronize subsequent reads ret zero, (ra)
// // Spinlock is owned, lower IRQL and spin in cache until it looks free. //
25: bis v0, zero, a0 // get previous IRQL value
SWAP_IRQL // lower IRQL
bis v0, zero, a0 // save synch level IRQL 26: LDP t3, 0(t5) // get current lock value beq t3, 10b // retry acquire if unowned br zero, 26b // loop in cache until free
#else
ldl a0, KiSynchIrql // get synch level IRQL
SWAP_IRQL // rasie IRQL to synch levcel
ret zero, (ra) // return
.end KeAcquireSpinLockRaiseToSynch
#endif
//++ // // KIRQL // KeAcquireSpinLockRaiseToDpc ( // IN PKSPIN_LOCK SpinLock // ) // // Routine Description: // // This function raises the current IRQL to dispatcher level and acquires // the specified spinlock. // // Arguments: // // SpinLock (a0) - Supplies a pointer to the spinlock that is to be // acquired. // // Return Value: // // The previous IRQL is returned as the function value. // //--
#if !defined(NT_UP)
ALTERNATE_ENTRY(KeAcquireSpinLockRaiseToDpc)
bis a0, zero, t5 // save spinlock address ldil a0, DISPATCH_LEVEL // set IRQL level br 10b // finish in common code
.end KeAcquireSpinLockRaiseToSynch
#else
LEAF_ENTRY(KeAcquireSpinLockRaiseToDpc)
ldil a0, DISPATCH_LEVEL // set new IRQL
SWAP_IRQL // old irql in v0
ret zero, (ra)
.end KeAcquireSpinLockRaiseToDpc
#endif
//++ // // VOID // KeReleaseSpinLock ( // IN PKSPIN_LOCK SpinLock // IN KIRQL OldIrql // ) // // Routine Description: // // This function releases an executive spin lock and lowers the IRQL // to its previous value. // // Arguments: // // SpinLock (a0) - Supplies a pointer to an executive spin lock. // // OldIrql (a1) - Supplies the previous IRQL value. // // Return Value: // // None. // //-- LEAF_ENTRY(KeReleaseSpinLock)
// // Release the specified spinlock. //
#if !defined(NT_UP)
mb // synchronize memory access STP zero, 0(a0) // set spin lock not owned
#endif
// // Lower the IRQL to the specified level. // // N.B. The lower IRQL code is duplicated here is avoid any extra overhead // since this is such a common operation. //
10: bis a1, zero, a0 // a0 = new irql
SWAP_IRQL // change to new irql
ret zero, (ra) // return
.end KeReleaseSpinLock
//++ // // BOOLEAN // KeTryToAcquireSpinLock ( // IN PKSPIN_LOCK SpinLock // OUT PKIRQL OldIrql // ) // // Routine Description: // // This function raises the current IRQL to DISPATCH_LEVEL and attempts // to acquires the specified executive spinlock. If the spinlock can be // acquired, then TRUE is returned. Otherwise, the IRQL is restored to // its previous value and FALSE is returned. // // Arguments: // // SpinLock (a0) - Supplies a pointer to a executive spinlock. // // OldIrql (a1) - Supplies a pointer to a variable that receives the // the previous IRQL value. // // Return Value: // // If the spin lock is acquired, then a value of TRUE is returned. // Otherwise, a value of FALSE is returned. // //--
LEAF_ENTRY(KeTryToAcquireSpinLock)
// // Raise IRQL to DISPATCH_LEVEL and try to acquire the specified spinlock. // // N.B. The raise IRQL code is duplicated here is avoid any extra overhead // since this is such a common operation. //
bis a0, zero, t5 // t5 = address of spin lock ldil a0, DISPATCH_LEVEL // new irql bis a1, zero, t11 // t11 = a1, a1 may be clobbered
SWAP_IRQL // a0 = new, on return v0 = old irql
// // Try to acquire the specified spinlock. // // N.B. A noninterlocked test is done before the interlocked attempt. This // allows spinning without interlocked cycles. //
#if !defined(NT_UP)
LDP t0, 0(t5) // get current lock value bne t0, 20f // if ne, lock owned 10: LDP_L t0, 0(t5) // get current lock value bis t5, zero, t3 // t3 = ownership value bne t0, 20f // if ne, spin lock owned STP_C t3, 0(t5) // set lock owned beq t3, 15f // if eq, store conditional failure mb // synchronize memory access
#endif
// // The attempt to acquire the specified spin lock succeeded. // // Save the old Irql at the address saved by the caller. // Insure that the old Irql is updated with longword granularity. //
ldq_u t1, 0(t11) // read quadword containing KIRQL bic t11, 3, t2 // get address of containing longword mskbl t1, t11, t1 // clear byte position of KIRQL bis v0, zero, a0 // save old irql insbl v0, t11, v0 // get KIRQL to correct byte bis t1, v0, t1 // merge KIRQL into quadword extll t1, t2, t1 // extract containing longword stl t1, 0(t2) // store containing longword ldil v0, TRUE // set return value ret zero, (ra) // return
// // The attempt to acquire the specified spin lock failed. Lower IRQL to its // previous value and return FALSE. // // N.B. The lower IRQL code is duplicated here is avoid any extra overhead // since this is such a common operation. //
#if !defined(NT_UP)
20: bis v0, zero, a0 // set old IRQL value
SWAP_IRQL // change back to old irql(a0)
ldil v0, FALSE // set return to failed ret zero, (ra) // return
// // Attempt to acquire spinlock failed. //
15: br zero, 10b // retry spinlock
#endif
.end KeTryToAcquireSpinLock
//++ // // KIRQL // KiAcquireSpinLock ( // IN PKSPIN_LOCK SpinLock // ) // // Routine Description: // // This function acquires a kernel spin lock. // // N.B. This function assumes that the current IRQL is set properly. // // Arguments: // // SpinLock (a0) - Supplies a pointer to a kernel spin lock. // // Return Value: // // None. // //--
LEAF_ENTRY(KiAcquireSpinLock)
ALTERNATE_ENTRY(KeAcquireSpinLockAtDpcLevel)
#if !defined(NT_UP)
GET_CURRENT_THREAD // v0 = current thread address
10: LDP_L t2, 0(a0) // get current lock value bis v0, zero, t3 // set ownership value bne t2, 15f // if ne, spin lock owned STP_C t3, 0(a0) // set spin lock owned beq t3, 15f // if eq, store conditional failure mb // synchronize memory access ret zero, (ra) // return
// // attempt to acquire spinlock failed. //
15: LDP t2, 0(a0) // get current lock value beq t2, 10b // retry acquire lock if unowned br zero, 15b // loop in cache until lock free
#else
ret zero, (ra) // return
#endif
.end KiAcquireSpinLock
//++ // // VOID // KiReleaseSpinLock ( // IN PKSPIN_LOCK SpinLock // ) // // Routine Description: // // This function releases a kernel spin lock. // // N.B. This function assumes that the current IRQL is set properly. // // Arguments: // // SpinLock (a0) - Supplies a pointer to an executive spin lock. // // Return Value: // // None. // //--
LEAF_ENTRY(KiReleaseSpinLock)
ALTERNATE_ENTRY(KeReleaseSpinLockFromDpcLevel)
#if !defined(NT_UP)
mb // synchronize memory accesss STP zero, 0(a0) // set spin lock not owned
#endif
ret zero, (ra) // return
.end KiReleaseSpinLock
//++ // // KIRQL // KiTryToAcquireSpinLock ( // IN PKSPIN_LOCK SpinLock // ) // // Routine Description: // // This function attempts to acquires the specified kernel spinlock. If // the spinlock can be acquired, then TRUE is returned. Otherwise, FALSE // is returned. // // N.B. This function assumes that the current IRQL is set properly. // // Arguments: // // SpinLock (a0) - Supplies a pointer to a kernel spin lock. // // Return Value: // // If the spin lock is acquired, then a value of TRUE is returned. // Otherwise, a value of FALSE is returned. // //--
LEAF_ENTRY(KiTryToAcquireSpinLock)
#if !defined(NT_UP)
GET_CURRENT_THREAD // v0 = current thread address
10: LDP_L t2, 0(a0) // get current lock value bis v0, zero, t3 // set ownership value bne t2, 20f // if ne, spin lock owned STP_C t3, 0(a0) // set spin lock owned beq t3, 15f // if eq, conditional store failed mb // synchronize memory access ldil v0, TRUE // set success return value ret zero, (ra) // return
20: ldil v0, FALSE // set failure return value ret zero, (ra) // return
// // Attempt to acquire spinlock failed. //
15: br zero, 10b // retry
#else
ldil v0, TRUE // set success return value ret zero, (ra) // return
#endif
.end KiTryToAcquireSpinLock
//++ // // BOOLEAN // KeTestSpinLock ( // IN PKSPIN_LOCK SpinLock // ) // // Routine Description: // // This function tests a kernel spin lock. If the spinlock is // busy, FALSE is returned. If not, TRUE is returned. The spinlock // is never acquired. This is provided to allow code to spin at low // IRQL, only raising the IRQL when there is a reasonable hope of // acquiring the lock. // // Arguments: // // SpinLock (a0) - Supplies a pointer to a kernel spin lock. // // Return Value: // // TRUE - Spinlock appears available // FALSE - SpinLock is busy //--
#if !defined(NT_UP)
LEAF_ENTRY(KeTestSpinLock)
LDP t0, (a0) // get current spinlock value ldil v0, 1 // default TRUE cmovne t0, zero, v0 // if t0 != 0, return FALSE ret zero, (ra) // return
.end KeTestSpinLock
#endif
SBTTL("Acquire Queued SpinLock and Raise IRQL") //++ // // VOID // KeAcquireInStackQueuedSpinLock ( // IN PKSPIN_LOCK SpinLock, // IN PKLOCK_QUEUE_HANDLE LockHandle // ) // // VOID // KeAcquireInStackQueuedSpinLockRaiseToSynch ( // IN PKSPIN_LOCK SpinLock, // IN PKLOCK_QUEUE_HANDLE LockHandle // ) // // Routine Description: // // This function raises the current IRQL to either synchronization or // dispatch level and acquires the specified queued spinlock. // // Arguments: // // SpinLock (a0) - Supplies a pointer to a spin lock. // // LockHandle (a1) - Supplies a pointer to a lock handle. // // Return Value: // // None. // //--
LEAF_ENTRY(KeAcquireInStackQueuedSpinLock)
ldil v0, DISPATCH_LEVEL // get dispatch level IRQL br zero, 5f //
ALTERNATE_ENTRY(KeAcquireInStackQueuedSpinLockRaiseToSynch)
ldl v0, KiSynchIrql // get synch level IRQL
5: mov a1, t5 // save address of lock handle
#if !defined(NT_UP)
STP zero, LqhNext(t5) // set next link to NULL STP a0, LqhLock(t5) // set spin lock address mb // synchronize memory access
#endif
mov v0, a0 // set new Irql value
SWAP_IRQL // raise Irql to specified level
stl v0, LqhOldIrql(t5) // save old IRQL
#if !defined(NT_UP)
ADDP t5, LqhNext, t5 // set address of lock queue br zero, KxqAcquireQueuedSpinLock // finish in common code
#else
ret zero, (ra) // return
#endif
.end KeAcquireInStackQueuedSpinLock
SBTTL("Acquire Queued SpinLock and Raise IRQL") //++ // // KIRQL // KeAcquireQueuedSpinLock ( // IN KSPIN_LOCK_QUEUE_NUMBER Number // ) // // KIRQL // KeAcquireQueuedSpinLockRaiseToSynch ( // IN KSPIN_LOCK_QUEUE_NUMBER Number // ) // // Routine Description: // // This function raises the current IRQL to synchronization level and // acquires the specified queued spinlock. // // Arguments: // // Number (a0) - Supplies the queued spinlock number. // // Return Value: // // The previous IRQL is returned as the function value. // //--
LEAF_ENTRY(KeAcquireQueuedSpinLock)
addq a0, a0, t5 // account for two addresses ldil a0, DISPATCH_LEVEL // get dispatch level IRQL br zero, 5f //
ALTERNATE_ENTRY(KeAcquireQueuedSpinLockRaiseToSynch)
addq a0, a0, t5 // account for two addresses ldl a0, KiSynchIrql // get synch level IRQL
5: SWAP_IRQL // raise Irql to specified level
#if !defined(NT_UP)
bis v0, zero, t0 // save previous Irql
GET_PROCESSOR_CONTROL_BLOCK_BASE // get current prcb address
SPADDP t5, v0, t5 // entry address lda t5, PbLockQueue(t5) //
ALTERNATE_ENTRY(KxqAcquireQueuedSpinLock)
LDP t4, LqLock(t5) // get associated spinlock address 10: LDP_L t3, 0(t4) // get current lock value bis t5, zero, t2 // set lock ownership value STP_C t2, 0(t4) // beq t2, 50f // if eq, conditional store failed bne t3, 30f // if ne, lock already owned bis t4, LOCK_QUEUE_OWNER, t4 // set lock owner bit in lock entry STP t4, LqLock(t5) // mb // synchronize subsequent reads 20: bis t0, zero, v0 // set old IRQL value
#endif
ret zero, (ra) // return
// // The lock is owned by another processor. Set the lock bit in the current // processor lock queue entry, set the next link in the previous lock queue // entry, and spin on the current processor's lock bit. //
#if !defined(NT_UP)
30: bis t4, LOCK_QUEUE_WAIT, t4 // set lock wait bit in lock entry STP t4, LqLock(t5) // mb // synchronize memory access STP t5, LqNext(t3) // set address of lock queue entry 40: LDP t4, LqLock(t5) // get lock address and lock wait bit blbc t4, 20b // if lbc (lock wait), ownership granted br zero, 40b // try again
// // Conditional store failed. //
50: br zero, 10b // try again
#endif
.end KeAcquireQueuedSpinLock
SBTTL("Release Queued SpinLock and Lower IRQL") //++ // // VOID // KeReleaseInStackQueuedSpinLock ( // IN PKLOCK_QUEUE_HANDLE LockHandle // ) // // Routine Description: // // This function releases a queued spinlock and lowers the IRQL to its // previous value. // // Arguments: // // LockHandle (a0) - Supplies a pointer to a lock handle. // // Return Value: // // None. // //--
LEAF_ENTRY(KeReleaseInStackQueuedSpinLock)
ldl t0, LqhOldIrql(a0) // save old IRQL ADDP a0, LqhNext, t5 // set address of lock queue br zero, KxqReleaseQueuedSpinLock // finish in common code
.end KeReleaseInStackQueuedSpinLock
SBTTL("Release Queued SpinLock and Lower IRQL") //++ // // VOID // KeReleaseQueuedSpinLock ( // IN KSPIN_LOCK_QUEUE_NUMBER Number, // IN KIRQL OldIrql // ) // // Routine Description: // // This function releases a queued spinlock and lowers the IRQL to its // previous value. // // Arguments: // // Number (a0) - Supplies the queued spinlock number. // // OldIrql (a1) - Supplies the previous IRQL value. // // Return Value: // // None. // //--
LEAF_ENTRY(KeReleaseQueuedSpinLock)
bis a1, zero, t0 // save old IRQL
#if !defined(NT_UP)
addq a0, a0, t5 // account for two addresses
GET_PROCESSOR_CONTROL_BLOCK_BASE // get current prcb address
SPADDP t5, v0, t5 // compute per processor lock queue lda t5, PbLockQueue(t5) // entry address
#endif
ALTERNATE_ENTRY(KxqReleaseQueuedSpinLock)
#if !defined(NT_UP)
mb // synchronize memory access LDP t4, LqLock(t5) // get associated spin lock address bic t4, LOCK_QUEUE_OWNER, t4 // clear lock owner bit in lock entry 10: LDP_L t3, 0(t4) // get current lock ownership value xor t3, t5, t2 // set lock ownership value bne t2, 20f // if ne, another processor waiting STP_C t2, 0(t4) // set new ownership value beq t2, 10b // if eq, conditional store failed STP t4, LqLock(t5) // store associated spin lock address mb // synchronize memory access
#endif
bis t0, zero, a0 // set old IRQL value
SWAP_IRQL // lower IRQL to previous level
ret zero, (ra) // return
// // Another processor has inserted its lock queue entry in the lock queue, // but has not yet written its lock queue entry address in the current // processor's next link. Spin until the lock queue address is written. //
#if !defined(NT_UP)
20: LDP t3, LqNext(t5) // get next lock queue entry address beq t3, 20b // if eq, address not written yet
// // Grant the next process in the lock queue ownership of the spinlock. //
LDP t2, LqLock(t3) // get spinlock address and wait bit STP zero, LqNext(t5) // clear next lock queue entry address STP t4, LqLock(t5) // store associated spin lock address bic t2, LOCK_QUEUE_WAIT, t2 // clear lock wait bit in lock entry bis t2, LOCK_QUEUE_OWNER, t2 // set lock owner bit in lock entry STP t2, LqLock(t3) // mb // synchronize memory access bis t0, zero, a0 // set old IRQL value
SWAP_IRQL // lower IRQL to previous level
ret zero, (ra) // return
#endif
.end KeReleaseQueuedSpinLock
SBTTL("Try to Acquire Queued SpinLock and Raise IRQL") //++ // // LOGICAL // KeTryToAcquireQueuedSpinLock ( // IN KSPIN_LOCK_QUEUE_NUMBER Number // OUT PKIRQL OldIrql // ) // // LOGICAL // KeTryToAcquireQueuedSpinLockRaiseToSynch ( // IN KSPIN_LOCK_QUEUE_NUMBER Number // OUT PKIRQL OldIrql // ) // // Routine Description: // // This function raises the current IRQL to synchronization level and // attempts to acquire the specified queued spinlock. If the spinlock // cannot be acquired, then IRQL is restored and FALSE is returned as // the function value. Otherwise, TRUE is returned as the function // value. // // Arguments: // // Number (a0) - Supplies the queued spinlock number. // // OldIrql (a1) - Supplies a pointer to a variable that receives the // the previous IRQL value. // // Return Value: // // If the spin lock is acquired, then a value of TRUE is returned. // Otherwise, a value of FALSE is returned. // //--
LEAF_ENTRY(KeTryToAcquireQueuedSpinLock)
addq a0, a0, t5 // account for two addresses ldil a0, DISPATCH_LEVEL // get dispatch level irql br zero, 5f //
ALTERNATE_ENTRY(KeTryToAcquireQueuedSpinLockRaiseToSynch)
addq a0, a0, t5 // account for two addresses ldl a0, KiSynchIrql // get synch level irql 5: bis a1, zero, t0 // save previous irql address
SWAP_IRQL // raise irql to specified level
bis v0, zero, t1 // save previous irql
#if !defined(NT_UP)
GET_PROCESSOR_CONTROL_BLOCK_BASE // get current prcb address
SPADDP t5, v0, t5 // compute per processor lock queue lda t5, PbLockQueue(t5) // entry address LDP t4, LqLock(t5) // get associated spinlock address
// // Try to acquire the specified spinlock. // // N.B. A noninterlocked test is done before the interlocked attempt. This // allows spinning without interlocked cycles. //
LDP t3, 0(t4) // get current lock value bne t3, 20f // if ne, lock owned 10: LDP_L t3, 0(t4) // get current lock value bis t5, zero, t2 // set lock ownership value bne t3, 20f // if ne, spin lock owned STP_C t2, 0(t4) // set lock owned beq t2, 30f // if eq, store conditional failure
// // The attempt to acquire the specified spin lock succeeded. Set the spin // lock owner bit and save the old irql at the address specified by the // caller. Insure that the old Irql is updated with longword granularity. //
bis t4, LOCK_QUEUE_OWNER, t4 // set lock owner bit in lock entry STP t4, LqLock(t5) // mb // synchronize memory access
#endif
ldq_u t2, 0(t0) // get quadword containing irql bic t0, 3, t3 // get containing longword address mskbl t2, t0, t2 // clear byte position of Irql insbl t1, t0, t1 // shift irql to correct byte bis t2, t1, t2 // merge irql into quadword extll t2, t3, t2 // extract containing longword stl t2, 0(t3) // store containing longword ldil v0, TRUE // set return value ret zero, (ra) // return
// // The attempt to acquire the specified spin lock failed. Lower IRQL to its // previous value and return FALSE. //
#if !defined(NT_UP)
20: bis t1, zero, a0 // set old irql value
SWAP_IRQL // lower irql to previous level
ldil v0, FALSE // set return value ret zero, (ra) // return
// // Attempt to acquire spinlock failed. //
30: br zero, 10b // retry spinlock
#endif
.end KeTryToAcquireQueuedSpinLock
SBTTL("Acquire Queued SpinLock at Current IRQL") //++ // VOID // KeAcquireInStackQueuedSpinLockAtDpcLevel ( // IN PKSPIN_LOCK SpinLock, // IN PKLOCK_QUEUE_HANDLE LockHandle // ) // // Routine Description: // // This function acquires the specified queued spinlock at the current // IRQL. // // Arguments: // // SpinLock (a0) - Supplies the address of a spin lock. // // LockHandle (a1) - Supplies the address of an in stack lock handle. // // Return Value: // // None. // //--
LEAF_ENTRY(KeAcquireInStackQueuedSpinLockAtDpcLevel)
#if !defined(NT_UP)
STP zero, LqhNext(a1) // set next link to NULL STP a0, LqhLock(a1) // set spin lock address
#endif
ADDP a1, LqhNext, a0 // compute address of lock queue
//++ // // VOID // KeAcquireQueuedSpinLockAtDpcLevel ( // IN PKSPIN_LOCK_QUEUE LockQueue // ) // // Routine Description: // // This function acquires the specified queued spinlock at the current // IRQL. // // Arguments: // // LockQueue (a0) - Supplies the address of the lock queue entry. // // Return Value: // // None. // //--
ALTERNATE_ENTRY(KeAcquireQueuedSpinLockAtDpcLevel)
#if !defined(NT_UP)
mb // synchronize memory access LDP t4, LqLock(a0) // get associated spinlock address 10: LDP_L t3, 0(t4) // get current lock value bis a0, zero, t2 // set lock ownership value STP_C t2, 0(t4) // beq t2, 50f // if eq, conditional store failed bne t3, 30f // if ne, lock already owned bis t4, LOCK_QUEUE_OWNER, t4 // set lock owner bit in lock entry STP t4, LqLock(a0) // mb // synchronize subsequent reads
#endif
20: ret zero, (ra) // return
// // The lock is owned by another processor. Set the lock bit in the current // processor lock queue entry, set the next link in the previous lock queue // entry, and spin on the current processor's lock bit. //
#if !defined(NT_UP)
30: bis t4, LOCK_QUEUE_WAIT, t4 // set lock wait bit in lock entry STP t4, LqLock(a0) // mb // synchronize memory access STP a0, LqNext(t3) // set address of lock queue entry 40: LDP t4, LqLock(a0) // get lock address and lock wait bit blbc t4, 20b // if lbc (lock wait), ownership granted br zero, 40b // try again
// // Conditional store failed. //
50: br zero, 10b // try again
#endif
.end KeAcquireInStackQueuedSpinLockAtDpcLevel
SBTTL("Release Queued SpinLock at Current IRQL") //++ // // VOID // KeReleaseInStackQueuedSpinLockFromDpcLevel ( // IN PKLOCK_QUEUE_HANDLE LockHandle // ) // // Routine Description: // // This function releases a queued spinlock and preserves the current // IRQL. // // Arguments: // // LockHandle (a0) - Supplies the address of a lock handle. // // Return Value: // // None. // //--
LEAF_ENTRY(KeReleaseInStackQueuedSpinLockFromDpcLevel)
ADDP a0, LqhNext, a0 // compute address of lock queue
//++ // // VOID // KeReleaseQueuedSpinLockFromDpcLevel ( // IN PKSPIN_LOCK_QUEUE LockQueue // ) // // Routine Description: // // This function releases a queued spinlock and preserves the current // IRQL. // // Arguments: // // LockQueue (a0) - Supplies the address of the lock queue entry. // // Return Value: // // None. // //--
ALTERNATE_ENTRY(KeReleaseQueuedSpinLockFromDpcLevel)
#if !defined(NT_UP)
mb // synchronize memory access LDP t4, LqLock(a0) // get associate spin lock address bic t4, LOCK_QUEUE_OWNER, t4 // clear lock owner bit in lock entry 10: LDP_L t3, 0(t4) // get current lock ownership value xor t3, a0, t2 // set lock ownership value bne t2, 20f // if ne, another processor waiting STP_C t2, 0(t4) // set new ownership value beq t2, 10b // if eq, conditional store failed STP t4, LqLock(a0) // mb // synchronize memory access ret zero, (ra) // return
// // Another processor has inserted its lock queue entry in the lock queue, // but has not yet written its lock queue entry address in the current // processor's next link. Spin until the lock queue address is written. //
20: LDP t3, LqNext(a0) // get next lock queue entry address beq t3, 20b // if eq, address not written yet
// // Grant the next process in the lock queue ownership of the spinlock. //
LDP t2, LqLock(t3) // get spinlock address and lock bit STP zero, LqNext(a0) // clear next lock queue entry address STP t4, LqLock(a0) // bic t2, LOCK_QUEUE_WAIT, t2 // clear lock wait bit in lock entry bis t2, LOCK_QUEUE_OWNER, t2 // set lock owner bit in lock entry STP t2, LqLock(t3) // mb // synchronize memory access
#endif
ret zero, (ra) // return
.end KeReleaseInStackQueuedSpinLockFromDpcLevel
|