// TITLE("Spin Locks") //++ // // Copyright (c) 1990 Microsoft Corporation // Copyright (c) 1992 Digital Equipment Corporation // // Module Name: // // spinlock.s // // Abstract: // // This module implements the routines for acquiring and releasing // spin locks. // // Author: // // David N. Cutler (davec) 23-Mar-1990 // Joe Notarangelo 06-Apr-1992 // // Environment: // // Kernel mode only. // // Revision History: // //-- #include "ksalpha.h" //++ // // VOID // KeInitializeSpinLock ( // IN PKSPIN_LOCK SpinLock // ) // // Routine Description: // // This function initializes an executive spin lock. // // Argument: // // SpinLock (a0) - Supplies a pointer to the executive spin lock. // // Return Value: // // None. // //-- LEAF_ENTRY( KeInitializeSpinLock ) stl zero, 0(a0) // set spin lock not owned ret zero, (ra) // return .end KeInitializeSpinLock //++ // // VOID // KeAcquireSpinLock ( // IN PKSPIN_LOCK SpinLock // OUT PKIRQL OldIrql // ) // // Routine Description: // // This function raises the current IRQL to DISPATCH_LEVEL and acquires // the specified executive spinlock. // // Arguments: // // SpinLock (a0) - Supplies a pointer to a executive spinlock. // // OldIrql (a1) - Supplies a pointer to a variable that receives the // the previous IRQL value. // // Return Value: // // None. // //-- LEAF_ENTRY(KeAcquireSpinLock) // // Raise IRQL to DISPATCH_LEVEL and acquire the specified spinlock. // // N.B. The raise IRQL code is duplicated here is avoid any extra overhead // since this is such a common operation. // // N.B. The previous IRQL must not be stored until the lock is owned. // // N.B. The longword surrounding the previous IRQL must not be read // until the lock is owned. // bis a0, zero, t5 // t5 = address of spin lock ldil a0, DISPATCH_LEVEL // set new IRQL bis a1, zero, t0 // t0 = a1, a1 may be destroyed SWAP_IRQL // swap irql, on return v0 = old irql // // Acquire the specified spinlock. // // N.B. code below branches forward if spinlock fails intentionally // because branch forwards are predicted to miss // #if !defined(NT_UP) 10: // ldl_l t3, 0(t5) // get current lock value bis t5, zero, t4 // set ownership value bne t3, 15f // if ne => lock owned stl_c t4, 0(t5) // set lock owned beq t4, 15f // if eq => stx_c failed mb // synchronize subsequent reads after // the spinlock is acquired #endif // // Save the old Irql at the address saved by the caller. // Insure that the old Irql is updated with longword granularity. // ldq_u t1, 0(t0) // read quadword surrounding KIRQL bic t0, 3, t2 // get address of containing longword mskbl t1, t0, t1 // clear KIRQL byte in quadword insbl v0, t0, v0 // get new KIRQL to correct byte bis t1, v0, t1 // merge KIRQL into quadword extll t1, t2, t1 // get longword containg KIRQL stl t1, 0(t2) // store containing longword ret zero, (ra) // return #if !defined(NT_UP) 15: // ldl t3, 0(t5) // get current lock value beq t3, 10b // retry acquire lock if unowned br zero, 15b // loop in cache until lock free #endif .end KeAcquireSpinLock SBTTL("Acquire SpinLock and Raise to Synch") //++ // // KIRQL // KeAcquireSpinLockRaiseToSynch ( // IN PKSPIN_LOCK SpinLock // ) // // Routine Description: // // This function raises the current IRQL to synchronization level and // acquires the specified spinlock. // // Arguments: // // SpinLock (a0) - Supplies a pointer to the spinlock that is to be // acquired. // // Return Value: // // The previous IRQL is returned as the function value. // //-- LEAF_ENTRY(KeAcquireSpinLockRaiseToSynch) #if !defined(NT_UP) bis a0, zero, t5 ldl a0, KiSynchIrql 10: // // Raise IRQL and attempt to acquire the specified spinlock. // SWAP_IRQL // save previous IRQL in v0 ldl_l t3, 0(t5) // get current lock value bis t5, zero, t4 // set ownership value bne t3, 25f // if ne, lock owned stl_c t4, 0(t5) // set lock owned beq t4, 25f // if eq, stl_c failed mb // synchronize subsequent reads ret zero, (ra) 25: // // Spinlock is owned, lower IRQL and spin in cache // until it looks free. // bis v0, zero, a0 SWAP_IRQL bis v0, zero, a0 26: ldl t3, 0(t5) // get current lock value beq t3, 10b // retry acquire if unowned br zero, 26b // loop in cache until free #else ldl a0, KiSynchIrql SWAP_IRQL ret zero, (ra) // return .end KeAcquireSpinLockRaiseToSynch #endif //++ // // KIRQL // KeAcquireSpinLockRaiseToDpc ( // IN PKSPIN_LOCK SpinLock // ) // // Routine Description: // // This function raises the current IRQL to dispatcher level and acquires // the specified spinlock. // // Arguments: // // SpinLock (a0) - Supplies a pointer to the spinlock that is to be // acquired. // // Return Value: // // The previous IRQL is returned as the function value. // //-- #if !defined(NT_UP) ALTERNATE_ENTRY(KeAcquireSpinLockRaiseToDpc) bis a0, zero, t5 ldil a0, DISPATCH_LEVEL br 10b // finish in common code .end KeAcquireSpinLockRaiseToSynch #else LEAF_ENTRY(KeAcquireSpinLockRaiseToDpc) ldil a0, DISPATCH_LEVEL // set new IRQL SWAP_IRQL // old irql in v0 ret zero, (ra) .end KeAcquireSpinLockRaiseToDpc #endif //++ // // VOID // KeReleaseSpinLock ( // IN PKSPIN_LOCK SpinLock // IN KIRQL OldIrql // ) // // Routine Description: // // This function releases an executive spin lock and lowers the IRQL // to its previous value. // // Arguments: // // SpinLock (a0) - Supplies a pointer to an executive spin lock. // // OldIrql (a1) - Supplies the previous IRQL value. // // Return Value: // // None. // //-- LEAF_ENTRY(KeReleaseSpinLock) // // Release the specified spinlock. // #if !defined(NT_UP) mb // synchronize all previous writes // before the spinlock is released stl zero, 0(a0) // set spin lock not owned #endif 10: // // Lower the IRQL to the specified level. // // N.B. The lower IRQL code is duplicated here is avoid any extra overhead // since this is such a common operation. // bis a1, zero, a0 // a0 = new irql SWAP_IRQL // change to new irql ret zero, (ra) // return .end KeReleaseSpinLock //++ // // BOOLEAN // KeTryToAcquireSpinLock ( // IN PKSPIN_LOCK SpinLock // OUT PKIRQL OldIrql // ) // // Routine Description: // // This function raises the current IRQL to DISPATCH_LEVEL and attempts // to acquires the specified executive spinlock. If the spinlock can be // acquired, then TRUE is returned. Otherwise, the IRQL is restored to // its previous value and FALSE is returned. // // Arguments: // // SpinLock (a0) - Supplies a pointer to a executive spinlock. // // OldIrql (a1) - Supplies a pointer to a variable that receives the // the previous IRQL value. // // Return Value: // // If the spin lock is acquired, then a value of TRUE is returned. // Otherwise, a value of FALSE is returned. // //-- LEAF_ENTRY(KeTryToAcquireSpinLock) // // Raise IRQL to DISPATCH_LEVEL and try to acquire the specified spinlock. // // N.B. The raise IRQL code is duplicated here is avoid any extra overhead // since this is such a common operation. // bis a0, zero, t5 // t5 = address of spin lock ldil a0, DISPATCH_LEVEL // new irql bis a1, zero, t11 // t11 = a1, a1 may be clobbered SWAP_IRQL // a0 = new, on return v0 = old irql // // Try to acquire the specified spinlock. // // N.B. A noninterlocked test is done before the interlocked attempt. This // allows spinning without interlocked cycles. // #if !defined(NT_UP) ldl t0, 0(t5) // get current lock value bne t0, 20f // if ne, lock owned 10: ldl_l t0, 0(t5) // get current lock value bis t5, zero, t3 // t3 = ownership value bne t0, 20f // if ne, spin lock owned stl_c t3, 0(t5) // set lock owned beq t3, 15f // if eq, store conditional failure mb // synchronize subsequent reads after // the spinlock is acquired #endif // // The attempt to acquire the specified spin lock succeeded. // // // Save the old Irql at the address saved by the caller. // Insure that the old Irql is updated with longword granularity. // ldq_u t1, 0(t11) // read quadword containing KIRQL bic t11, 3, t2 // get address of containing longword mskbl t1, t11, t1 // clear byte position of KIRQL bis v0, zero, a0 // save old irql insbl v0, t11, v0 // get KIRQL to correct byte bis t1, v0, t1 // merge KIRQL into quadword extll t1, t2, t1 // extract containing longword stl t1, 0(t2) // store containing longword ldil v0, TRUE // set return value ret zero, (ra) // return // // The attempt to acquire the specified spin lock failed. Lower IRQL to its // previous value and return FALSE. // // N.B. The lower IRQL code is duplicated here is avoid any extra overhead // since this is such a common operation. // #if !defined(NT_UP) 20: // bis v0, zero, a0 // set old IRQL value SWAP_IRQL // change back to old irql(a0) ldil v0, FALSE // set return to failed ret zero, (ra) // return 15: // br zero, 10b // retry spinlock #endif .end KeTryToAcquireSpinLock //++ // // KIRQL // KiAcquireSpinLock ( // IN PKSPIN_LOCK SpinLock // ) // // Routine Description: // // This function acquires a kernel spin lock. // // N.B. This function assumes that the current IRQL is set properly. // // Arguments: // // SpinLock (a0) - Supplies a pointer to a kernel spin lock. // // Return Value: // // None. // //-- LEAF_ENTRY(KiAcquireSpinLock) ALTERNATE_ENTRY(KeAcquireSpinLockAtDpcLevel) #if !defined(NT_UP) GET_CURRENT_THREAD // v0 = current thread address 10: // ldl_l t2, 0(a0) // get current lock value bis v0, zero, t3 // set ownership value bne t2, 15f // if ne, spin lock owned stl_c t3, 0(a0) // set spin lock owned beq t3, 15f // if eq, store conditional failure mb // synchronize subsequent reads after // the spinlock is acquired ret zero, (ra) // return 15: // ldl t2, 0(a0) // get current lock value beq t2, 10b // retry acquire lock if unowned br zero, 15b // loop in cache until lock free #else ret zero, (ra) // return #endif .end KiAcquireSpinLock //++ // // VOID // KiReleaseSpinLock ( // IN PKSPIN_LOCK SpinLock // ) // // Routine Description: // // This function releases a kernel spin lock. // // N.B. This function assumes that the current IRQL is set properly. // // Arguments: // // SpinLock (a0) - Supplies a pointer to an executive spin lock. // // Return Value: // // None. // //-- LEAF_ENTRY(KiReleaseSpinLock) ALTERNATE_ENTRY(KeReleaseSpinLockFromDpcLevel) #if !defined(NT_UP) mb // synchronize all previous writes // before the spinlock is released stl zero, 0(a0) // set spin lock not owned #endif ret zero, (ra) // return .end KiReleaseSpinLock //++ // // KIRQL // KiTryToAcquireSpinLock ( // IN PKSPIN_LOCK SpinLock // ) // // Routine Description: // // This function attempts to acquires the specified kernel spinlock. If // the spinlock can be acquired, then TRUE is returned. Otherwise, FALSE // is returned. // // N.B. This function assumes that the current IRQL is set properly. // // Arguments: // // SpinLock (a0) - Supplies a pointer to a kernel spin lock. // // Return Value: // // If the spin lock is acquired, then a value of TRUE is returned. // Otherwise, a value of FALSE is returned. // //-- LEAF_ENTRY(KiTryToAcquireSpinLock) #if !defined(NT_UP) GET_CURRENT_THREAD // v0 = current thread address 10: // ldl_l t2, 0(a0) // get current lock value bis v0, zero, t3 // set ownership value bne t2, 20f // if ne, spin lock owned stl_c t3, 0(a0) // set spin lock owned beq t3, 15f // if eq, stl_c failed mb // synchronize subsequent reads after // the spinlock is acquired ldil v0, TRUE // set success return value ret zero, (ra) // return 20: // ldil v0, FALSE // set failure return value ret zero, (ra) // return 15: // br zero, 10b // retry #else ldil v0, TRUE // set success return value ret zero, (ra) // return #endif .end KiTryToAcquireSpinLock