You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1287 lines
32 KiB
1287 lines
32 KiB
TITLE "Spin Locks"
|
|
;++
|
|
;
|
|
; Copyright (c) 1989-1998 Microsoft Corporation
|
|
;
|
|
; Module Name:
|
|
;
|
|
; spinlock.asm
|
|
;
|
|
; Abstract:
|
|
;
|
|
; This module implements x86 spinlock functions for the PC+MP HAL.
|
|
;
|
|
; Author:
|
|
;
|
|
; Bryan Willman (bryanwi) 13 Dec 89
|
|
;
|
|
; Environment:
|
|
;
|
|
; Kernel mode only.
|
|
;
|
|
; Revision History:
|
|
;
|
|
; Ron Mosgrove (o-RonMo) Dec 93 - modified for PC+MP HAL.
|
|
;--
|
|
|
|
.486p
|
|
|
|
include callconv.inc ; calling convention macros
|
|
include i386\kimacro.inc
|
|
include hal386.inc
|
|
include mac386.inc
|
|
include apic.inc
|
|
include ntapic.inc
|
|
|
|
EXTRNP _KeBugCheckEx,5,IMPORT
|
|
EXTRNP KfRaiseIrql, 1,,FASTCALL
|
|
EXTRNP KfLowerIrql, 1,,FASTCALL
|
|
EXTRNP _KeSetEventBoostPriority, 2, IMPORT
|
|
EXTRNP _KeWaitForSingleObject,5, IMPORT
|
|
extrn _HalpVectorToIRQL:byte
|
|
extrn _HalpIRQLtoTPR:byte
|
|
|
|
ifdef NT_UP
|
|
|
|
LOCK_ADD equ add
|
|
LOCK_DEC equ dec
|
|
LOCK_CMPXCHG equ cmpxchg
|
|
|
|
else
|
|
|
|
LOCK_ADD equ lock add
|
|
LOCK_DEC equ lock dec
|
|
LOCK_CMPXCHG equ lock cmpxchg
|
|
|
|
endif
|
|
|
|
_TEXT SEGMENT PARA PUBLIC 'CODE'
|
|
ASSUME DS:FLAT, ES:FLAT, SS:FLAT, FS:NOTHING, GS:NOTHING
|
|
|
|
SUBTTL "Acquire Kernel Spin Lock"
|
|
;++
|
|
;
|
|
; KIRQL
|
|
; FASTCALL
|
|
; KfAcquireSpinLock (
|
|
; IN PKSPIN_LOCK SpinLock
|
|
; )
|
|
;
|
|
; Routine Description:
|
|
;
|
|
; This function raises to DISPATCH_LEVEL and acquires the specified
|
|
; spin lock.
|
|
;
|
|
; Arguments:
|
|
;
|
|
; SpinLock (ecx) - Supplies a pointer to a kernel spin lock.
|
|
;
|
|
; Return Value:
|
|
;
|
|
; The old IRQL is returned as the function value.
|
|
;
|
|
;--
|
|
|
|
align 16
|
|
cPublicFastCall KfAcquireSpinLock ,1
|
|
cPublicFpo 0,0
|
|
|
|
mov edx, dword ptr APIC[LU_TPR] ; get old IRQL vector
|
|
mov dword ptr APIC[LU_TPR], DPC_VECTOR ; raise IRQL
|
|
jmp short sls10 ; finish in common code
|
|
|
|
fstENDP KfAcquireSpinLock
|
|
|
|
SUBTTL "Acquire Kernel Spin Lock"
|
|
;++
|
|
;
|
|
; KIRQL
|
|
; FASTCALL
|
|
; KeAcquireSpinLockRaiseToSynch (
|
|
; IN PKSPIN_LOCK SpinLock
|
|
; )
|
|
;
|
|
; Routine Description:
|
|
;
|
|
; This function raises to SYNCH_LEVEL and acquires the specified
|
|
; spin lock.
|
|
;
|
|
; Arguments:
|
|
;
|
|
; SpinLock (ecx) - Supplies a pointer to a kernel spin lock.
|
|
;
|
|
; Return Value:
|
|
;
|
|
; The old IRQL is returned as the function value.
|
|
;
|
|
;--
|
|
|
|
align 16
|
|
cPublicFastCall KeAcquireSpinLockRaiseToSynch,1
|
|
cPublicFpo 0,0
|
|
|
|
mov edx, dword ptr APIC[LU_TPR] ; get old vector
|
|
mov dword ptr APIC[LU_TPR], APIC_SYNCH_VECTOR ; raise IRQL
|
|
sls10: shr edx, 4 ; extract high 4 bits of vector
|
|
movzx eax, _HalpVectorToIRQL[edx] ; translate TPR to old IRQL
|
|
|
|
ifndef NT_UP
|
|
|
|
;
|
|
; Attempt to acquire the specified spin lock.
|
|
;
|
|
|
|
sls20: ACQUIRE_SPINLOCK ecx, <short sls30> ;
|
|
|
|
fstRET KeAcquireSpinLockRaiseToSynch
|
|
|
|
;
|
|
; Lock is owned - spin until it is free, then try again.
|
|
;
|
|
|
|
sls30: SPIN_ON_SPINLOCK ecx, sls20 ;
|
|
|
|
else
|
|
|
|
fstRET KeAcquireSpinLockRaiseToSynch
|
|
|
|
endif
|
|
|
|
fstENDP KeAcquireSpinLockRaiseToSynch
|
|
|
|
SUBTTL "KeAcquireSpinLockRaiseToSynchMCE"
|
|
;++
|
|
;
|
|
; KIRQL
|
|
; FASTCALL
|
|
; KeAcquireSpinLockRaiseToSynchMCE (
|
|
; IN PKSPIN_LOCK SpinLock
|
|
; )
|
|
;
|
|
; Routine Description:
|
|
;
|
|
; This function performs the same function as KeAcquireSpinLockRaiseToSynch
|
|
; but provides a work around for an IFU errata for Pentium Pro processors
|
|
; prior to stepping 619.
|
|
;
|
|
; Arguments:
|
|
;
|
|
; (ecx) = SpinLock - Supplies a pointer to a kernel spin lock.
|
|
;
|
|
; Return Value:
|
|
;
|
|
; OldIrql (TOS+8) - pointer to place old irql.
|
|
;
|
|
;--
|
|
|
|
ifndef NT_UP
|
|
|
|
align 16
|
|
cPublicFastCall KeAcquireSpinLockRaiseToSynchMCE,1
|
|
cPublicFpo 0,0
|
|
|
|
mov edx, dword ptr APIC[LU_TPR] ; (ecx) = Old Priority (Vector)
|
|
mov eax, edx
|
|
shr eax, 4
|
|
movzx eax, _HalpVectorToIRQL[eax] ; (al) = OldIrql
|
|
|
|
;
|
|
; Test lock
|
|
;
|
|
; TEST_SPINLOCK ecx,<short slm30> ; NOTE - Macro expanded below:
|
|
|
|
test dword ptr [ecx], 1
|
|
nop ; On a P6 prior to stepping B1 (619), we
|
|
nop ; need these 5 NOPs to ensure that we
|
|
nop ; do not take a machine check exception.
|
|
nop ; The cost is just 1.5 clocks as the P6
|
|
nop ; just tosses the NOPs.
|
|
|
|
jnz short slm30
|
|
|
|
;
|
|
; Raise irql.
|
|
;
|
|
|
|
slm10: mov dword ptr APIC[LU_TPR], APIC_SYNCH_VECTOR
|
|
|
|
;
|
|
; Attempt to assert the lock
|
|
;
|
|
|
|
ACQUIRE_SPINLOCK ecx,<short slm20>
|
|
fstRET KeAcquireSpinLockRaiseToSynchMCE
|
|
|
|
;
|
|
; Lock is owned, spin till it looks free, then go get it
|
|
;
|
|
|
|
align dword
|
|
|
|
slm20: mov dword ptr APIC[LU_TPR], edx
|
|
|
|
align dword
|
|
slm30: SPIN_ON_SPINLOCK ecx,slm10
|
|
|
|
fstENDP KeAcquireSpinLockRaiseToSynchMCE
|
|
|
|
endif
|
|
|
|
SUBTTL "Release Kernel Spin Lock"
|
|
;++
|
|
;
|
|
; VOID
|
|
; FASTCALL
|
|
; KeReleaseSpinLock (
|
|
; IN PKSPIN_LOCK SpinLock,
|
|
; IN KIRQL OldIrql
|
|
; )
|
|
;
|
|
; Routine Description:
|
|
;
|
|
; This function releases a spin lock and lowers to the old IRQL.
|
|
;
|
|
; Arguments:
|
|
;
|
|
; SpinLock (ecx) - Supplies a pointer to a spin lock.
|
|
; OldIrql (dl) - Supplies the old IRQL value.
|
|
;
|
|
; Return Value:
|
|
;
|
|
; None.
|
|
;
|
|
;--
|
|
|
|
align 16
|
|
cPublicFastCall KfReleaseSpinLock ,2
|
|
cPublicFpo 0,0
|
|
|
|
movzx eax, dl ; zero extend old IRQL
|
|
|
|
ifndef NT_UP
|
|
|
|
RELEASE_SPINLOCK ecx ; release spin lock
|
|
|
|
endif
|
|
|
|
;
|
|
; Lower IRQL to its previous level.
|
|
;
|
|
; N.B. Ensure that the requested priority is set before returning.
|
|
;
|
|
|
|
movzx ecx, _HalpIRQLtoTPR[eax] ; translate IRQL to TPR value
|
|
mov dword ptr APIC[LU_TPR], ecx ; lower to old IRQL
|
|
mov eax, dword ptr APIC[LU_TPR] ; synchronize
|
|
|
|
fstRET KfReleaseSpinLock
|
|
|
|
fstENDP KfReleaseSpinLock
|
|
|
|
SUBTTL "Acquire Lock With Interrupts Disabled"
|
|
;++
|
|
;
|
|
; ULONG
|
|
; FASTCALL
|
|
; HalpAcquireHighLevelLock (
|
|
; IN PKSPIN_LOCK SpinLock
|
|
; )
|
|
;
|
|
; Routine Description:
|
|
;
|
|
; This function disables interrupts and acquires a spinlock.
|
|
;
|
|
; Arguments:
|
|
;
|
|
; SpinLock (ecx) - Supplies a pointer to a spin lock.
|
|
;
|
|
; Return Value:
|
|
;
|
|
; The old EFLAGS are returned as the function value.
|
|
;
|
|
;--
|
|
|
|
align 16
|
|
cPublicFastCall HalpAcquireHighLevelLock, 1
|
|
|
|
pushfd ; save EFLAGS
|
|
pop eax ;
|
|
ahll10: cli ; disable interrupts
|
|
|
|
ACQUIRE_SPINLOCK ecx, ahll20 ; attempt to acquire spin lock
|
|
|
|
fstRET HalpAcquireHighLevelLock
|
|
|
|
ahll20: push eax ; restore EFLAGS
|
|
popfd ;
|
|
|
|
SPIN_ON_SPINLOCK ecx, <ahll10> ; wait for lock to be free
|
|
|
|
fstENDP HalpAcquireHighLevelLock
|
|
|
|
SUBTTL "Release Lock And Enable Interrupts"
|
|
;++
|
|
;
|
|
; VOID
|
|
; FASTCALL
|
|
; HalpReleaseHighLevelLock (
|
|
; IN PKSPIN_LOCK SpinLock,
|
|
; IN ULONG Eflags
|
|
; )
|
|
;
|
|
; Routine Description:
|
|
;
|
|
; This function releases a kernel spin lock and restores the old EFLAGS.
|
|
;
|
|
; Arguments:
|
|
;
|
|
; SpinLock (ecx) - Supplies a pointer to a spin lock.
|
|
; Eflags (edx) - supplies the old EFLAGS value.
|
|
;
|
|
; Return Value:
|
|
;
|
|
; None.
|
|
;
|
|
;--
|
|
|
|
align 16
|
|
cPublicFastCall HalpReleaseHighLevelLock, 2
|
|
|
|
RELEASE_SPINLOCK ecx ; release spin lock
|
|
|
|
push edx ; restore old EFLAGS
|
|
popfd ;
|
|
|
|
fstRET HalpReleaseHighLevelLock
|
|
|
|
fstENDP HalpReleaseHighLevelLock
|
|
|
|
SUBTTL "Acquire Fast Mutex"
|
|
;++
|
|
;
|
|
; VOID
|
|
; FASTCALL
|
|
; ExAcquireFastMutex (
|
|
; IN PFAST_MUTEX FastMutex
|
|
; )
|
|
;
|
|
; Routine description:
|
|
;
|
|
; This function acquires ownership of the specified FastMutex.
|
|
;
|
|
; Arguments:
|
|
;
|
|
; (ecx) = FastMutex - Supplies a pointer to the fast mutex.
|
|
;
|
|
; Return Value:
|
|
;
|
|
; None.
|
|
;
|
|
;--
|
|
|
|
align 16
|
|
cPublicFastCall ExAcquireFastMutex,1
|
|
cPublicFpo 0,0
|
|
|
|
mov eax, dword ptr APIC[LU_TPR] ; (eax) = Old Priority (Vector)
|
|
|
|
if DBG
|
|
;
|
|
; Caller must already be at or below APC_LEVEL.
|
|
;
|
|
|
|
cmp eax, APC_VECTOR
|
|
jg short afm11 ; irql too high ==> fatal.
|
|
endif
|
|
|
|
mov dword ptr APIC[LU_TPR], APC_VECTOR ; Write New Priority to the TPR
|
|
|
|
LOCK_DEC dword ptr [ecx].FmCount ; Get count
|
|
jnz short afm10 ; Not the owner so go wait.
|
|
|
|
mov dword ptr [ecx].FmOldIrql, eax
|
|
|
|
;
|
|
; Use esp to track the owning thread for debugging purposes.
|
|
; !thread from kd will find the owning thread. Note that the
|
|
; owner isn't cleared on release, check if the mutex is owned
|
|
; first.
|
|
;
|
|
|
|
mov dword ptr [ecx].FmOwner, esp
|
|
fstRet ExAcquireFastMutex
|
|
|
|
cPublicFpo 0,0
|
|
afm10:
|
|
inc dword ptr [ecx].FmContention
|
|
|
|
cPublicFpo 0,2
|
|
push ecx
|
|
push eax
|
|
add ecx, FmEvent ; Wait on Event
|
|
stdCall _KeWaitForSingleObject,<ecx,WrExecutive,0,0,0>
|
|
pop eax ; (al) = OldTpr
|
|
pop ecx ; (ecx) = FAST_MUTEX
|
|
|
|
mov dword ptr [ecx].FmOldIrql, eax
|
|
|
|
;
|
|
; Use esp to track the owning thread for debugging purposes.
|
|
; !thread from kd will find the owning thread. Note that the
|
|
; owner isn't cleared on release, check if the mutex is owned
|
|
; first.
|
|
;
|
|
|
|
mov dword ptr [ecx].FmOwner, esp
|
|
fstRet ExAcquireFastMutex
|
|
|
|
if DBG
|
|
|
|
cPublicFpo 0,1
|
|
afm11: stdCall _KeBugCheckEx,<IRQL_NOT_GREATER_OR_EQUAL,ecx,eax,033h,0>
|
|
|
|
endif
|
|
|
|
fstENDP ExAcquireFastMutex
|
|
|
|
SUBTTL "Release Fast Mutex"
|
|
;++
|
|
;
|
|
; VOID
|
|
; FASTCALL
|
|
; ExReleaseFastMutex (
|
|
; IN PFAST_MUTEX FastMutex
|
|
; )
|
|
;
|
|
; Routine description:
|
|
;
|
|
; This function releases ownership of the FastMutex.
|
|
;
|
|
; Arguments:
|
|
;
|
|
; (ecx) = FastMutex - Supplies a pointer to the fast mutex.
|
|
;
|
|
; Return Value:
|
|
;
|
|
; None.
|
|
;
|
|
;--
|
|
|
|
align 16
|
|
cPublicFastCall ExReleaseFastMutex,1
|
|
cPublicFpo 0,0
|
|
|
|
if DBG
|
|
;
|
|
; Caller must already be at APC_LEVEL or have APCs blocked.
|
|
;
|
|
|
|
mov eax, dword ptr APIC[LU_TPR] ; (eax) = Old Priority (Vector)
|
|
cmp eax, APC_VECTOR
|
|
je short rfm04 ; irql is ok.
|
|
|
|
cPublicFpo 0,1
|
|
stdCall _KeBugCheckEx,<IRQL_NOT_GREATER_OR_EQUAL,ecx,eax,034h,0>
|
|
|
|
rfm04:
|
|
endif
|
|
mov eax, dword ptr [ecx].FmOldIrql ; (eax) = OldTpr
|
|
|
|
LOCK_ADD dword ptr [ecx].FmCount, 1 ; Remove our count
|
|
jle short rfm05 ; if <= 0, set event
|
|
|
|
cPublicFpo 0,0
|
|
mov dword ptr APIC[LU_TPR], eax
|
|
mov ecx, dword ptr APIC[LU_TPR]
|
|
if DBG
|
|
cmp eax, ecx ; Verify TPR is what was
|
|
je short @f ; written
|
|
int 3
|
|
@@:
|
|
endif
|
|
fstRet ExReleaseFastMutex
|
|
|
|
|
|
cPublicFpo 0,1
|
|
rfm05: add ecx, FmEvent
|
|
push eax ; save new tpr
|
|
stdCall _KeSetEventBoostPriority, <ecx, 0>
|
|
pop eax ; restore tpr
|
|
|
|
cPublicFpo 0,0
|
|
mov dword ptr APIC[LU_TPR], eax
|
|
mov ecx, dword ptr APIC[LU_TPR]
|
|
if DBG
|
|
cmp eax, ecx ; Verify TPR is what was
|
|
je short @f ; written
|
|
int 3
|
|
@@:
|
|
endif
|
|
fstRet ExReleaseFastMutex
|
|
|
|
if DBG
|
|
|
|
endif
|
|
|
|
fstENDP ExReleaseFastMutex
|
|
|
|
SUBTTL "Try To Acquire Fast Mutex"
|
|
;++
|
|
;
|
|
; BOOLEAN
|
|
; FASTCALL
|
|
; ExTryToAcquireFastMutex (
|
|
; IN PFAST_MUTEX FastMutex
|
|
; )
|
|
;
|
|
; Routine description:
|
|
;
|
|
; This function acquires ownership of the FastMutex.
|
|
;
|
|
; Arguments:
|
|
;
|
|
; (ecx) = FastMutex - Supplies a pointer to the fast mutex.
|
|
;
|
|
; Return Value:
|
|
;
|
|
; Returns TRUE if the FAST_MUTEX was acquired; otherwise FALSE.
|
|
;
|
|
;--
|
|
|
|
align 16
|
|
cPublicFastCall ExTryToAcquireFastMutex,1
|
|
cPublicFpo 0,0
|
|
|
|
if DBG
|
|
;
|
|
; Caller must already be at or below APC_LEVEL.
|
|
;
|
|
|
|
mov eax, dword ptr APIC[LU_TPR] ; (eax) = Old Priority (Vector)
|
|
cmp eax, APC_VECTOR
|
|
jg short tam11 ; irql too high ==> fatal.
|
|
endif
|
|
|
|
;
|
|
; Try to acquire.
|
|
;
|
|
|
|
push dword ptr APIC[LU_TPR] ; Save Old Priority (Vector)
|
|
mov dword ptr APIC[LU_TPR], APC_VECTOR ; Write New Priority to the TPR
|
|
|
|
mov edx, 0 ; Value to set
|
|
mov eax, 1 ; Value to compare against
|
|
LOCK_CMPXCHG dword ptr [ecx].FmCount, edx ; Attempt to acquire
|
|
jnz short tam20 ; got it?
|
|
|
|
cPublicFpo 0,0
|
|
mov eax, 1 ; return TRUE
|
|
pop dword ptr [ecx].FmOldIrql ; Store Old TPR
|
|
|
|
;
|
|
; Use esp to track the owning thread for debugging purposes.
|
|
; !thread from kd will find the owning thread. Note that the
|
|
; owner isn't cleared on release, check if the mutex is owned
|
|
; first.
|
|
;
|
|
|
|
mov dword ptr [ecx].FmOwner, esp
|
|
|
|
fstRet ExTryToAcquireFastMutex
|
|
|
|
tam20: pop ecx ; (ecx) = Old TPR
|
|
mov dword ptr APIC[LU_TPR], ecx
|
|
mov eax, dword ptr APIC[LU_TPR]
|
|
|
|
if DBG
|
|
cmp ecx, eax ; Verify TPR is what was
|
|
je short @f ; written
|
|
int 3
|
|
@@:
|
|
endif
|
|
|
|
xor eax, eax ; return FALSE
|
|
YIELD
|
|
fstRet ExTryToAcquireFastMutex ; all done
|
|
|
|
if DBG
|
|
|
|
cPublicFpo 0,1
|
|
tam11: stdCall _KeBugCheckEx,<IRQL_NOT_GREATER_OR_EQUAL,ecx,eax,033h,0>
|
|
|
|
endif
|
|
|
|
fstENDP ExTryToAcquireFastMutex
|
|
|
|
SUBTTL "Acquire In Stack Queued SpinLock"
|
|
;++
|
|
;
|
|
; VOID
|
|
; FASTCALL
|
|
; KeAcquireInStackQueuedSpinLock (
|
|
; IN PKSPIN_LOCK SpinLock,
|
|
; IN PKLOCK_QUEUE_HANDLE LockHandle
|
|
; )
|
|
;
|
|
; VOID
|
|
; FASTCALL
|
|
; KeAcquireInStackQueuedSpinLockRaiseToSynch (
|
|
; IN PKSPIN_LOCK SpinLock,
|
|
; IN PKLOCK_QUEUE_HANDLE LockHandle
|
|
; )
|
|
;
|
|
; Routine Description:
|
|
;
|
|
; These functions raise IRQL and acquire an in-stack queued spin lock.
|
|
;
|
|
; Arguments:
|
|
;
|
|
; SpinLock (ecx) - Supplies a pointer to a spin lock.
|
|
; LockHandle (edx) - supplies a pointer to a lock context.
|
|
;
|
|
; Return Value:
|
|
;
|
|
; None.
|
|
;
|
|
;--
|
|
|
|
align 16
|
|
cPublicFastCall KeAcquireInStackQueuedSpinLock, 2
|
|
cPublicFpo 0,0
|
|
|
|
mov eax, dword ptr APIC[LU_TPR] ; get old IRQL vector
|
|
mov dword ptr APIC[LU_TPR], DPC_VECTOR ; raise IRQL
|
|
jmp short iqsl10 ; finish in common code
|
|
|
|
fstENDP KeAcquireInStackQueuedSpinLock
|
|
|
|
|
|
align 16
|
|
cPublicFastCall KeAcquireInStackQueuedSpinLockRaiseToSynch, 2
|
|
cPublicFpo 0,0
|
|
|
|
mov eax, dword ptr APIC[LU_TPR] ; get old IRQL vector
|
|
mov dword ptr APIC[LU_TPR], APIC_SYNCH_VECTOR ; raise IRQL
|
|
iqsl10: shr eax, 4 ; extract high 4 bits of vector
|
|
mov al, _HalpVectorToIRQL[eax] ; translate to old IRQL
|
|
mov [edx].LqhOldIrql, al ; save old IRQL in lock context
|
|
|
|
;
|
|
; Set spin lock address in lock context and clear next queue link.
|
|
;
|
|
|
|
ifndef NT_UP
|
|
|
|
mov [edx].LqhLock, ecx ; set spin lock address
|
|
and dword ptr [edx].LqhNext, 0 ; clear next link
|
|
|
|
ifdef CAPKERN_SYNCH_POINTS
|
|
|
|
push ecx ; lock address
|
|
push 000010101h ; 1 Dword, Timestamp, Subcode = 1
|
|
call _CAP_Log_NInt ;
|
|
add esp, 8 ;
|
|
|
|
endif
|
|
|
|
;
|
|
; Exchange the value of the lock with the address of the lock context.
|
|
;
|
|
|
|
mov eax, edx ; save lock context address
|
|
xchg [ecx], edx ; exchange lock context address
|
|
cmp edx, 0 ; check if lock is already held
|
|
jnz short iqsl30 ; if nz, lock already held
|
|
|
|
;
|
|
; N.B. The spin lock address is dword aligned and the bottom two bits are
|
|
; used as indicators.
|
|
;
|
|
; Bit 0 is LOCK_QUEUE_WAIT.
|
|
; Bit 1 is LOCK_QUEUE_OWNER.
|
|
;
|
|
|
|
or [eax].LqLock, LOCK_QUEUE_OWNER ; set lock owner
|
|
|
|
endif
|
|
|
|
iqsl20: fstRET KeAcquireInStackQueuedSpinLockRaiseToSynch
|
|
|
|
;
|
|
; The lock is already held by another processor. Set the wait bit in the
|
|
; lock context, then set the next field in the lock context of the last
|
|
; waiter in the lock queue.
|
|
;
|
|
|
|
ifndef NT_UP
|
|
|
|
iqsl30: or [eax].LqLock, LOCK_QUEUE_WAIT ; set lock wait
|
|
mov [edx].LqNext, eax ; set next entry in previous last
|
|
|
|
ifdef CAPKERN_SYNCH_POINTS
|
|
|
|
xor edx, edx ; clear wait counter
|
|
iqsl40: inc edx ; count wait time
|
|
test [eax].LqLock, LOCK_QUEUE_WAIT ; check if lock ownership granted
|
|
jz short iqsl50 ; if z, lock owner granted
|
|
|
|
YIELD ; yield to other SMT processors
|
|
|
|
jmp short iqsl40 ;
|
|
|
|
iqsl50: push ecx ; lock address
|
|
push edx ; wait counter
|
|
push 000020104h ; 2 Dwords, Timestamp, Subcode = 4
|
|
call _CAP_Log_NInt ;
|
|
add esp, 12 ;
|
|
|
|
fstRET KeAcquireInStackQueuedSpinLockRaiseToSynch
|
|
|
|
else
|
|
|
|
iqsl40: test [eax].LqLock, LOCK_QUEUE_WAIT ; check if lock ownership granted
|
|
jz short iqsl20 ; if z, lock ownership granted
|
|
|
|
YIELD ; yield to other SMT processors
|
|
|
|
jmp short iqsl40 ;
|
|
|
|
endif
|
|
|
|
endif
|
|
|
|
fstENDP KeAcquireInStackQueuedSpinLockRaiseToSynch
|
|
|
|
SUBTTL "Acquire Queued Spin Lock"
|
|
;++
|
|
;
|
|
; KIRQL
|
|
; FASTCALL
|
|
; KeAcquireQueuedSpinLock (
|
|
; IN KSPIN_LOCK_QUEUE_NUMBER Number
|
|
; )
|
|
;
|
|
; KIRQL
|
|
; FASTCALL
|
|
; KeAcquireQueuedSpinLockRaiseToSynch (
|
|
; IN KSPIN_LOCK_QUEUE_NUMBER Number
|
|
; )
|
|
;
|
|
; Routine Description:
|
|
;
|
|
; These function raise IRQL and acquire a processor specific queued spin
|
|
; lock.
|
|
;
|
|
; Arguments:
|
|
;
|
|
; Number (ecx) - Supplies the queued spinlock number.
|
|
;
|
|
; Return Value:
|
|
;
|
|
; The old IRQL is returned as the function value.
|
|
;
|
|
;--
|
|
|
|
.errnz (LOCK_QUEUE_HEADER_SIZE - 8)
|
|
|
|
align 16
|
|
cPublicFastCall KeAcquireQueuedSpinLock, 1
|
|
cPublicFpo 0,0
|
|
|
|
mov eax, dword ptr APIC[LU_TPR] ; get old IRQL vector
|
|
mov dword ptr APIC[LU_TPR], DPC_VECTOR ; raise IRQL
|
|
jmp short aqsl10 ; finish in common code
|
|
|
|
fstENDP KeAcquireQueuedSpinLock
|
|
|
|
align 16
|
|
cPublicFastCall KeAcquireQueuedSpinLockRaiseToSynch, 1
|
|
cPublicFpo 0,0
|
|
|
|
mov eax, dword ptr APIC[LU_TPR] ; get old IRQL vector
|
|
mov dword ptr APIC[LU_TPR], APIC_SYNCH_VECTOR ; raise IRQL
|
|
aqsl10: shr eax, 4 ; extract high 4 bits of vector
|
|
movzx eax, byte ptr _HalpVectorToIRQL[eax] ; translate to old IRQL
|
|
|
|
;
|
|
; Get address of per processor lock queue entry.
|
|
;
|
|
|
|
ifndef NT_UP
|
|
|
|
mov edx, PCR[PcPrcb] ; get address of PRCB
|
|
lea edx, [edx+ecx*8].PbLockQueue ; get lock queue address
|
|
mov ecx, [edx].LqLock ; get spin lock address
|
|
|
|
ifdef CAPKERN_SYNCH_POINTS
|
|
|
|
push ecx ; lock address
|
|
push 000010101h ; 1 Dword, Timestamp, Subcode = 1
|
|
call _CAP_Log_NInt ;
|
|
add esp, 8 ;
|
|
|
|
endif
|
|
|
|
push eax ; save old IRQL
|
|
|
|
cPublicFpo 0,1
|
|
|
|
if DBG
|
|
|
|
test ecx, LOCK_QUEUE_OWNER + LOCK_QUEUE_WAIT ; inconsistent state?
|
|
jnz short aqsl60 ; if nz, inconsistent state
|
|
|
|
endif
|
|
|
|
;
|
|
; Exchange the value of the lock with the address of the lock context.
|
|
;
|
|
|
|
mov eax, edx ; save lock queue entry address
|
|
xchg [ecx], edx ; exchange lock queue address
|
|
cmp edx, 0 ; check if lock is already held
|
|
jnz short aqsl30 ; if nz, lock already held
|
|
|
|
;
|
|
; N.B. The spin lock address is dword aligned and the bottom two bits are
|
|
; used as indicators.
|
|
;
|
|
; Bit 0 is LOCK_QUEUE_WAIT.
|
|
; Bit 1 is LOCK_QUEUE_OWNER.
|
|
;
|
|
|
|
or [eax].LqLock, LOCK_QUEUE_OWNER ; set lock owner
|
|
aqsl20: pop eax ; restore old IRQL
|
|
|
|
cPublicFpo 0,0
|
|
|
|
endif
|
|
|
|
fstRET KeAcquireQueuedSpinLockRaiseToSynch
|
|
|
|
;
|
|
; The lock is already held by another processor. Set the wait bit in the
|
|
; lock context, then set the next field in the lock context of the last
|
|
; waiter in the lock queue.
|
|
;
|
|
|
|
ifndef NT_UP
|
|
|
|
cPublicFpo 0,1
|
|
|
|
aqsl30: or [eax].LqLock, LOCK_QUEUE_WAIT ; set lock wait
|
|
mov [edx].LqNext, eax ; set next entry in previous last
|
|
|
|
ifdef CAPKERN_SYNCH_POINTS
|
|
|
|
xor edx, edx ; clear wait counter
|
|
aqsl40: inc edx ; count wait time
|
|
test [eax].LqLock, LOCK_QUEUE_WAIT ; check if lock ownership granted
|
|
jz short aqsl50 ; if z, lock owner granted
|
|
|
|
YIELD ; yield to other SMT processors
|
|
|
|
jmp short aqsl40 ;
|
|
|
|
aqsl50: push ecx ; lock address
|
|
push edx ; wait counter
|
|
push 000020104h ; 2 Dwords, Timestamp, Subcode = 4
|
|
call _CAP_Log_NInt ;
|
|
add esp, 12 ;
|
|
jmp short aqsl20 ;
|
|
|
|
else
|
|
|
|
aqsl40: test [eax].LqLock, LOCK_QUEUE_WAIT ; check if lock ownership granted
|
|
jz short aqsl20 ; if z, lock owner granted
|
|
|
|
YIELD ; yield to other SMT processors
|
|
|
|
jmp short aqsl40 ;
|
|
|
|
endif
|
|
|
|
;
|
|
; Inconsistent state in lock queue entry.
|
|
;
|
|
|
|
if DBG
|
|
|
|
cPublicFpo 0,1
|
|
|
|
aqsl60: stdCall _KeBugCheckEx,<SPIN_LOCK_ALREADY_OWNED, ecx, edx, 0, 0>
|
|
int 3 ; so stacktrace works
|
|
|
|
endif
|
|
|
|
endif
|
|
|
|
fstENDP KeAcquireQueuedSpinLockRaiseToSynch
|
|
|
|
SUBTTL "Release Queued SpinLock"
|
|
;++
|
|
;
|
|
; VOID
|
|
; FASTCALL
|
|
; KeReleaseQueuedSpinLock (
|
|
; IN KSPIN_LOCK_QUEUE_NUMBER Number,
|
|
; IN KIRQL OldIrql
|
|
; )
|
|
;
|
|
; Arguments:
|
|
;
|
|
; Number (ecx) - Supplies the queued spinlock number.
|
|
;
|
|
; OldIrql (dl) - Supplies the old IRQL.
|
|
;
|
|
; VOID
|
|
; KeReleaseInStackQueuedSpinLock (
|
|
; IN PKLOCK_QUEUE_HANDLE LockHandle
|
|
; )
|
|
;
|
|
; Arguments:
|
|
;
|
|
; LockHandle (ecx) - Address of Lock Queue Handle structure.
|
|
;
|
|
; Routine Description:
|
|
;
|
|
; These functions release a queued spinlock and lower IRQL to the old
|
|
; value.
|
|
;
|
|
; Return Value:
|
|
;
|
|
; None.
|
|
;
|
|
;--
|
|
|
|
.errnz (LqhNext)
|
|
.errnz (LOCK_QUEUE_OWNER - 2)
|
|
|
|
align 16
|
|
cPublicFastCall KeReleaseInStackQueuedSpinLock, 1
|
|
cPublicFpo 0,0
|
|
|
|
mov dl, byte ptr [ecx].LqhOldIrql ; set old IRQL
|
|
mov eax, ecx ; set lock queue entry address
|
|
jmp short rqsl10 ; finish in common code
|
|
|
|
fstENDP KeReleaseInStackQueuedSpinLock
|
|
|
|
align 16
|
|
cPublicFastCall KeReleaseQueuedSpinLock, 2
|
|
cPublicFpo 0,0
|
|
|
|
ifndef NT_UP
|
|
|
|
mov eax, PCR[PcPrcb] ; get address of PRCB
|
|
lea eax, [eax+ecx*8].PbLockQueue ; set lock queue entry address
|
|
|
|
endif
|
|
|
|
rqsl10: movzx edx, dl ; zero extend old IRQL
|
|
|
|
ifndef NT_UP
|
|
|
|
push ebx ; save nonvolatile register
|
|
|
|
cPublicFpo 0,1
|
|
|
|
mov ebx, [eax].LqNext ; get next entry address
|
|
mov ecx, [eax].LqLock ; get spin lock home address
|
|
|
|
ifdef CAPKERN_SYNCH_POINTS
|
|
|
|
push ecx ; lock address
|
|
push 000010107h ; 1 Dword, Timestamp, Subcode = 7
|
|
call _CAP_Log_NInt ;
|
|
add esp, 8 ;
|
|
|
|
endif
|
|
|
|
;
|
|
; Make sure we own the lock and clear the bit
|
|
;
|
|
|
|
if DBG
|
|
btr ecx, 1 ; clear lock owner bit
|
|
jnc short rqsl80 ; if nc, owner not set
|
|
cmp dword ptr [ecx], 0 ; lock must be owned for a release
|
|
jz short rqsl80
|
|
else
|
|
|
|
and ecx, NOT LOCK_QUEUE_OWNER ; Clear out the owner bit
|
|
|
|
endif
|
|
|
|
|
|
;
|
|
; Test if lock waiter present.
|
|
;
|
|
|
|
test ebx, ebx ; test if lock waiter present
|
|
mov [eax].LqLock, ecx ; clear lock owner bit
|
|
jnz short rqsl40 ; if nz, lock waiter present
|
|
|
|
;
|
|
; Attempt to release queued spin lock.
|
|
;
|
|
|
|
push eax ; save lock queue entry address
|
|
lock cmpxchg [ecx], ebx ; release spin lock if no waiter
|
|
pop eax ; restore lock queue entry address
|
|
jnz short rqsl50 ; if nz, lock waiter present
|
|
rqs120: pop ebx ; restore nonvolatile register
|
|
|
|
cPublicFpo 0,0
|
|
|
|
endif
|
|
|
|
;
|
|
; Lower IRQL to its previous level.
|
|
;
|
|
; N.B. Ensure that the requested priority is set before returning.
|
|
;
|
|
|
|
rqsl30: movzx ecx, byte ptr _HalpIRQLtoTPR[edx] ; translate IRQL to TPR value
|
|
mov dword ptr APIC[LU_TPR], ecx ; lower to old IRQL
|
|
mov eax, dword ptr APIC[LU_TPR] ; synchronize
|
|
|
|
fstRET KeReleaseQueuedSpinLock
|
|
|
|
;
|
|
; Lock waiter is present.
|
|
;
|
|
; Clear wait bit and set owner bit in next owner lock queue entry.
|
|
;
|
|
|
|
ifndef NT_UP
|
|
|
|
cPublicFpo 0,1
|
|
|
|
rqsl40: xor [ebx].LqLock, (LOCK_QUEUE_OWNER+LOCK_QUEUE_WAIT) ; set bits
|
|
and [eax].LqNext, 0 ; clear next waiter address
|
|
jmp short rqs120 ;
|
|
|
|
;
|
|
; Another processor is attempting to acquire the spin lock.
|
|
;
|
|
|
|
ifdef CAPKERN_SYNCH_POINTS
|
|
|
|
rqsl50: push ecx ; lock address (for CAP_Log)
|
|
xor ecx, ecx ; clear wait counter
|
|
rqsl60: inc ecx ; increment wait counter
|
|
mov ebx, [eax].LqNext ; get address of next entry
|
|
test ebx, ebx ; check if waiter present
|
|
jnz short rqsl70 ; if nz, waiter is present
|
|
|
|
YIELD ; yield to other SMT processors
|
|
|
|
jmp short rqsl60 ;
|
|
|
|
rqsl70: push ecx ; wait counter
|
|
push 000020104h ; 2 Dwords, Timestamp, Subcode = 4
|
|
call _CAP_Log_NInt ;
|
|
add esp, 12 ;
|
|
jmp short rqsl40 ;
|
|
|
|
else
|
|
|
|
rqsl50: mov ebx, [eax].LqNext ; get address of next entry
|
|
test ebx, ebx ; check if waiter present
|
|
jnz short rqsl40 ; if nz, waiter is present
|
|
|
|
YIELD ; yield to other SMT processors
|
|
|
|
jmp short rqsl50 ;
|
|
|
|
endif
|
|
|
|
;
|
|
; Inconsistent state in lock queue entry.
|
|
;
|
|
|
|
if DBG
|
|
|
|
cPublicFpo 0,1
|
|
|
|
rqsl80: stdCall _KeBugCheckEx, <SPIN_LOCK_NOT_OWNED, ecx, eax, 0, 1>
|
|
int 3 ; so stacktrace works
|
|
|
|
endif
|
|
|
|
endif
|
|
|
|
fstENDP KeReleaseQueuedSpinLock
|
|
|
|
SUBTTL "Try to Acquire Queued SpinLock"
|
|
|
|
;++
|
|
;
|
|
; LOGICAL
|
|
; KeTryToAcquireQueuedSpinLock (
|
|
; IN KSPIN_LOCK_QUEUE_NUMBER Number,
|
|
; OUT PKIRQL OldIrql
|
|
; )
|
|
;
|
|
; LOGICAL
|
|
; KeTryToAcquireQueuedSpinLockRaiseToSynch (
|
|
; IN KSPIN_LOCK_QUEUE_NUMBER Number,
|
|
; OUT PKIRQL OldIrql
|
|
; )
|
|
;
|
|
; Routine Description:
|
|
;
|
|
; This function raises the current IRQL to DISPATCH/SYNCH level
|
|
; and attempts to acquire the specified queued spinlock. If the
|
|
; spinlock is already owned by another thread, IRQL is restored
|
|
; to its previous value and FALSE is returned.
|
|
;
|
|
; Arguments:
|
|
;
|
|
; Number (ecx) - Supplies the queued spinlock number.
|
|
; OldIrql (edx) - A pointer to the variable to receive the old
|
|
; IRQL.
|
|
;
|
|
; Return Value:
|
|
;
|
|
; TRUE if the lock was acquired, FALSE otherwise.
|
|
; N.B. ZF is set if FALSE returned, clear otherwise.
|
|
;
|
|
;--
|
|
|
|
|
|
align 16
|
|
cPublicFastCall KeTryToAcquireQueuedSpinLockRaiseToSynch,2
|
|
cPublicFpo 0,0
|
|
|
|
push APIC_SYNCH_VECTOR ; raise to SYNCH
|
|
jmp short taqsl10 ; continue in common code
|
|
|
|
fstENDP KeTryToAcquireQueuedSpinLockRaiseToSynch
|
|
|
|
|
|
cPublicFastCall KeTryToAcquireQueuedSpinLock,2
|
|
cPublicFpo 0,0
|
|
|
|
push DPC_VECTOR ; raise to DPC level
|
|
|
|
; Attempt to get the lock with interrupts disabled, raising
|
|
; the priority in the interrupt controller only if acquisition
|
|
; is successful.
|
|
taqsl10:
|
|
|
|
ifndef NT_UP
|
|
|
|
push edx ; save address of OldIrql
|
|
pushfd ; save interrupt state
|
|
cPublicFpo 0,3
|
|
|
|
; Get address of Lock Queue entry
|
|
|
|
cli
|
|
mov edx, PCR[PcPrcb] ; get address of PRCB
|
|
lea edx, [edx+ecx*8].PbLockQueue ; get &PRCB->LockQueue[Number]
|
|
|
|
; Get address of the actual lock.
|
|
|
|
mov ecx, [edx].LqLock
|
|
|
|
ifdef CAPKERN_SYNCH_POINTS
|
|
push ecx ; lock address
|
|
push 000010108h ; 1 Dword, Timestamp, Subcode = 8
|
|
call _CAP_Log_NInt
|
|
add esp, 8
|
|
endif
|
|
|
|
if DBG
|
|
|
|
test ecx, LOCK_QUEUE_OWNER+LOCK_QUEUE_WAIT
|
|
jnz short taqsl98 ; jiff lock already held (or
|
|
; this proc already waiting).
|
|
endif
|
|
|
|
; quick test, get out if already taken
|
|
|
|
cmp dword ptr [ecx], 0 ; check if already taken
|
|
jnz short taqsl60 ; jif already taken
|
|
xor eax, eax ; comparison value (not locked)
|
|
|
|
; Store the Lock Queue entry address in the lock ONLY if the
|
|
; current lock value is 0.
|
|
|
|
lock cmpxchg [ecx], edx
|
|
jnz short taqsl60
|
|
|
|
; Lock has been acquired.
|
|
|
|
; note: the actual lock address will be word aligned, we use
|
|
; the bottom two bits as indicators, bit 0 is LOCK_QUEUE_WAIT,
|
|
; bit 1 is LOCK_QUEUE_OWNER.
|
|
|
|
or ecx, LOCK_QUEUE_OWNER ; mark self as lock owner
|
|
mov [edx].LqLock, ecx
|
|
|
|
mov eax, [esp+8] ; get new IRQL
|
|
mov edx, [esp+4] ; get addr to save OldIrql
|
|
|
|
else
|
|
|
|
mov eax, [esp] ; get new IRQL
|
|
|
|
endif
|
|
|
|
; Raise IRQL and return success.
|
|
|
|
; Get old priority (vector) from Local APIC's Task Priority
|
|
; Register and set the new priority.
|
|
|
|
mov ecx, dword ptr APIC[LU_TPR] ; (ecx) = Old Priority
|
|
mov dword ptr APIC[LU_TPR], eax ; Set New Priority
|
|
|
|
ifndef NT_UP
|
|
|
|
popfd ; restore interrupt state
|
|
add esp, 8 ; free locals
|
|
|
|
else
|
|
|
|
add esp, 4 ; free local
|
|
endif
|
|
|
|
cPublicFpo 0,0
|
|
|
|
shr ecx, 4
|
|
movzx eax, _HalpVectorToIRQL[ecx] ; (al) = OldIrql
|
|
mov [edx], al ; save OldIrql
|
|
xor eax, eax ; return TRUE
|
|
or eax, 1
|
|
|
|
fstRET KeTryToAcquireQueuedSpinLock
|
|
|
|
ifndef NT_UP
|
|
|
|
taqsl60:
|
|
; The lock is already held by another processor. Indicate
|
|
; failure to the caller.
|
|
|
|
popfd ; restore interrupt state
|
|
add esp, 8 ; free locals
|
|
xor eax, eax ; return FALSE
|
|
fstRET KeTryToAcquireQueuedSpinLock
|
|
|
|
if DBG
|
|
|
|
cPublicFpo 0,2
|
|
|
|
taqsl98: stdCall _KeBugCheckEx,<SPIN_LOCK_ALREADY_OWNED,ecx,edx,0,0>
|
|
int 3 ; so stacktrace works
|
|
|
|
endif
|
|
|
|
endif
|
|
|
|
fstENDP KeTryToAcquireQueuedSpinLock
|
|
_TEXT ends
|
|
|
|
end
|