if NT_INST else TITLE "Spin Locks" ;++ ; ; Copyright (c) 1989-1998 Microsoft Corporation ; ; Module Name: ; ; spinlock.asm ; ; Abstract: ; ; This module implements x86 spinlock functions for the PC+MP HAL. ; ; Author: ; ; Bryan Willman (bryanwi) 13 Dec 89 ; ; Environment: ; ; Kernel mode only. ; ; Revision History: ; ; Ron Mosgrove (o-RonMo) Dec 93 - modified for PC+MP HAL. ;-- PAGE .486p include callconv.inc ; calling convention macros include i386\kimacro.inc include hal386.inc include mac386.inc include apic.inc include ntapic.inc EXTRNP _KeBugCheckEx,5,IMPORT EXTRNP KfRaiseIrql, 1,,FASTCALL EXTRNP KfLowerIrql, 1,,FASTCALL EXTRNP _KeSetEventBoostPriority, 2, IMPORT EXTRNP _KeWaitForSingleObject,5, IMPORT extrn _HalpVectorToIRQL:byte extrn _HalpIRQLtoTPR:byte ifdef NT_UP LOCK_ADD equ add LOCK_DEC equ dec LOCK_CMPXCHG equ cmpxchg else LOCK_ADD equ lock add LOCK_DEC equ lock dec LOCK_CMPXCHG equ lock cmpxchg endif _TEXT SEGMENT PARA PUBLIC 'CODE' ASSUME DS:FLAT, ES:FLAT, SS:FLAT, FS:NOTHING, GS:NOTHING PAGE SUBTTL "Acquire Kernel Spin Lock" ;++ ; ; KIRQL ; FASTCALL ; KfAcquireSpinLock ( ; IN PKSPIN_LOCK SpinLock ; ) ; ; Routine Description: ; ; This function raises to DISPATCH_LEVEL and then acquires a the ; kernel spin lock. ; ; Arguments: ; ; (ecx) = SpinLock - Supplies a pointer to a kernel spin lock. ; ; Return Value: ; ; OldIrql (TOS+8) - pointer to place old irql. ; ;-- align 16 cPublicFastCall KfAcquireSpinLock ,1 cPublicFpo 0,0 mov edx, dword ptr APIC[LU_TPR] ; (edx) = Old Priority (Vector) mov dword ptr APIC[LU_TPR], DPC_VECTOR ; Write New Priority to the TPR shr edx, 4 movzx eax, _HalpVectorToIRQL[edx] ; (al) = OldIrql ifndef NT_UP ; ; Attempt to assert the lock ; sl10: ACQUIRE_SPINLOCK ecx, fstRET KfAcquireSpinLock ; ; Lock is owned, spin till it looks free, then go get it again. ; align dword sl20: SPIN_ON_SPINLOCK ecx,sl10 endif fstRET KfAcquireSpinLock fstENDP KfAcquireSpinLock PAGE SUBTTL "Acquire Kernel Spin Lock" ;++ ; ; KIRQL ; FASTCALL ; KeAcquireSpinLockRaiseToSynch ( ; IN PKSPIN_LOCK SpinLock ; ) ; ; Routine Description: ; ; This function acquires the SpinLock at SYNCH_LEVEL. The function ; is optimized for hotter locks (the lock is tested before acquiring, ; and any spin occurs at OldIrql). ; ; Arguments: ; ; (ecx) = SpinLock - Supplies a pointer to a kernel spin lock. ; ; Return Value: ; ; OldIrql (TOS+8) - pointer to place old irql. ; ;-- align 16 cPublicFastCall KeAcquireSpinLockRaiseToSynch,1 cPublicFpo 0,0 mov edx, dword ptr APIC[LU_TPR] ; (ecx) = Old Priority (Vector) mov eax, edx shr eax, 4 movzx eax, _HalpVectorToIRQL[eax] ; (al) = OldIrql ifdef NT_UP mov dword ptr APIC[LU_TPR], APIC_SYNCH_VECTOR ; Write New Priority to the TPR fstRET KeAcquireSpinLockRaiseToSynch else ; ; Test lock ; TEST_SPINLOCK ecx, ; ; Raise irql. ; sls10: mov dword ptr APIC[LU_TPR], APIC_SYNCH_VECTOR ; ; Attempt to assert the lock ; ACQUIRE_SPINLOCK ecx, fstRET KeAcquireSpinLockRaiseToSynch ; ; Lock is owned, spin till it looks free, then go get it ; align dword sls20: mov dword ptr APIC[LU_TPR], edx align dword sls30: SPIN_ON_SPINLOCK ecx,sls10 endif fstENDP KeAcquireSpinLockRaiseToSynch ifndef NT_UP ;++ ; ; KIRQL ; FASTCALL ; KeAcquireSpinLockRaiseToSynchMCE ( ; IN PKSPIN_LOCK SpinLock ; ) ; ; Routine Description: ; ; This function performs the same function as KeAcquireSpinLockRaiseToSynch ; but provides a work around for an IFU errata for Pentium Pro processors ; prior to stepping 619. ; ; Arguments: ; ; (ecx) = SpinLock - Supplies a pointer to a kernel spin lock. ; ; Return Value: ; ; OldIrql (TOS+8) - pointer to place old irql. ; ;-- align 16 cPublicFastCall KeAcquireSpinLockRaiseToSynchMCE,1 cPublicFpo 0,0 mov edx, dword ptr APIC[LU_TPR] ; (ecx) = Old Priority (Vector) mov eax, edx shr eax, 4 movzx eax, _HalpVectorToIRQL[eax] ; (al) = OldIrql ; ; Test lock ; ; TEST_SPINLOCK ecx, ; NOTE - Macro expanded below: test dword ptr [ecx], 1 nop ; On a P6 prior to stepping B1 (619), we nop ; need these 5 NOPs to ensure that we nop ; do not take a machine check exception. nop ; The cost is just 1.5 clocks as the P6 nop ; just tosses the NOPs. jnz short slm30 ; ; Raise irql. ; slm10: mov dword ptr APIC[LU_TPR], APIC_SYNCH_VECTOR ; ; Attempt to assert the lock ; ACQUIRE_SPINLOCK ecx, fstRET KeAcquireSpinLockRaiseToSynchMCE ; ; Lock is owned, spin till it looks free, then go get it ; align dword slm20: mov dword ptr APIC[LU_TPR], edx align dword slm30: SPIN_ON_SPINLOCK ecx,slm10 fstENDP KeAcquireSpinLockRaiseToSynchMCE endif PAGE SUBTTL "Release Kernel Spin Lock" ;++ ; ; VOID ; FASTCALL ; KeReleaseSpinLock ( ; IN PKSPIN_LOCK SpinLock, ; IN KIRQL NewIrql ; ) ; ; Routine Description: ; ; This function releases a kernel spin lock and lowers to the new irql. ; ; Arguments: ; ; (ecx) = SpinLock - Supplies a pointer to a kernel spin lock. ; (dl) = NewIrql - New irql value to set. ; ; Return Value: ; ; None. ; ;-- align 16 cPublicFastCall KfReleaseSpinLock ,2 cPublicFpo 0,0 xor eax, eax mov al, dl ; (eax) = new irql value ifndef NT_UP RELEASE_SPINLOCK ecx ; release spinlock endif xor ecx, ecx mov cl, _HalpIRQLtoTPR[eax] ; get TPR value corresponding to IRQL mov dword ptr APIC[LU_TPR], ecx ; ; We have to ensure that the requested priority is set before ; we return. The caller is counting on it. ; mov eax, dword ptr APIC[LU_TPR] if DBG cmp ecx, eax ; Verify IRQL read back is same as je short @f ; set value int 3 @@: endif fstRET KfReleaseSpinLock fstENDP KfReleaseSpinLock ;++ ; ; KIRQL ; FASTCALL ; HalpAcquireHighLevelLock ( ; IN PKSPIN_LOCK SpinLock ; ) ; ; Routine Description: ; ; Acquires a spinlock with interrupts disabled. ; ; Arguments: ; ; (ecx) = SpinLock - Supplies a pointer to a kernel spin lock. ; ; Return Value: ; ; OldIrql (TOS+8) - pointer to place old irql. ; ;-- cPublicFastCall HalpAcquireHighLevelLock ,1 pushfd pop eax ahll10: cli ACQUIRE_SPINLOCK ecx, ahll20 fstRET HalpAcquireHighLevelLock ahll20: push eax popfd SPIN_ON_SPINLOCK ecx, fstENDP HalpAcquireHighLevelLock ;++ ; ; VOID ; FASTCALL ; HalpReleaseHighLevelLock ( ; IN PKSPIN_LOCK SpinLock, ; IN KIRQL NewIrql ; ) ; ; Routine Description: ; ; This function releases a kernel spin lock and lowers to the new irql. ; ; Arguments: ; ; (ecx) = SpinLock - Supplies a pointer to a kernel spin lock. ; (dl) = NewIrql - New irql value to set. ; ; Return Value: ; ; None. ; ;-- cPublicFastCall HalpReleaseHighLevelLock ,2 RELEASE_SPINLOCK ecx push edx popfd fstRET HalpReleaseHighLevelLock fstENDP HalpReleaseHighLevelLock ;++ ; ; VOID ; FASTCALL ; ExAcquireFastMutex ( ; IN PFAST_MUTEX FastMutex ; ) ; ; Routine description: ; ; This function acquires ownership of the specified FastMutex. ; ; Arguments: ; ; (ecx) = FastMutex - Supplies a pointer to the fast mutex. ; ; Return Value: ; ; None. ; ;-- cPublicFastCall ExAcquireFastMutex,1 cPublicFpo 0,0 mov eax, dword ptr APIC[LU_TPR] ; (eax) = Old Priority (Vector) if DBG ; ; Caller must already be at or below APC_LEVEL. ; cmp eax, APC_VECTOR jg short afm11 ; irql too high ==> fatal. endif mov dword ptr APIC[LU_TPR], APC_VECTOR ; Write New Priority to the TPR LOCK_DEC dword ptr [ecx].FmCount ; Get count jz short afm_ret ; The owner? Yes, Done inc dword ptr [ecx].FmContention cPublicFpo 0,2 push ecx push eax add ecx, FmEvent ; Wait on Event stdCall _KeWaitForSingleObject, pop eax ; (al) = OldTpr pop ecx ; (ecx) = FAST_MUTEX cPublicFpo 0,0 afm_ret: mov byte ptr [ecx].FmOldIrql, al ; ; Use esp to track the owning thread for debugging purposes. ; !thread from kd will find the owning thread. Note that the ; owner isn't cleared on release, check if the mutex is owned ; first. ; mov dword ptr [ecx].FmOwner, esp fstRet ExAcquireFastMutex if DBG cPublicFpo 0,1 afm11: stdCall _KeBugCheckEx, endif fstENDP ExAcquireFastMutex ;++ ; ; VOID ; FASTCALL ; ExReleaseFastMutex ( ; IN PFAST_MUTEX FastMutex ; ) ; ; Routine description: ; ; This function releases ownership of the FastMutex. ; ; Arguments: ; ; (ecx) = FastMutex - Supplies a pointer to the fast mutex. ; ; Return Value: ; ; None. ; ;-- cPublicFastCall ExReleaseFastMutex,1 cPublicFpo 0,0 if DBG ; ; Caller must already be at APC_LEVEL or have APCs blocked. ; mov eax, dword ptr APIC[LU_TPR] ; (eax) = Old Priority (Vector) cmp eax, APC_VECTOR je short rfm04 ; irql is ok. if 0 mov eax, PCR[PcPrcb] mov eax, [eax].PbCurrentThread ; (eax) = Current Thread cmp dword ptr [eax]+ThKernelApcDisable, 0 jne short rfm04 ; APCs disabled, this is ok cmp dword ptr [eax]+ThTeb, 0 je short rfm04 ; No TEB ==> system thread, this is ok test dword ptr [eax]+ThTeb, 080000000h jnz short rfm04 ; TEB in system space, this is ok endif jmp short rfm20 rfm04: endif xor eax, eax mov al, byte ptr [ecx].FmOldIrql ; (eax) = OldTpr LOCK_ADD dword ptr [ecx].FmCount, 1 ; Remove our count js short rfm05 ; if < 0, set event jnz short rfm10 ; if != 0, don't set event cPublicFpo 0,1 rfm05: add ecx, FmEvent push eax ; save new tpr stdCall _KeSetEventBoostPriority, pop eax ; restore tpr cPublicFpo 0,0 rfm10: mov dword ptr APIC[LU_TPR], eax mov ecx, dword ptr APIC[LU_TPR] if DBG cmp eax, ecx ; Verify TPR is what was je short @f ; written int 3 @@: endif fstRet ExReleaseFastMutex if DBG cPublicFpo 0,1 rfm20: stdCall _KeBugCheckEx, endif fstENDP ExReleaseFastMutex ;++ ; ; BOOLEAN ; FASTCALL ; ExTryToAcquireFastMutex ( ; IN PFAST_MUTEX FastMutex ; ) ; ; Routine description: ; ; This function acquires ownership of the FastMutex. ; ; Arguments: ; ; (ecx) = FastMutex - Supplies a pointer to the fast mutex. ; ; Return Value: ; ; Returns TRUE if the FAST_MUTEX was acquired; otherwise FALSE. ; ;-- cPublicFastCall ExTryToAcquireFastMutex,1 cPublicFpo 0,0 if DBG ; ; Caller must already be at or below APC_LEVEL. ; mov eax, dword ptr APIC[LU_TPR] ; (eax) = Old Priority (Vector) cmp eax, APC_VECTOR jg short tam11 ; irql too high ==> fatal. endif ; ; Try to acquire. ; cmp dword ptr [ecx].FmCount, 1 ; Busy? jne short tam25 ; Yes, abort mov eax, dword ptr APIC[LU_TPR] ; (eax) = Old Priority (Vector) mov dword ptr APIC[LU_TPR], APC_VECTOR ; Write New Priority to the TPR cPublicFpo 0,1 push eax ; Save Old TPR mov edx, 0 ; Value to set mov eax, 1 ; Value to compare against LOCK_CMPXCHG dword ptr [ecx].FmCount, edx ; Attempt to acquire jnz short tam20 ; got it? cPublicFpo 0,0 pop edx ; (edx) = Old TPR mov eax, 1 ; return TRUE mov byte ptr [ecx].FmOldIrql, dl ; Store Old TPR fstRet ExTryToAcquireFastMutex tam20: pop ecx ; (ecx) = Old TPR mov dword ptr APIC[LU_TPR], ecx mov eax, dword ptr APIC[LU_TPR] if DBG cmp ecx, eax ; Verify TPR is what was je short @f ; written int 3 @@: endif tam25: xor eax, eax ; return FALSE YIELD fstRet ExTryToAcquireFastMutex ; all done if DBG cPublicFpo 0,1 tam11: stdCall _KeBugCheckEx, endif fstENDP ExTryToAcquireFastMutex page ,132 subttl "Acquire Queued SpinLock Raise to Synch" ; compile time assert sizeof(KSPIN_LOCK_QUEUE) == 8 .errnz (LOCK_QUEUE_HEADER_SIZE - 8) align 16 ;++ ; ; VOID ; KeAcquireInStackQueuedSpinLock ( ; IN PKSPIN_LOCK SpinLock, ; IN PKLOCK_QUEUE_HANDLE LockHandle ; ) ; ; VOID ; KeAcquireInStackQueuedSpinLockRaiseToSynch ( ; IN PKSPIN_LOCK SpinLock, ; IN PKLOCK_QUEUE_HANDLE LockHandle ; ) ; ; Routine Description: ; ; KeAcquireInStackQueuedSpinLock... ; ; The Kx versions use a LOCK_QUEUE_HANDLE structure rather than ; LOCK_QUEUE structures in the PRCB. Old IRQL is stored in the ; LOCK_QUEUE_HANDLE. ; ; Arguments: ; ; SpinLock (ecx) Address of Actual Lock. ; LockHandle (edx) Address of lock context. ; ; Return Value: ; ; None. Actually returns OldIrql because common code is used ; for all implementations. ; ;-- cPublicFastCall KeAcquireInStackQueuedSpinLockRaiseToSynch,2 cPublicFpo 0,0 ifdef NT_UP ; In the Uniprocessor case, just raise IRQL to SYNCH mov eax, dword ptr APIC[LU_TPR] ; (eax) = Old Priority shr eax, 4 mov al, byte ptr _HalpVectorToIRQL[eax] ; (al) = OldIrql mov dword ptr APIC[LU_TPR], APIC_SYNCH_VECTOR mov [edx].LqhOldIrql, al ; save old IRQL in lock handle fstRET KeAcquireInStackQueuedSpinLockRaiseToSynch else ; MP case, use KeAcquireInStackQueuedSpinLock to get the lock and raise ; to SYNCH asap afterwards. call @KeAcquireInStackQueuedSpinLock@8 mov dword ptr APIC[LU_TPR], APIC_SYNCH_VECTOR fstRET KeAcquireInStackQueuedSpinLockRaiseToSynch endif fstENDP KeAcquireInStackQueuedSpinLockRaiseToSynch cPublicFastCall KeAcquireInStackQueuedSpinLock,2 cPublicFpo 0,0 ; Raise IRQL to DISPATCH_LEVEL mov eax, dword ptr APIC[LU_TPR] ; (eax) = Old Priority shr eax, 4 mov al, byte ptr _HalpVectorToIRQL[eax] ; (al) = OldIrql mov dword ptr APIC[LU_TPR], DPC_VECTOR ; raise to DISPATCH level mov [edx].LqhOldIrql, al ; save old IRQL in lock handle ifdef NT_UP fstRET KeAcquireInStackQueuedSpinLock else ; Save actual lock address in lock handle. mov [edx].LqhLock, ecx mov dword ptr [edx].LqhNext, 0 ; ecx contains the address of the actual lock ; and edx the address of a queued lock structure. cPublicFpo 0,1 mov eax, edx ; save Lock Queue entry address if DBG test ecx, LOCK_QUEUE_OWNER+LOCK_QUEUE_WAIT jnz short iqsl98 ; jiff lock already held (or ; this proc already waiting). endif ; Exchange the value of the lock with the address of this ; Lock Queue entry. xchg [ecx], edx cmp edx, 0 ; check if lock is held jnz short iqsl40 ; jiff held ; lock has been acquired. cPublicFpo 0,0 ; note: the actual lock address is word aligned, we use ; the bottom two bits as indicators, bit 0 is LOCK_QUEUE_WAIT, ; bit 1 is LOCK_QUEUE_OWNER. or ecx, LOCK_QUEUE_OWNER ; mark self as lock owner mov [eax].LqLock, ecx iqsl20: fstRET KeAcquireInStackQueuedSpinLock cPublicFpo 0,1 iqsl40: ; The lock is already held by another processor. Set the wait ; bit in this processor's Lock Queue entry, then set the next ; field in the Lock Queue entry of the last processor to attempt ; to acquire the lock (this is the address returned by the xchg ; above) to point to THIS processor's lock queue entry. or ecx, LOCK_QUEUE_WAIT ; set waiting bit mov [eax].LqLock, ecx mov [edx].LqNext, eax ; set previous acquirer's ; next field. ; Wait. @@: test [eax].LqLock, LOCK_QUEUE_WAIT ; check if still waiting jz short iqsl20 ; jif lock acquired YIELD ; fire avoidance. jmp short @b ; else, continue waiting ; Wait. if DBG cPublicFpo 0,1 iqsl98: stdCall _KeBugCheckEx, int 3 ; so stacktrace works endif endif fstENDP KeAcquireInStackQueuedSpinLock ;++ ; ; KIRQL ; KeAcquireQueuedSpinLockRaiseToSynch ( ; IN KSPIN_LOCK_QUEUE_NUMBER Number ; ) ; ; Routine Description: ; ; This function raises the current IRQL to DISPATCH/SYNCH level ; and acquires the specified queued spinlock. ; ; Arguments: ; ; Number (ecx) - Supplies the queued spinlock number. ; ; Return Value: ; ; The previous IRQL is returned as the function value. ; ;-- cPublicFastCall KeAcquireQueuedSpinLockRaiseToSynch,1 cPublicFpo 0,0 ifdef NT_UP ; In the Uniprocessor case, just raise IRQL to SYNCH mov eax, dword ptr APIC[LU_TPR] ; (eax) = Old Priority shr eax, 4 movzx eax, byte ptr _HalpVectorToIRQL[eax] ; (al) = OldIrql mov dword ptr APIC[LU_TPR], APIC_SYNCH_VECTOR fstRET KeAcquireQueuedSpinLockRaiseToSynch else ; MP case, use KeAcquireQueuedSpinLock to get the lock and raise ; to SYNCH asap afterwards. call @KeAcquireQueuedSpinLock@4 mov dword ptr APIC[LU_TPR], APIC_SYNCH_VECTOR fstRET KeAcquireQueuedSpinLockRaiseToSynch endif fstENDP KeAcquireQueuedSpinLockRaiseToSynch page ,132 subttl "Acquire Queued SpinLock" ;++ ; ; KIRQL ; KeAcquireQueuedSpinLock ( ; IN KSPIN_LOCK_QUEUE_NUMBER Number ; ) ; ; Routine Description: ; ; This function raises the current IRQL to DISPATCH level ; and acquires the specified queued spinlock. ; ; Arguments: ; ; Number (ecx) - Supplies the queued spinlock number. ; ; Return Value: ; ; The previous IRQL is returned as the function value. ; ;-- cPublicFastCall KeAcquireQueuedSpinLock,1 cPublicFpo 0,0 ; Get old priority (vector) from Local APIC's Task Priority ; Register and set the new priority. mov eax, dword ptr APIC[LU_TPR] ; (eax) = Old Priority shr eax, 4 movzx eax, byte ptr _HalpVectorToIRQL[eax] ; (al) = OldIrql mov dword ptr APIC[LU_TPR], DPC_VECTOR ; raise to DISPATCH level ifdef NT_UP ; in the Uniprocessor version all we do is raise IRQL. fstRET KeAcquireQueuedSpinLock else ; Get address of Lock Queue entry mov edx, PCR[PcPrcb] ; get address of PRCB lea edx, [edx+ecx*8].PbLockQueue ; get &PRCB->LockQueue[Number] ; Get address of the actual lock. mov ecx, [edx].LqLock aqsl10: push eax ; save old IRQL cPublicFpo 0,1 mov eax, edx ; save Lock Queue entry address if DBG test ecx, LOCK_QUEUE_OWNER+LOCK_QUEUE_WAIT jnz short aqsl98 ; jiff lock already held (or ; this proc already waiting). endif ; Exchange the value of the lock with the address of this ; Lock Queue entry. xchg [ecx], edx cmp edx, 0 ; check if lock is held jnz short aqsl40 ; jiff held ; lock has been acquired. cPublicFpo 0,0 ; note: the actual lock address is word aligned, we use ; the bottom two bits as indicators, bit 0 is LOCK_QUEUE_WAIT, ; bit 1 is LOCK_QUEUE_OWNER. or ecx, LOCK_QUEUE_OWNER ; mark self as lock owner mov [eax].LqLock, ecx aqsl20: pop eax ; return old IRQL fstRET KeAcquireQueuedSpinLock cPublicFpo 0,1 aqsl40: ; The lock is already held by another processor. Set the wait ; bit in this processor's Lock Queue entry, then set the next ; field in the Lock Queue entry of the last processor to attempt ; to acquire the lock (this is the address returned by the xchg ; above) to point to THIS processor's lock queue entry. or ecx, LOCK_QUEUE_WAIT ; set waiting bit mov [eax].LqLock, ecx mov [edx].LqNext, eax ; set previous acquirer's ; next field. cPublicFpo 0,0 ; Wait. @@: test [eax].LqLock, LOCK_QUEUE_WAIT ; check if still waiting jz short aqsl20 ; jif lock acquired YIELD ; fire avoidance. jmp short @b ; else, continue waiting if DBG cPublicFpo 0,1 aqsl98: stdCall _KeBugCheckEx, int 3 ; so stacktrace works endif endif fstENDP KeAcquireQueuedSpinLock page ,132 subttl "Release Queued SpinLock" ;++ ; ; VOID ; KeReleaseInStackQueuedSpinLock ( ; IN PKLOCK_QUEUE_HANDLE LockHandle ; ) ; ; Routine Description: ; ; This function releases a queued spinlock and lowers the IRQL to ; its previous value. ; ; This differs from KeReleaseQueuedSpinLock in that this version ; uses a caller supplied lock context where that one uses a ; predefined lock context in the processor's PRCB. ; ; This version sets up a compatible register context and uses ; KeReleaseQueuedSpinLock to do the actual work. ; ; Arguments: ; ; LockHandle (ecx) - Address of Lock Queue Handle structure. ; ; Return Value: ; ; None. ; ;-- cPublicFastCall KeReleaseInStackQueuedSpinLock,1 cPublicFpo 0,0 movzx edx, byte ptr [ecx].LqhOldIrql ; get old irql ifndef NT_UP lea eax, [ecx].LqhNext ; get address of lock struct jmp short rqsl10 ; continue in common code else ; Set the IO APIC's Task Priority Register to the value ; corresponding to the new IRQL. movzx ecx, byte ptr _HalpIRQLtoTPR[edx] mov dword ptr APIC[LU_TPR], ecx ; Ensure that the requested priority is set before returning, ; the caller is counting on it. mov eax, dword ptr APIC[LU_TPR] if DBG cmp ecx, eax ; Verify IRQL read back is same as je short @f ; set value int 3 @@: endif fstRET KeReleaseInStackQueuedSpinLock endif fstENDP KeReleaseInStackQueuedSpinLock ;++ ; ; VOID ; KeReleaseQueuedSpinLock ( ; IN KSPIN_LOCK_QUEUE_NUMBER Number, ; IN KIRQL OldIrql ; ) ; ; Routine Description: ; ; This function releases a queued spinlock and lowers the IRQL to ; its previous value. ; ; Arguments: ; ; Number (ecx) - Supplies the queued spinlock number. ; OldIrql (dl) - Supplies the IRQL value to lower to. ; ; Return Value: ; ; None. ; ;-- cPublicFastCall KeReleaseQueuedSpinLock,2 cPublicFpo 0,0 .errnz (LOCK_QUEUE_OWNER - 2) ; error if not bit 1 for btr ifndef NT_UP ; Get address of Lock Queue entry mov eax, PCR[PcPrcb] ; get address of PRCB endif movzx edx, dl ; Irql = 8 bits from edx ifndef NT_UP lea eax, [eax+ecx*8].PbLockQueue ; get &PRCB->LockQueue[Number] rqsl10: push ebx ; need another register cPublicFpo 0,1 ; Clear the lock field in the Lock Queue entry. mov ebx, [eax].LqNext mov ecx, [eax].LqLock ; Quick check: If Lock Queue entry's Next field is not NULL, ; there is another waiter. Don't bother with ANY atomic ops ; in this case. ; ; Note: test clears CF and sets ZF appropriately, the following ; btr sets CF appropriately for the owner check. test ebx, ebx ; clear the "I am owner" bit in the Lock entry. btr ecx, 1 ; clear owner bit. if DBG jnc short rqsl98 ; bugcheck if was not set ; tests CF endif mov [eax].LqLock, ecx ; clear lock bit in queue entry jnz short rqsl40 ; jif another processor waits ; tests ZF ; ebx contains zero here which will be used to set the new owner NULL push eax ; save &PRCB->LockQueue[Number] cPublicFpo 0,2 ; Use compare exchange to attempt to clear the actual lock. ; If there are still no processors waiting for the lock when ; the compare exchange happens, the old contents of the lock ; should be the address of this lock entry (eax). lock cmpxchg [ecx], ebx ; store 0 if no waiters pop eax ; restore lock queue address cPublicFpo 0,1 jnz short rqsl60 ; jif store failed ; The lock has been released. Lower IRQL and return to caller. endif rqsl20: ; Set the IO APIC's Task Priority Register to the value ; corresponding to the new IRQL. movzx ecx, byte ptr _HalpIRQLtoTPR[edx] ifndef NT_UP pop ebx ; restore ebx cPublicFpo 0,0 endif mov dword ptr APIC[LU_TPR], ecx ; Ensure that the requested priority is set before returning, ; the caller is counting on it. mov eax, dword ptr APIC[LU_TPR] if DBG cmp ecx, eax ; Verify IRQL read back is same as je short @f ; set value int 3 @@: endif fstRET KeReleaseQueuedSpinLock ifndef NT_UP cPublicFpo 0,1 ; Another processor is waiting on this lock. Hand the lock ; to that processor by getting the address of its LockQueue ; entry, turning ON its owner bit and OFF its wait bit. rqsl40: xor [ebx].LqLock, (LOCK_QUEUE_OWNER+LOCK_QUEUE_WAIT) ; Done, the other processor now owns the lock, clear the next ; field in my LockQueue entry (to preserve the order for entering ; the queue again) and proceed to lower IRQL and return. mov [eax].LqNext, 0 jmp short rqsl20 ; We get here if another processor is attempting to acquire ; the lock but had not yet updated the next field in this ; processor's Queued Lock Next field. Wait for the next ; field to be updated. rqsl60: mov ebx, [eax].LqNext test ebx, ebx ; check if still 0 jnz short rqsl40 ; jif Next field now set. YIELD ; wait a bit jmp short rqsl60 ; continue waiting if DBG cPublicFpo 0,1 rqsl98: stdCall _KeBugCheckEx, int 3 ; so stacktrace works endif endif fstENDP KeReleaseQueuedSpinLock page ,132 subttl "Try to Acquire Queued SpinLock" ;++ ; ; LOGICAL ; KeTryToAcquireQueuedSpinLock ( ; IN KSPIN_LOCK_QUEUE_NUMBER Number, ; OUT PKIRQL OldIrql ; ) ; ; LOGICAL ; KeTryToAcquireQueuedSpinLockRaiseToSynch ( ; IN KSPIN_LOCK_QUEUE_NUMBER Number, ; OUT PKIRQL OldIrql ; ) ; ; Routine Description: ; ; This function raises the current IRQL to DISPATCH/SYNCH level ; and attempts to acquire the specified queued spinlock. If the ; spinlock is already owned by another thread, IRQL is restored ; to its previous value and FALSE is returned. ; ; Arguments: ; ; Number (ecx) - Supplies the queued spinlock number. ; OldIrql (edx) - A pointer to the variable to receive the old ; IRQL. ; ; Return Value: ; ; TRUE if the lock was acquired, FALSE otherwise. ; N.B. ZF is set if FALSE returned, clear otherwise. ; ;-- align 16 cPublicFastCall KeTryToAcquireQueuedSpinLockRaiseToSynch,2 cPublicFpo 0,0 push APIC_SYNCH_VECTOR ; raise to SYNCH jmp short taqsl10 ; continue in common code fstENDP KeTryToAcquireQueuedSpinLockRaiseToSynch cPublicFastCall KeTryToAcquireQueuedSpinLock,2 cPublicFpo 0,0 push DPC_VECTOR ; raise to DPC level ; Attempt to get the lock with interrupts disabled, raising ; the priority in the interrupt controller only if acquisition ; is successful. taqsl10: ifndef NT_UP push edx ; save address of OldIrql pushfd ; save interrupt state cPublicFpo 0,3 ; Get address of Lock Queue entry cli mov edx, PCR[PcPrcb] ; get address of PRCB lea edx, [edx+ecx*8].PbLockQueue ; get &PRCB->LockQueue[Number] ; Get address of the actual lock. mov ecx, [edx].LqLock if DBG test ecx, LOCK_QUEUE_OWNER+LOCK_QUEUE_WAIT jnz short taqsl98 ; jiff lock already held (or ; this proc already waiting). endif ; quick test, get out if already taken cmp dword ptr [ecx], 0 ; check if already taken jnz short taqsl60 ; jif already taken xor eax, eax ; comparison value (not locked) ; Store the Lock Queue entry address in the lock ONLY if the ; current lock value is 0. lock cmpxchg [ecx], edx jnz short taqsl60 ; Lock has been acquired. ; note: the actual lock address will be word aligned, we use ; the bottom two bits as indicators, bit 0 is LOCK_QUEUE_WAIT, ; bit 1 is LOCK_QUEUE_OWNER. or ecx, LOCK_QUEUE_OWNER ; mark self as lock owner mov [edx].LqLock, ecx mov eax, [esp+8] ; get new IRQL mov edx, [esp+4] ; get addr to save OldIrql else mov eax, [esp] ; get new IRQL endif ; Raise IRQL and return success. ; Get old priority (vector) from Local APIC's Task Priority ; Register and set the new priority. mov ecx, dword ptr APIC[LU_TPR] ; (ecx) = Old Priority mov dword ptr APIC[LU_TPR], eax ; Set New Priority ifndef NT_UP popfd ; restore interrupt state add esp, 8 ; free locals else add esp, 4 ; free local endif cPublicFpo 0,0 shr ecx, 4 movzx eax, _HalpVectorToIRQL[ecx] ; (al) = OldIrql mov [edx], al ; save OldIrql xor eax, eax ; return TRUE or eax, 1 fstRET KeTryToAcquireQueuedSpinLock ifndef NT_UP taqsl60: ; The lock is already held by another processor. Indicate ; failure to the caller. popfd ; restore interrupt state add esp, 8 ; free locals xor eax, eax ; return FALSE fstRET KeTryToAcquireQueuedSpinLock if DBG cPublicFpo 0,2 taqsl98: stdCall _KeBugCheckEx, int 3 ; so stacktrace works endif endif fstENDP KeTryToAcquireQueuedSpinLock _TEXT ends ENDIF ; NT_INST end