title "Irql Processing" ;++ ; ; Copyright (c) 1989 Microsoft Corporation ; ; Module Name: ; ; ixlock.asm ; ; Abstract: ; ; This module implements various locking functions optimized for this hal. ; ; Author: ; ; Ken Reneris (kenr) 21-April-1994 ; ; Environment: ; ; Kernel mode only. ; ; Revision History: ; ;-- .486p .xlist include hal386.inc include callconv.inc ; calling convention macros include i386\ix8259.inc include i386\kimacro.inc include mac386.inc .list EXTRNP _KeBugCheckEx,5,IMPORT EXTRNP _KeSetEventBoostPriority, 2, IMPORT EXTRNP _KeWaitForSingleObject,5, IMPORT extrn FindHigherIrqlMask:DWORD extrn SWInterruptHandlerTable:DWORD EXTRNP _KeRaiseIrql,2 EXTRNP _KeLowerIrql,1 ifdef NT_UP LOCK_ADD equ add LOCK_DEC equ dec LOCK_CMPXCHG equ cmpxchg else LOCK_ADD equ lock add LOCK_DEC equ lock dec LOCK_CMPXCHG equ lock cmpxchg endif _TEXT$01 SEGMENT PARA PUBLIC 'CODE' ASSUME DS:FLAT, ES:FLAT, SS:FLAT, FS:NOTHING, GS:NOTHING PAGE subttl "AcquireSpinLock" ;++ ; ; KIRQL ; KfAcquireSpinLock ( ; IN PKSPIN_LOCK SpinLock ; ) ; ; Routine Description: ; ; This function raises to DISPATCH_LEVEL and then acquires a the ; kernel spin lock. ; ; In a UP hal spinlock serialization is accomplished by raising the ; IRQL to DISPATCH_LEVEL. The SpinLock is not used; however, for ; debugging purposes if the UP hal is compiled with the NT_UP flag ; not set (ie, MP) we take the SpinLock. ; ; Arguments: ; ; (ecx) = SpinLock Supplies a pointer to an kernel spin lock. ; ; Return Value: ; ; OldIrql ; ;-- cPublicFastCall KfAcquireSpinLock,1 cPublicFpo 0,0 mov eax, PCR[PcIrql] ; (eax) = Old Irql mov dword ptr PCR[PcIrql], DISPATCH_LEVEL ; set new irql ifndef NT_UP asl10: ACQUIRE_SPINLOCK ecx, endif ifdef IRQL_METRICS inc HalRaiseIrqlCount endif if DBG cmp al, DISPATCH_LEVEL ; old > new? ja short asl99 ; yes, go bugcheck endif fstRET KfAcquireSpinLock ifndef NT_UP asl20: SPIN_ON_SPINLOCK ecx, endif if DBG cPublicFpo 2,1 asl99: movzx eax, al stdCall _KeBugCheckEx, ; never returns endif fstRET KfAcquireSpinLock fstENDP KfAcquireSpinLock ;++ ; ; KIRQL ; KeAcquireSpinLockRaiseToSynch ( ; IN PKSPIN_LOCK SpinLock ; ) ; ; Routine Description: ; ; This function acquires the SpinLock at SYNCH_LEVEL. The function ; is optmized for hoter locks (the lock is tested before acquired. ; Any spin should occur at OldIrql; however, since this is a UP hal ; we don't have the code for it) ; ; In a UP hal spinlock serialization is accomplished by raising the ; IRQL to SYNCH_LEVEL. The SpinLock is not used; however, for ; debugging purposes if the UP hal is compiled with the NT_UP flag ; not set (ie, MP) we take the SpinLock. ; ; Arguments: ; ; (ecx) = SpinLock Supplies a pointer to an kernel spin lock. ; ; Return Value: ; ; OldIrql ; ;-- cPublicFastCall KeAcquireSpinLockRaiseToSynch,1 cPublicFpo 0,0 mov eax, PCR[PcIrql] ; (eax) = Old Irql mov dword ptr PCR[PcIrql], SYNCH_LEVEL ; set new irql ifndef NT_UP asls10: ACQUIRE_SPINLOCK ecx, endif ifdef IRQL_METRICS inc HalRaiseIrqlCount endif if DBG cmp al, SYNCH_LEVEL ; old > new? ja short asls99 ; yes, go bugcheck endif fstRET KeAcquireSpinLockRaiseToSynch ifndef NT_UP asls20: SPIN_ON_SPINLOCK ecx, endif if DBG cPublicFpo 2,1 asls99: movzx eax, al stdCall _KeBugCheckEx, ; never returns endif fstRET KeAcquireSpinLockRaiseToSynch fstENDP KeAcquireSpinLockRaiseToSynch PAGE SUBTTL "Release Kernel Spin Lock" ;++ ; ; VOID ; KfReleaseSpinLock ( ; IN PKSPIN_LOCK SpinLock, ; IN KIRQL NewIrql ; ) ; ; Routine Description: ; ; This function releases a kernel spin lock and lowers to the new irql ; ; In a UP hal spinlock serialization is accomplished by raising the ; IRQL to DISPATCH_LEVEL. The SpinLock is not used; however, for ; debugging purposes if the UP hal is compiled with the NT_UP flag ; not set (ie, MP) we use the SpinLock. ; ; Arguments: ; ; (ecx) = SpinLock Supplies a pointer to an executive spin lock. ; (dl) = NewIrql New irql value to set ; ; Return Value: ; ; None. ; ;-- align 16 cPublicFastCall KfReleaseSpinLock ,2 cPublicFpo 0,0 ifndef NT_UP RELEASE_SPINLOCK ecx ; release it endif xor ecx, ecx if DBG cmp dl, PCR[PcIrql] ja short rsl99 endif pushfd cli movzx edx, dl mov PCR[PcIrql], edx ; store old irql mov cl, dl ; (ecx) = 32bit extended OldIrql mov edx, PCR[PcIRR] and edx, FindHigherIrqlMask[ecx*4] ; (edx) is the bitmask of ; pending interrupts we need to jne short rsl20 ; dispatch now. popfd fstRet KfReleaseSpinLock ; all done if DBG rsl99: mov eax, PCR[PcIrql] movzx edx, dl stdCall _KeBugCheckEx, ; never returns endif cPublicFpo 0,1 rsl20: bsr ecx, edx ; (ecx) = Pending irq level cmp ecx, DISPATCH_LEVEL jle short rsl40 mov eax, PCR[PcIDR] ; Clear all the interrupt SET_8259_MASK ; masks rsl40: mov edx, 1 shl edx, cl xor PCR[PcIRR], edx ; clear bit in IRR call SWInterruptHandlerTable[ecx*4] ; Dispatch the pending int. popfd cPublicFpo 0, 0 fstRet KfReleaseSpinLock ; all done fstENDP KfReleaseSpinLock ;++ ; ; VOID ; FASTCALL ; ExAcquireFastMutex ( ; IN PFAST_MUTEX FastMutex ; ) ; ; Routine description: ; ; This function acquires ownership of the FastMutex ; ; Arguments: ; ; (ecx) = FastMutex - Supplies a pointer to the fast mutex ; ; Return Value: ; ; None. ; ;-- cPublicFastCall ExAcquireFastMutex,1 cPublicFpo 0,0 mov eax, PCR[PcIrql] ; (eax) = OldIrql if DBG cmp eax, APC_LEVEL ; Is OldIrql > NewIrql? ja short afm99 ; Yes, bugcheck pushfd cli ; prevent swapcontext while ; snapping current thread. mov edx, PCR[PcPrcb] mov edx, [edx].PbCurrentThread ; (edx) = Current Thread popfd cmp [ecx].FmOwner, edx ; Already owned by this thread? je short afm98 ; Yes, error endif mov dword ptr PCR[PcIrql], APC_LEVEL ; Set NewIrql LOCK_DEC dword ptr [ecx].FmCount ; Get count jnz short afm10 ; Not the owner, go wait. ; ; Leave a notion of owner behind. ; ; Note: if you change this, change ExAcquireFastMutexUnsafe. ; if DBG pushfd cli ; prevent swapcontext while ; snapping current thread. mov edx, PCR[PcPrcb] mov edx, [edx].PbCurrentThread ; (edx) = Current Thread popfd mov [ecx].FmOwner, edx ; Save in Fast Mutex else ; ; Use esp to track the owning thread for debugging purposes. ; !thread from kd will find the owning thread. Note that the ; owner isn't cleared on release, check if the mutex is owned ; first. ; mov [ecx].FmOwner, esp endif mov dword ptr [ecx].FmOldIrql, eax fstRet ExAcquireFastMutex afm10: inc dword ptr [ecx].FmContention cPublicFpo 0,2 push eax ; save OldIrql push ecx ; Save FAST_MUTEX add ecx, FmEvent ; Wait on Event stdCall _KeWaitForSingleObject, pop ecx ; (ecx) = FAST_MUTEX pop eax ; (al) = OldIrql cPublicFpo 1,0 afm_ret: ; ; Leave a notion of owner behind. ; ; Note: if you change this, change ExAcquireFastMutexUnsafe. ; if DBG pushfd cli ; prevent swapcontext while ; snapping current thread. mov edx, PCR[PcPrcb] mov edx, [edx].PbCurrentThread ; (edx) = Current Thread popfd mov [ecx].FmOwner, edx ; Save in Fast Mutex else ; ; Use esp to track the owning thread for debugging purposes. ; !thread from kd will find the owning thread. Note that the ; owner isn't cleared on release, check if the mutex is owned ; first. ; mov [ecx].FmOwner, esp endif mov byte ptr [ecx].FmOldIrql, al fstRet ExAcquireFastMutex if DBG ; KeBugCheckEx(MUTEX_ALREADY_OWNED, FastMutex, CurrentThread, 0, 4) ; (never returns) afm98: stdCall _KeBugCheckEx, ; KeBugCheckEx(IRQL_NOT_LESS_OR_EQUAL, CurrentIrql, APC_LEVEL, 0, 5) ; (never returns) afm99: movzx eax, al stdCall _KeBugCheckEx, fstRet ExAcquireFastMutex endif fstENDP ExAcquireFastMutex ;++ ; ; VOID ; FASTCALL ; ExReleaseFastMutex ( ; IN PFAST_MUTEX FastMutex ; ) ; ; Routine description: ; ; This function releases ownership of the FastMutex ; ; Arguments: ; ; (ecx) = FastMutex - Supplies a pointer to the fast mutex ; ; Return Value: ; ; None. ; ;-- cPublicFastCall ExReleaseFastMutex,1 cPublicFpo 0,0 xor eax, eax if DBG pushfd cli ; prevent swapcontext while ; snapping current thread. mov edx, PCR[PcPrcb] mov edx, [edx].PbCurrentThread ; (edx) = CurrentThread popfd cmp [ecx].FmOwner, edx ; Owner == CurrentThread? jne short rfm_threaderror ; No, bugcheck or byte ptr [ecx].FmOwner, 1 ; not the owner anymore endif mov al, byte ptr [ecx].FmOldIrql ; (eax) = OldIrql LOCK_ADD dword ptr [ecx].FmCount, 1 ; Remove our count js short rfm05 ; if < 0, set event jnz short rfm10 ; if != 0, don't set event rfm05: cPublicFpo 0,2 push eax ; Save OldIrql add ecx, FmEvent stdCall _KeSetEventBoostPriority, pop eax cPublicFpo 0,0 rfm10: cli mov PCR[PcIrql], eax mov edx, PCR[PcIRR] and edx, FindHigherIrqlMask[eax*4] ; (edx) is the bitmask of ; pending interrupts we need to jne short rfm20 ; dispatch now. sti fstRet ExReleaseFastMutex ; all done if DBG ; KeBugCheck(THREAD_NOT_MUTEX_OWNER, FastMutex, Thread, Owner, 6) ; (never returns) rfm_threaderror: stdCall _KeBugCheckEx, endif rfm20: bsr ecx, edx ; (ecx) = Pending irq level cmp ecx, DISPATCH_LEVEL jle short rfm40 mov eax, PCR[PcIDR] ; Clear all the interrupt SET_8259_MASK ; masks rfm40: mov edx, 1 shl edx, cl xor PCR[PcIRR], edx ; clear bit in IRR call SWInterruptHandlerTable[ecx*4] ; Dispatch the pending int. sti fstRet ExReleaseFastMutex ; all done fstENDP ExReleaseFastMutex ;++ ; ; BOOLEAN ; FASTCALL ; ExTryToAcquireFastMutex ( ; IN PFAST_MUTEX FastMutex ; ) ; ; Routine description: ; ; This function acquires ownership of the FastMutex ; ; Arguments: ; ; (ecx) = FastMutex - Supplies a pointer to the fast mutex ; ; Return Value: ; ; Returns TRUE if the FAST_MUTEX was acquired; otherwise false ; ;-- cPublicFastCall ExTryToAcquireFastMutex,1 cPublicFpo 0,0 if DBG mov eax, PCR[PcIrql] ; (al) = OldIrql cmp eax, APC_LEVEL ; Is OldIrql > NewIrql? ja short tam99 ; Yes, bugcheck push eax ; Save OldIrql else push dword ptr PCR[PcIrql] ; Save OldIrql endif ; ; Try to acquire. ; mov edx, 0 ; Value to set mov eax, 1 ; Value to compare against mov dword ptr PCR[PcIrql], APC_LEVEL LOCK_CMPXCHG dword ptr [ecx].FmCount, edx ; Attempt to acquire jnz short tam20 ; got it? pop dword ptr [ecx].FmOldIrql ; Store OldIrql if DBG pushfd cli ; prevent swapcontext while ; snapping current thread. mov edx, PCR[PcPrcb] mov edx, [edx].PbCurrentThread ; (edx) = Current Thread popfd mov [ecx].FmOwner, edx ; Save in Fast Mutex else ; ; Use esp to track the owning thread for debugging purposes. ; !thread from kd will find the owning thread. Note that the ; owner isn't cleared on release, check if the mutex is owned ; first. ; mov dword ptr [ecx].FmOwner, esp endif mov eax, 1 ; return TRUE fstRet ExTryToAcquireFastMutex tam20: pop dword ptr PCR[PcIrql] ; restore entry Irql YIELD xor eax, eax ; return FALSE fstRet ExTryToAcquireFastMutex ; all done if DBG ; KeBugCheckEx(IRQL_NOT_LESS_OR_EQUAL, CurrentIrql, APC_LEVEL, 0, 5) ; (never returns) tam99: movzx eax, al stdCall _KeBugCheckEx, xor eax, eax ; return FALSE fstRet ExTryToAcquireFastMutex endif fstENDP ExTryToAcquireFastMutex page ,132 subttl "Acquire Queued SpinLock" ;++ ; ; KIRQL ; KeAcquireQueuedSpinLock ( ; IN KSPIN_LOCK_QUEUE_NUMBER Number ; ) ; ; KIRQL ; KeAcquireQueuedSpinLockRaiseToSynch ( ; IN KSPIN_LOCK_QUEUE_NUMBER Number ; ) ; ; VOID ; KeAcquireInStackQueuedSpinLock ( ; IN PKSPIN_LOCK SpinLock, ; IN PKLOCK_QUEUE_HANDLE LockHandle ; ) ; ; VOID ; KeAcquireInStackQueuedSpinLockRaiseToSynch ( ; IN PKSPIN_LOCK SpinLock, ; IN PKLOCK_QUEUE_HANDLE LockHandle ; ) ; ; Routine Description: ; ; This function raises the current IRQL to DISPATCH/SYNCH level ; and acquires the specified queued spinlock. ; ; Arguments: ; ; Number (ecx) - Supplies the queued spinlock number. ; ; Return Value: ; ; The previous IRQL is returned as the function value. ; ; ; Routine Description: ; ; The Kx versions use a LOCK_QUEUE_HANDLE structure rather than ; LOCK_QUEUE structures in the PRCB. Old IRQL is stored in the ; LOCK_QUEUE_HANDLE. ; ; Arguments: ; ; SpinLock (ecx) Address of Actual Lock. ; LockHandle (edx) Address of lock context. ; ; Return Value: ; ; None. Actually returns OldIrql because common code is used ; for all implementations. ; ;-- ; compile time assert sizeof(KSPIN_LOCK_QUEUE) == 8 .errnz (LOCK_QUEUE_HEADER_SIZE - 8) ; VOID ; KeAcquireInStackQueuedSpinLockRaiseToSynch ( ; IN PKSPIN_LOCK SpinLock, ; IN PKLOCK_QUEUE_HANDLE LockHandle ; ) align 16 cPublicFastCall KeAcquireInStackQueuedSpinLockRaiseToSynch,2 cPublicFpo 0,1 push SYNCH_LEVEL ; raise to SYNCH_LEVEL jmp short aqsl5 ; continue in common code fstENDP KeAcquireInStackQueuedSpinLockRaiseToSynch ; VOID ; KeAcquireInStackQueuedSpinLockRaiseToSynch ( ; IN PKSPIN_LOCK SpinLock, ; IN PKLOCK_QUEUE_HANDLE LockHandle ; ) cPublicFastCall KeAcquireInStackQueuedSpinLock,2 cPublicFpo 0,1 ; Get old IRQL and raise to DISPATCH_LEVEL push DISPATCH_LEVEL aqsl5: pop eax push dword ptr PCR[PcIrql] mov dword ptr PCR[PcIrql], eax if DBG cmp [esp], eax ja short aqsl endif ifndef NT_UP ; store OldIrql and address of actual lock in the queued ; spinlock structure in the lock queue handle structure. mov eax, [esp] mov [edx].LqhLock, ecx mov dword ptr [edx].LqhNext, 0 mov [edx].LqhOldIrql, al ; continue in common code. common code expects the ; address of the "lock structure" in edx, this is at ; offset LqhNext in the lock queue handle structure. ; not accidentally this offset is zero. .errnz LqhNext ;; lea edx, [edx].LqhNext jmp short aqsl15 ; continue in common code else pop eax ; get old irql and set mov [edx].LqhOldIrql, al ; in lock queue handle. ifdef IRQL_METRICS inc HalRaiseIrqlCount endif fstRET KeAcquireInStackQueuedSpinLock endif fstENDP KeAcquireInStackQueuedSpinLock ; KIRQL ; KeAcquireQueuedSpinLockRaiseToSynch ( ; IN KSPIN_LOCK_QUEUE_NUMBER Number ; ) cPublicFastCall KeAcquireQueuedSpinLockRaiseToSynch,1 cPublicFpo 0,1 push SYNCH_LEVEL jmp short aqsl10 ; continue in common code fstENDP KeAcquireQueuedSpinLockRaiseToSynch ; KIRQL ; KeAcquireQueuedSpinLock ( ; IN KSPIN_LOCK_QUEUE_NUMBER Number ; ) ; cPublicFastCall KeAcquireQueuedSpinLock,1 cPublicFpo 0,1 ; Get old IRQL and raise to DISPATCH_LEVEL push DISPATCH_LEVEL aqsl10: pop eax push dword ptr PCR[PcIrql] mov dword ptr PCR[PcIrql], eax if DBG cmp [esp], eax ja short aqsl endif ifndef NT_UP ; Get address of Lock Queue entry mov edx, PCR[PcPrcb] ; get address of PRCB lea edx, [edx+ecx*8].PbLockQueue ; get &PRCB->LockQueue[Number] ; Get address of the actual lock. mov ecx, [edx].LqLock aqsl15: mov eax, edx ; save Lock Queue entry address ; Exchange the value of the lock with the address of this ; Lock Queue entry. xchg [ecx], edx cmp edx, 0 ; check if lock is held jnz short @f ; jiff held ; note: the actual lock address will be word aligned, we use ; the bottom two bits as indicators, bit 0 is LOCK_QUEUE_WAIT, ; bit 1 is LOCK_QUEUE_OWNER. or ecx, LOCK_QUEUE_OWNER ; mark self as lock owner mov [eax].LqLock, ecx ; lock has been acquired, return. endif aqsl20: pop eax ; restore return value ifdef IRQL_METRICS inc HalRaiseIrqlCount endif fstRET KeAcquireQueuedSpinLock ifndef NT_UP @@: ; The lock is already held by another processor. Set the wait ; bit in this processor's Lock Queue entry, then set the next ; field in the Lock Queue entry of the last processor to attempt ; to acquire the lock (this is the address returned by the xchg ; above) to point to THIS processor's lock queue entry. or ecx, LOCK_QUEUE_WAIT ; set lock bit mov [eax].LqLock, ecx mov [edx].LqNext, eax ; set previous acquirer's ; next field. ; Wait. @@: YIELD ; fire avoidance. test [eax].LqLock, LOCK_QUEUE_WAIT ; check if still waiting jz short aqsl20 ; jif lock acquired jmp short @b ; else, continue waiting endif if DBG ; Raising to a lower IRQL. BugCheck. ; ; KeBugCheckEx(IRQL_NOT_GREATER_OR_EQUAL, ; current (old) IRQL, ; desired IRQL, ; lock number (only if Ke routine, not Kx), ; 8); aqsl: pop edx stdCall _KeBugCheckEx, ; never returns (but help the debugger back-trace) int 3 endif fstENDP KeAcquireQueuedSpinLock page ,132 subttl "Release Queued SpinLock" ;++ ; ; VOID ; KeReleaseInStackQueuedSpinLock ( ; IN PKLOCK_QUEUE_HANDLE LockHandle ; ) ; ; Routine Description: ; ; This function releases a queued spinlock and lowers the IRQL to ; its previous value. ; ; This differs from KeReleaseQueuedSpinLock in that this version ; uses a caller supplied lock context where that one uses a ; predefined lock context in the processor's PRCB. ; ; This version sets up a compatible register context and uses ; KeReleaseQueuedSpinLock to do the actual work. ; ; Arguments: ; ; LockHandle (ecx) - Address of Lock Queue Handle structure. ; ; Return Value: ; ; None. ; ;-- cPublicFastCall KeReleaseInStackQueuedSpinLock,1 cPublicFpo 0,0 movzx edx, byte ptr [ecx].LqhOldIrql ; get old irql ifndef NT_UP lea eax, [ecx].LqhNext ; get address of lock struct jmp short rqsl10 ; continue in common code else jmp short rqsl30 ; continue in common code endif fstENDP KeReleaseInStackQueuedSpinLock ;++ ; ; VOID ; KeReleaseQueuedSpinLock ( ; IN KSPIN_LOCK_QUEUE_NUMBER Number, ; IN KIRQL OldIrql ; ) ; ; Routine Description: ; ; This function releases a queued spinlock and lowers the IRQL to ; its previous value. ; ; Arguments: ; ; Number (ecx) - Supplies the queued spinlock number. ; OldIrql (dl) - Supplies the IRQL value to lower to. ; ; Return Value: ; ; None. ; ;-- cPublicFastCall KeReleaseQueuedSpinLock,2 cPublicFpo 0,0 ifndef NT_UP ; Get address of Lock Queue entry mov eax, PCR[PcPrcb] ; get address of PRCB lea eax, [eax+ecx*8].PbLockQueue ; get &PRCB->LockQueue[Number] rqsl10: push ebx ; need another register cPublicFpo 0,1 ; Clear the lock field in the Lock Queue entry. mov ebx, [eax].LqNext mov ecx, [eax].LqLock ;; and ecx, NOT (LOCK_QUEUE_OWNER) ; clear lock bit ; Quick check: If Lock Queue entry's Next field is not NULL, ; there is another waiter. Don't bother with ANY atomic ops ; in this case. ; ; Note: test clears CF and sets ZF appropriately, the following ; btr sets CF appropriately for the owner check. test ebx, ebx ; clear the "I am owner" bit in the Lock entry. btr ecx, 1 ; clear owner bit if DBG jnc short rqsl98 ; bugcheck if was not set ; tests CF endif mov [eax].LqLock, ecx ; clear lock bit in queue entry jnz short rqsl40 ; jif another processor waits ; ebx contains zero here which will be used to set the new owner NULL push eax ; save &PRCB->LockQueue[Number] cPublicFpo 0,2 ; Use compare exchange to attempt to clear the actual lock. ; If there are still no processors waiting for the lock when ; the compare exchange happens, the old contents of the lock ; should be the address of this lock entry (eax). lock cmpxchg [ecx], ebx ; store 0 if no waiters pop eax ; restore lock queue address cPublicFpo 0,1 jnz short rqsl60 ; jif store failed ; The lock has been released. Lower IRQL and return to caller. rqsl20: pop ebx ; restore ebx cPublicFpo 0,0 endif rqsl30: pushfd ; disable interrupts cli movzx edx, dl xor ecx, ecx mov dword ptr PCR[PcIrql], edx ; set new (lower) OldIrql mov cl, dl ; ecx = zero extended OldIrql mov edx, PCR[PcIRR] ; Check interrupt requests and edx, FindHigherIrqlMask[ecx*4] ; edx = pending interrupts ; enabled by lower IRQL. jne short rqsl80 ; Dispatch pending interrupts. popfd ; restore interrupt state fstRET KeReleaseQueuedSpinLock ifndef NT_UP ; Another processor is waiting on this lock. Hand the lock ; to that processor by getting the address of its LockQueue ; entry, turning ON its owner bit and OFF its wait bit. rqsl40: xor [ebx].LqLock, (LOCK_QUEUE_OWNER+LOCK_QUEUE_WAIT) ; Done, the other processor now owns the lock, clear the next ; field in my LockQueue entry (to preserve the order for entering ; the queue again) and proceed to lower IRQL and return. mov [eax].LqNext, 0 jmp short rqsl20 ; We get here if another processor is attempting to acquire ; the lock but had not yet updated the next field in this ; processor's Queued Lock Next field. Wait for the next ; field to be updated. rqsl60: mov ebx, [eax].LqNext test ebx, ebx ; check if still 0 jnz short rqsl40 ; jif Next field now set. YIELD ; wait a bit jmp short rqsl60 ; continue waiting endif cPublicFpo 0,1 rqsl80: bsr ecx, edx ; (ecx) = Pending irq level cmp ecx, DISPATCH_LEVEL ; if new int at dispatch level jle short @f ; no need to change 8259 masks mov eax, PCR[PcIDR] ; Clear all the interrupt SET_8259_MASK ; masks @@: mov edx, 1 shl edx, cl xor PCR[PcIRR], edx ; clear bit in IRR call SWInterruptHandlerTable[ecx*4] ; Dispatch the pending int. popfd cPublicFpo 0, 0 fstRet KfReleaseSpinLock ; all done ifndef NT_UP if DBG cPublicFpo 0,1 rqsl98: stdCall _KeBugCheckEx, int 3 ; so stacktrace works endif endif fstENDP KeReleaseQueuedSpinLock page ,132 subttl "Try to Acquire Queued SpinLock" ;++ ; ; LOGICAL ; KeTryToAcquireQueuedSpinLock ( ; IN KSPIN_LOCK_QUEUE_NUMBER Number, ; OUT PKIRQL OldIrql ; ) ; ; LOGICAL ; KeTryToAcquireQueuedSpinLockRaiseToSynch ( ; IN KSPIN_LOCK_QUEUE_NUMBER Number, ; OUT PKIRQL OldIrql ; ) ; ; Routine Description: ; ; This function raises the current IRQL to DISPATCH/SYNCH level ; and attempts to acquire the specified queued spinlock. If the ; spinlock is already owned by another thread, IRQL is restored ; to its previous value and FALSE is returned. ; ; Arguments: ; ; Number (ecx) - Supplies the queued spinlock number. ; OldIrql (edx) - A pointer to the variable to receive the old ; IRQL. ; ; Return Value: ; ; TRUE if the lock was acquired, FALSE otherwise. ; N.B. ZF is set if FALSE returned, clear otherwise. ; ;-- align 16 cPublicFastCall KeTryToAcquireQueuedSpinLockRaiseToSynch,2 cPublicFpo 0,0 mov eax, SYNCH_LEVEL ; raise to SYNCH jmp short taqsl10 ; continue in common code fstENDP KeTryToAcquireQueuedSpinLockRaiseToSynch cPublicFastCall KeTryToAcquireQueuedSpinLock,2 cPublicFpo 0,0 mov eax, DISPATCH_LEVEL ; raise to DPC level ; Attempt to get the lock with interrupts disabled, raising ; the priority in the interrupt controller only if acquisition ; is successful. taqsl10: if DBG cmp al, PCR[PcIrql] jb short taqsl98 endif ifndef NT_UP push edx ; save address of OldIrql pushfd ; save interrupt state cli ; disable interrupts ; Get address of Lock Queue entry mov edx, PCR[PcPrcb] ; get address of PRCB lea edx, [edx+ecx*8].PbLockQueue ; get &PRCB->LockQueue[Number] ; Get address of the actual lock. mov ecx, [edx].LqLock if DBG test ecx, LOCK_QUEUE_OWNER+LOCK_QUEUE_WAIT jnz short taqsl99 ; jiff lock already held (or ; this proc already waiting). endif cmp dword ptr [ecx], 0 ; check if already taken push eax ; save new IRQL jnz taqsl60 ; jif already taken xor eax, eax ; comparison value (not locked) ; Store the Lock Queue entry address in the lock ONLY if the ; current lock value is 0. lock cmpxchg [ecx], edx jnz short taqsl60 ; Lock has been acquired. ; note: the actual lock address will be word aligned, we use ; the bottom two bits as indicators, bit 0 is LOCK_QUEUE_WAIT, ; bit 1 is LOCK_QUEUE_OWNER. or ecx, LOCK_QUEUE_OWNER ; mark self as lock owner mov [edx].LqLock, ecx pop eax endif ; Raise IRQL and return success. mov ecx, PCR[PcIrql] ; ecx = OldIrql mov dword ptr PCR[PcIrql], eax ; set new IRQL ifndef NT_UP popfd ; restore interrupt state pop edx endif mov [edx], cl ; *OldIrql = OldIrql xor eax, eax or eax, 1 ; return TRUE fstRET KeTryToAcquireQueuedSpinLock ifndef NT_UP taqsl60: ; The lock is already held by another processor. Indicate ; failure to the caller. pop eax ; pop new IRQL off stack popfd ; restore interrupt state pop edx ; pop saved OldIrql address xor eax, eax ; return FALSE fstRET KeTryToAcquireQueuedSpinLock endif if DBG taqsl98: stdCall _KeBugCheckEx, taqsl99: stdCall _KeBugCheckEx, ; never returns (help the debugger back-trace) int 3 endif fstENDP KeTryToAcquireQueuedSpinLock _TEXT$01 ends end