windows-server-2003/base/hals/halx86/i386/ixlock.asm

        title  "Irql Processing"
;++
;
; Copyright (c) 1989  Microsoft Corporation
;
; Module Name:
;
;    ixlock.asm
;
; Abstract:
;
;    This module implements various locking functions optimized for this hal.
;
; Author:
;
;    Ken Reneris (kenr) 21-April-1994
;
; Environment:
;
;    Kernel mode only.
;
; Revision History:
;
;--

.486p
        .xlist
include hal386.inc
include callconv.inc                    ; calling convention macros
include i386\ix8259.inc
include i386\kimacro.inc
include mac386.inc
        .list

        EXTRNP _KeBugCheckEx,5,IMPORT
        EXTRNP _KeSetEventBoostPriority, 2, IMPORT
        EXTRNP _KeWaitForSingleObject,5, IMPORT

        extrn  FindHigherIrqlMask:DWORD
        extrn  SWInterruptHandlerTable:DWORD

        EXTRNP _KeRaiseIrql,2
        EXTRNP _KeLowerIrql,1

ifdef NT_UP
    LOCK_ADD  equ   add
    LOCK_DEC  equ   dec
    LOCK_CMPXCHG  equ   cmpxchg
else
    LOCK_ADD  equ   lock add
    LOCK_DEC  equ   lock dec
    LOCK_CMPXCHG  equ   lock cmpxchg
endif


_TEXT$01   SEGMENT PARA PUBLIC 'CODE'
        ASSUME  DS:FLAT, ES:FLAT, SS:FLAT, FS:NOTHING, GS:NOTHING

        PAGE
        subttl  "AcquireSpinLock"

;++
;
;  KIRQL
;  KfAcquireSpinLock (
;     IN PKSPIN_LOCK SpinLock
;     )
;
;  Routine Description:
;
;     This function raises to DISPATCH_LEVEL and then acquires a the
;     kernel spin lock.
;
;     In a UP hal spinlock serialization is accomplished by raising the
;     IRQL to DISPATCH_LEVEL.  The SpinLock is not used; however, for
;     debugging purposes if the UP hal is compiled with the NT_UP flag
;     not set (ie, MP) we take the SpinLock.
;
;  Arguments:
;
;     (ecx) = SpinLock Supplies a pointer to an kernel spin lock.
;
;  Return Value:
;
;     OldIrql
;
;--

cPublicFastCall KfAcquireSpinLock,1
cPublicFpo 0,0

        mov     eax, PCR[PcIrql]         ; (eax) = Old Irql
        mov     dword ptr PCR[PcIrql], DISPATCH_LEVEL    ; set new irql

ifndef NT_UP
asl10:  ACQUIRE_SPINLOCK    ecx,<short asl20>
endif

ifdef IRQL_METRICS
        inc     HalRaiseIrqlCount
endif
if DBG
        cmp     al, DISPATCH_LEVEL      ; old > new?
        ja      short asl99             ; yes, go bugcheck
endif
        fstRET    KfAcquireSpinLock

ifndef NT_UP
asl20:  SPIN_ON_SPINLOCK    ecx,<short asl10>
endif

if DBG
cPublicFpo 2,1
asl99:  movzx   eax, al
        stdCall  _KeBugCheckEx,<IRQL_NOT_GREATER_OR_EQUAL,eax,DISPATCH_LEVEL,0,1>
        ; never returns
endif
        fstRET    KfAcquireSpinLock
fstENDP KfAcquireSpinLock

;++
;
;  KIRQL
;  KeAcquireSpinLockRaiseToSynch (
;     IN PKSPIN_LOCK SpinLock
;     )
;
;  Routine Description:
;
;     This function acquires the SpinLock at SYNCH_LEVEL.  The function
;     is optmized for hoter locks (the lock is tested before acquired.
;     Any spin should occur at OldIrql; however, since this is a UP hal
;     we don't have the code for it)
;
;     In a UP hal spinlock serialization is accomplished by raising the
;     IRQL to SYNCH_LEVEL.  The SpinLock is not used; however, for
;     debugging purposes if the UP hal is compiled with the NT_UP flag
;     not set (ie, MP) we take the SpinLock.
;
;  Arguments:
;
;     (ecx) = SpinLock Supplies a pointer to an kernel spin lock.
;
;  Return Value:
;
;     OldIrql
;
;--

cPublicFastCall KeAcquireSpinLockRaiseToSynch,1
cPublicFpo 0,0

        mov     eax, PCR[PcIrql]         ; (eax) = Old Irql
        mov     dword ptr PCR[PcIrql], SYNCH_LEVEL   ; set new irql

ifndef NT_UP
asls10: ACQUIRE_SPINLOCK    ecx,<short asls20>
endif

ifdef IRQL_METRICS
        inc     HalRaiseIrqlCount
endif
if DBG
        cmp     al, SYNCH_LEVEL         ; old > new?
        ja      short asls99            ; yes, go bugcheck
endif
        fstRET  KeAcquireSpinLockRaiseToSynch

ifndef NT_UP
asls20: SPIN_ON_SPINLOCK    ecx,<short asls10>
endif

if DBG
cPublicFpo 2,1
asls99: movzx   eax, al
        stdCall  _KeBugCheckEx,<IRQL_NOT_GREATER_OR_EQUAL,eax,SYNCH_LEVEL,0,2>
        ; never returns
endif
        fstRET  KeAcquireSpinLockRaiseToSynch
fstENDP KeAcquireSpinLockRaiseToSynch

        PAGE
        SUBTTL "Release Kernel Spin Lock"

;++
;
;  VOID
;  KfReleaseSpinLock (
;     IN PKSPIN_LOCK SpinLock,
;     IN KIRQL       NewIrql
;     )
;
;  Routine Description:
;
;     This function releases a kernel spin lock and lowers to the new irql
;
;     In a UP hal spinlock serialization is accomplished by raising the
;     IRQL to DISPATCH_LEVEL.  The SpinLock is not used; however, for
;     debugging purposes if the UP hal is compiled with the NT_UP flag
;     not set (ie, MP) we use the SpinLock.
;
;  Arguments:
;
;     (ecx) = SpinLock Supplies a pointer to an executive spin lock.
;     (dl)  = NewIrql  New irql value to set
;
;  Return Value:
;
;     None.
;
;--

align 16
cPublicFastCall KfReleaseSpinLock  ,2
cPublicFpo 0,0
ifndef NT_UP
        RELEASE_SPINLOCK    ecx         ; release it
endif
        xor     ecx, ecx
if DBG
        cmp     dl, PCR[PcIrql]
        ja      short rsl99
endif
        pushfd
        cli

        movzx   edx, dl
        mov     PCR[PcIrql], edx        ; store old irql

        mov     cl, dl                  ; (ecx) = 32bit extended OldIrql
        mov     edx, PCR[PcIRR]
        and     edx, FindHigherIrqlMask[ecx*4]  ; (edx) is the bitmask of
                                                ; pending interrupts we need to
        jne     short rsl20                     ; dispatch now.

        popfd
        fstRet  KfReleaseSpinLock               ; all done

if DBG
rsl99:  mov     eax, PCR[PcIrql]
        movzx   edx, dl
        stdCall _KeBugCheckEx,<IRQL_NOT_LESS_OR_EQUAL,eax,edx,0,3>
        ; never returns
endif

cPublicFpo 0,1
rsl20:  bsr     ecx, edx                        ; (ecx) = Pending irq level
        cmp     ecx, DISPATCH_LEVEL
        jle     short rsl40

        mov     eax, PCR[PcIDR]                 ; Clear all the interrupt
        SET_8259_MASK                           ; masks
rsl40:
        mov     edx, 1
        shl     edx, cl
        xor     PCR[PcIRR], edx                 ; clear bit in IRR
        call    SWInterruptHandlerTable[ecx*4]  ; Dispatch the pending int.
        popfd

cPublicFpo 0, 0
        fstRet  KfReleaseSpinLock               ; all done

fstENDP KfReleaseSpinLock

;++
;
;  VOID
;  FASTCALL
;  ExAcquireFastMutex (
;     IN PFAST_MUTEX    FastMutex
;     )
;
;  Routine description:
;
;   This function acquires ownership of the FastMutex
;
;  Arguments:
;
;     (ecx) = FastMutex - Supplies a pointer to the fast mutex
;
;  Return Value:
;
;     None.
;
;--

cPublicFastCall ExAcquireFastMutex,1
cPublicFpo 0,0
        mov     eax, PCR[PcIrql]            ; (eax) = OldIrql
if DBG
        cmp     eax, APC_LEVEL              ; Is OldIrql > NewIrql?
        ja      short afm99                 ; Yes, bugcheck

        pushfd
        cli                                 ; prevent swapcontext while
                                            ; snapping current thread.
        mov     edx, PCR[PcPrcb]
        mov     edx, [edx].PbCurrentThread  ; (edx) = Current Thread
        popfd

        cmp     [ecx].FmOwner, edx          ; Already owned by this thread?
        je      short afm98                 ; Yes, error
endif

        mov     dword ptr PCR[PcIrql], APC_LEVEL     ; Set NewIrql
   LOCK_DEC     dword ptr [ecx].FmCount             ; Get count

        jnz     short afm10                 ; Not the owner, go wait.

	    ;
	    ; Leave a notion of owner behind.
	    ;
	    ; Note: if you change this, change ExAcquireFastMutexUnsafe.
        ;
	
if DBG
        pushfd
        cli                                     ; prevent swapcontext while
                                                ; snapping current thread.

        mov     edx, PCR[PcPrcb]
        mov     edx, [edx].PbCurrentThread  ; (edx) = Current Thread
        popfd

        mov     [ecx].FmOwner, edx          ; Save in Fast Mutex
else
        ;
        ; Use esp to track the owning thread for debugging purposes.
        ; !thread from kd will find the owning thread.  Note that the
        ; owner isn't cleared on release, check if the mutex is owned
        ; first.
        ;

        mov     [ecx].FmOwner, esp
endif
        mov     dword ptr [ecx].FmOldIrql, eax
        fstRet  ExAcquireFastMutex

afm10:
        inc     dword ptr [ecx].FmContention

cPublicFpo 0,2
        push    eax                         ; save OldIrql
        push    ecx                         ; Save FAST_MUTEX
        add     ecx, FmEvent                ; Wait on Event

        stdCall _KeWaitForSingleObject,<ecx,WrExecutive,0,0,0>

        pop     ecx                         ; (ecx) = FAST_MUTEX
        pop     eax                         ; (al) = OldIrql

cPublicFpo 1,0
afm_ret:

	    ;
	    ; Leave a notion of owner behind.
	    ;
	    ; Note: if you change this, change ExAcquireFastMutexUnsafe.
        ;
	
if DBG
        pushfd
        cli                                 ; prevent swapcontext while
                                            ; snapping current thread.

        mov     edx, PCR[PcPrcb]
        mov     edx, [edx].PbCurrentThread  ; (edx) = Current Thread
        popfd

        mov     [ecx].FmOwner, edx          ; Save in Fast Mutex
else
        ;
        ; Use esp to track the owning thread for debugging purposes.
        ; !thread from kd will find the owning thread.  Note that the
        ; owner isn't cleared on release, check if the mutex is owned
        ; first.
        ;

        mov     [ecx].FmOwner, esp
endif
        mov     byte ptr [ecx].FmOldIrql, al
        fstRet  ExAcquireFastMutex

if DBG

        ; KeBugCheckEx(MUTEX_ALREADY_OWNED, FastMutex, CurrentThread, 0, 4)
        ; (never returns)

afm98:  stdCall _KeBugCheckEx,<MUTEX_ALREADY_OWNED,ecx,edx,0,4>

        ; KeBugCheckEx(IRQL_NOT_LESS_OR_EQUAL, CurrentIrql, APC_LEVEL, 0, 5)
        ; (never returns)

afm99:  movzx   eax, al
        stdCall _KeBugCheckEx,<IRQL_NOT_LESS_OR_EQUAL,eax,APC_LEVEL,0,5>

        fstRet  ExAcquireFastMutex
endif

fstENDP ExAcquireFastMutex


;++
;
;  VOID
;  FASTCALL
;  ExReleaseFastMutex (
;     IN PFAST_MUTEX    FastMutex
;     )
;
;  Routine description:
;
;   This function releases ownership of the FastMutex
;
;  Arguments:
;
;     (ecx) = FastMutex - Supplies a pointer to the fast mutex
;
;  Return Value:
;
;     None.
;
;--

cPublicFastCall ExReleaseFastMutex,1
cPublicFpo 0,0
        xor     eax, eax
if DBG
        pushfd
        cli                                     ; prevent swapcontext while
                                                ; snapping current thread.
        mov     edx, PCR[PcPrcb]
        mov     edx, [edx].PbCurrentThread      ; (edx) = CurrentThread
        popfd
        cmp     [ecx].FmOwner, edx              ; Owner == CurrentThread?
        jne     short rfm_threaderror           ; No, bugcheck

        or      byte ptr [ecx].FmOwner, 1       ; not the owner anymore
endif

        mov     al, byte ptr [ecx].FmOldIrql    ; (eax) = OldIrql
   LOCK_ADD     dword ptr [ecx].FmCount, 1      ; Remove our count
        js      short rfm05                     ; if < 0, set event
        jnz     short rfm10                     ; if != 0, don't set event

rfm05:
cPublicFpo 0,2
        push    eax                             ; Save OldIrql
        add     ecx, FmEvent
        stdCall _KeSetEventBoostPriority, <ecx, 0>
        pop     eax

cPublicFpo 0,0
rfm10:
        cli
        mov     PCR[PcIrql], eax
        mov     edx, PCR[PcIRR]
        and     edx, FindHigherIrqlMask[eax*4]  ; (edx) is the bitmask of
                                                ; pending interrupts we need to
        jne     short rfm20                     ; dispatch now.

        sti
        fstRet  ExReleaseFastMutex              ; all done
if DBG

        ; KeBugCheck(THREAD_NOT_MUTEX_OWNER, FastMutex, Thread, Owner, 6)
        ; (never returns)

rfm_threaderror:
        stdCall _KeBugCheckEx,<THREAD_NOT_MUTEX_OWNER,ecx,edx,[ecx].FmOwner,6>

endif

rfm20:  bsr     ecx, edx                        ; (ecx) = Pending irq level
        cmp     ecx, DISPATCH_LEVEL
        jle     short rfm40

        mov     eax, PCR[PcIDR]                 ; Clear all the interrupt
        SET_8259_MASK                           ; masks
rfm40:
        mov     edx, 1
        shl     edx, cl
        xor     PCR[PcIRR], edx                 ; clear bit in IRR
        call    SWInterruptHandlerTable[ecx*4]  ; Dispatch the pending int.
        sti
        fstRet  ExReleaseFastMutex              ; all done
fstENDP ExReleaseFastMutex

;++
;
;  BOOLEAN
;  FASTCALL
;  ExTryToAcquireFastMutex (
;     IN PFAST_MUTEX    FastMutex
;     )
;
;  Routine description:
;
;   This function acquires ownership of the FastMutex
;
;  Arguments:
;
;     (ecx) = FastMutex - Supplies a pointer to the fast mutex
;
;  Return Value:
;
;     Returns TRUE if the FAST_MUTEX was acquired; otherwise false
;
;--

cPublicFastCall ExTryToAcquireFastMutex,1
cPublicFpo 0,0

if DBG
        mov     eax, PCR[PcIrql]                ; (al) = OldIrql
        cmp     eax, APC_LEVEL                  ; Is OldIrql > NewIrql?
        ja      short tam99                     ; Yes, bugcheck
        push    eax                             ; Save OldIrql
else
        push    dword ptr PCR[PcIrql]           ; Save OldIrql
endif

        ;
        ; Try to acquire.
        ;

        mov     edx, 0                          ; Value to set
        mov     eax, 1                          ; Value to compare against

        mov     dword ptr PCR[PcIrql], APC_LEVEL

   LOCK_CMPXCHG dword ptr [ecx].FmCount, edx    ; Attempt to acquire
        jnz     short tam20                     ; got it?

        pop     dword ptr [ecx].FmOldIrql       ; Store OldIrql

if DBG
        pushfd
        cli                                     ; prevent swapcontext while
                                                ; snapping current thread.
        mov     edx, PCR[PcPrcb]
        mov     edx, [edx].PbCurrentThread      ; (edx) = Current Thread
        popfd
        mov     [ecx].FmOwner, edx              ; Save in Fast Mutex
else
        ;
        ; Use esp to track the owning thread for debugging purposes.
        ; !thread from kd will find the owning thread.  Note that the
        ; owner isn't cleared on release, check if the mutex is owned
        ; first.
        ;

        mov	dword ptr [ecx].FmOwner, esp
endif
        mov     eax, 1                          ; return TRUE
        fstRet  ExTryToAcquireFastMutex

tam20:  pop     dword ptr PCR[PcIrql]           ; restore entry Irql
        YIELD
        xor     eax, eax                        ; return FALSE
        fstRet  ExTryToAcquireFastMutex         ; all done

if DBG
        ; KeBugCheckEx(IRQL_NOT_LESS_OR_EQUAL, CurrentIrql, APC_LEVEL, 0, 5)
        ; (never returns)

tam99:  movzx   eax, al
        stdCall _KeBugCheckEx,<IRQL_NOT_LESS_OR_EQUAL,eax,APC_LEVEL,0,7>

        xor     eax, eax                        ; return FALSE
        fstRet  ExTryToAcquireFastMutex
endif

fstENDP ExTryToAcquireFastMutex

        page    ,132
        subttl  "Acquire Queued SpinLock"

;++
;
; KIRQL
; KeAcquireQueuedSpinLock (
;     IN KSPIN_LOCK_QUEUE_NUMBER Number
;     )
;
; KIRQL
; KeAcquireQueuedSpinLockRaiseToSynch (
;     IN KSPIN_LOCK_QUEUE_NUMBER Number
;     )
;
; VOID
; KeAcquireInStackQueuedSpinLock (
;     IN PKSPIN_LOCK SpinLock,
;     IN PKLOCK_QUEUE_HANDLE LockHandle
;     )
;
; VOID
; KeAcquireInStackQueuedSpinLockRaiseToSynch (
;     IN PKSPIN_LOCK SpinLock,
;     IN PKLOCK_QUEUE_HANDLE LockHandle
;     )
;
; Routine Description:
;
;    This function raises the current IRQL to DISPATCH/SYNCH level
;    and acquires the specified queued spinlock.
;
; Arguments:
;
;    Number (ecx) - Supplies the queued spinlock number.
;
; Return Value:
;
;    The previous IRQL is returned as the function value.
;
;
; Routine Description:
;
;    The Kx versions use a LOCK_QUEUE_HANDLE structure rather than
;    LOCK_QUEUE structures in the PRCB.   Old IRQL is stored in the
;    LOCK_QUEUE_HANDLE.
;
; Arguments:
;
;    SpinLock   (ecx) Address of Actual Lock.
;    LockHandle (edx) Address of lock context.
;
; Return Value:
;
;   None.  Actually returns OldIrql because common code is used
;          for all implementations.
;
;--

        ; compile time assert sizeof(KSPIN_LOCK_QUEUE) == 8

        .errnz  (LOCK_QUEUE_HEADER_SIZE - 8)


; VOID
; KeAcquireInStackQueuedSpinLockRaiseToSynch (
;     IN PKSPIN_LOCK SpinLock,
;     IN PKLOCK_QUEUE_HANDLE LockHandle
;     )

align 16
cPublicFastCall KeAcquireInStackQueuedSpinLockRaiseToSynch,2
cPublicFpo 0,1

        push    SYNCH_LEVEL                     ; raise to SYNCH_LEVEL
        jmp     short aqsl5                     ; continue in common code

fstENDP KeAcquireInStackQueuedSpinLockRaiseToSynch


; VOID
; KeAcquireInStackQueuedSpinLockRaiseToSynch (
;     IN PKSPIN_LOCK SpinLock,
;     IN PKLOCK_QUEUE_HANDLE LockHandle
;     )

cPublicFastCall KeAcquireInStackQueuedSpinLock,2
cPublicFpo 0,1

        ; Get old IRQL and raise to DISPATCH_LEVEL

        push    DISPATCH_LEVEL
aqsl5:
        pop     eax
        push    dword ptr PCR[PcIrql]
        mov     dword ptr PCR[PcIrql], eax

if DBG
        cmp     [esp], eax
        ja      short aqsl
endif

ifndef NT_UP

        ; store OldIrql and address of actual lock in the queued
        ; spinlock structure in the lock queue handle structure.

        mov     eax, [esp]
        mov     [edx].LqhLock, ecx
        mov     dword ptr [edx].LqhNext, 0
        mov     [edx].LqhOldIrql, al

        ; continue in common code.  common code expects the
        ; address of the "lock structure" in edx, this is at
        ; offset LqhNext in the lock queue handle structure.
        ; not accidentally this offset is zero.

        .errnz  LqhNext
;;      lea     edx, [edx].LqhNext
        jmp     short aqsl15                    ; continue in common code

else

        pop     eax                             ; get old irql and set
        mov     [edx].LqhOldIrql, al            ; in lock queue handle.

ifdef IRQL_METRICS
        inc     HalRaiseIrqlCount
endif

        fstRET  KeAcquireInStackQueuedSpinLock

endif

fstENDP KeAcquireInStackQueuedSpinLock


; KIRQL
; KeAcquireQueuedSpinLockRaiseToSynch (
;     IN KSPIN_LOCK_QUEUE_NUMBER Number
;     )

cPublicFastCall KeAcquireQueuedSpinLockRaiseToSynch,1
cPublicFpo 0,1

        push    SYNCH_LEVEL
        jmp     short aqsl10                    ; continue in common code

fstENDP KeAcquireQueuedSpinLockRaiseToSynch


; KIRQL
; KeAcquireQueuedSpinLock (
;     IN KSPIN_LOCK_QUEUE_NUMBER Number
;     )
;

cPublicFastCall KeAcquireQueuedSpinLock,1
cPublicFpo 0,1

        ; Get old IRQL and raise to DISPATCH_LEVEL

        push    DISPATCH_LEVEL
aqsl10:
        pop     eax
        push    dword ptr PCR[PcIrql]
        mov     dword ptr PCR[PcIrql], eax

if DBG
        cmp     [esp], eax
        ja      short aqsl
endif

ifndef NT_UP

        ; Get address of Lock Queue entry

        mov     edx, PCR[PcPrcb]                ; get address of PRCB
        lea     edx, [edx+ecx*8].PbLockQueue    ; get &PRCB->LockQueue[Number]

        ; Get address of the actual lock.

        mov     ecx, [edx].LqLock
aqsl15:
        mov     eax, edx                        ; save Lock Queue entry address

        ; Exchange the value of the lock with the address of this
        ; Lock Queue entry.

        xchg    [ecx], edx

        cmp     edx, 0                          ; check if lock is held
        jnz     short @f                        ; jiff held

        ; note: the actual lock address will be word aligned, we use
        ; the bottom two bits as indicators, bit 0 is LOCK_QUEUE_WAIT,
        ; bit 1 is LOCK_QUEUE_OWNER.

        or      ecx, LOCK_QUEUE_OWNER           ; mark self as lock owner
        mov     [eax].LqLock, ecx

        ; lock has been acquired, return.

endif

aqsl20: pop     eax                             ; restore return value

ifdef IRQL_METRICS
        inc     HalRaiseIrqlCount
endif

        fstRET  KeAcquireQueuedSpinLock

ifndef NT_UP

@@:
        ; The lock is already held by another processor.  Set the wait
        ; bit in this processor's Lock Queue entry, then set the next
        ; field in the Lock Queue entry of the last processor to attempt
        ; to acquire the lock (this is the address returned by the xchg
        ; above) to point to THIS processor's lock queue entry.

        or      ecx, LOCK_QUEUE_WAIT            ; set lock bit
        mov     [eax].LqLock, ecx

        mov     [edx].LqNext, eax               ; set previous acquirer's
                                                ; next field.

        ; Wait.
@@:
        YIELD                                   ; fire avoidance.
        test    [eax].LqLock, LOCK_QUEUE_WAIT   ; check if still waiting
        jz      short aqsl20                    ; jif lock acquired
        jmp     short @b                        ; else, continue waiting

endif

if DBG

        ; Raising to a lower IRQL. BugCheck.
        ;
        ; KeBugCheckEx(IRQL_NOT_GREATER_OR_EQUAL,
        ;              current (old) IRQL,
        ;              desired IRQL,
        ;              lock number (only if Ke routine, not Kx),
        ;              8);

aqsl:   pop     edx
        stdCall _KeBugCheckEx,<IRQL_NOT_GREATER_OR_EQUAL,edx,eax,ecx,8>
        ; never returns (but help the debugger back-trace)
        int     3

endif

fstENDP KeAcquireQueuedSpinLock


        page    ,132
        subttl  "Release Queued SpinLock"

;++
;
; VOID
; KeReleaseInStackQueuedSpinLock (
;     IN PKLOCK_QUEUE_HANDLE LockHandle
;     )
;
; Routine Description:
;
;    This function releases a queued spinlock and lowers the IRQL to
;    its previous value.
;
;    This differs from KeReleaseQueuedSpinLock in that this version
;    uses a caller supplied lock context where that one uses a
;    predefined lock context in the processor's PRCB.
;
;    This version sets up a compatible register context and uses
;    KeReleaseQueuedSpinLock to do the actual work.
;
; Arguments:
;
;    LockHandle (ecx) - Address of Lock Queue Handle structure.
;
; Return Value:
;
;    None.
;
;--

cPublicFastCall KeReleaseInStackQueuedSpinLock,1
cPublicFpo 0,0

        movzx   edx, byte ptr [ecx].LqhOldIrql  ; get old irql

ifndef NT_UP

        lea     eax, [ecx].LqhNext              ; get address of lock struct
        jmp     short rqsl10                    ; continue in common code

else

        jmp     short rqsl30                    ; continue in common code

endif


fstENDP KeReleaseInStackQueuedSpinLock


;++
;
; VOID
; KeReleaseQueuedSpinLock (
;     IN KSPIN_LOCK_QUEUE_NUMBER Number,
;     IN KIRQL                   OldIrql
;     )
;
; Routine Description:
;
;    This function releases a queued spinlock and lowers the IRQL to
;    its previous value.
;
; Arguments:
;
;    Number  (ecx) - Supplies the queued spinlock number.
;    OldIrql (dl)  - Supplies the IRQL value to lower to.
;
; Return Value:
;
;    None.
;
;--

cPublicFastCall KeReleaseQueuedSpinLock,2
cPublicFpo 0,0

ifndef NT_UP

        ; Get address of Lock Queue entry

        mov     eax, PCR[PcPrcb]                ; get address of PRCB
        lea     eax, [eax+ecx*8].PbLockQueue    ; get &PRCB->LockQueue[Number]
rqsl10:
        push    ebx                             ; need another register
cPublicFpo 0,1

        ; Clear the lock field in the Lock Queue entry.
        mov     ebx, [eax].LqNext
        mov     ecx, [eax].LqLock
;;        and     ecx, NOT (LOCK_QUEUE_OWNER)     ; clear lock bit

        ; Quick check: If Lock Queue entry's Next field is not NULL,
        ; there is another waiter.  Don't bother with ANY atomic ops
        ; in this case.
        ;
        ; Note: test clears CF and sets ZF appropriately, the following
        ; btr sets CF appropriately for the owner check.

        test    ebx, ebx

        ; clear the "I am owner" bit in the Lock entry.

        btr     ecx, 1                          ; clear owner bit

if DBG

        jnc     short rqsl98                    ; bugcheck if was not set
                                                ; tests CF
endif

        mov     [eax].LqLock, ecx               ; clear lock bit in queue entry
        jnz     short rqsl40                    ; jif another processor waits

        ; ebx contains zero here which will be used to set the new owner NULL

        push    eax                             ; save &PRCB->LockQueue[Number]
cPublicFpo 0,2

        ; Use compare exchange to attempt to clear the actual lock.
        ; If there are still no processors waiting for the lock when
        ; the compare exchange happens, the old contents of the lock
        ; should be the address of this lock entry (eax).

        lock cmpxchg [ecx], ebx                 ; store 0 if no waiters
        pop     eax                             ; restore lock queue address
cPublicFpo 0,1
        jnz     short rqsl60                    ; jif store failed

        ; The lock has been released.  Lower IRQL and return to caller.

rqsl20:
        pop     ebx                             ; restore ebx
cPublicFpo 0,0

endif

rqsl30:
        pushfd                                  ; disable interrupts
        cli

        movzx   edx, dl
        xor     ecx, ecx
        mov     dword ptr PCR[PcIrql], edx      ; set new (lower) OldIrql
        mov     cl, dl                          ; ecx = zero extended OldIrql

        mov     edx, PCR[PcIRR]                 ; Check interrupt requests
        and     edx, FindHigherIrqlMask[ecx*4]  ; edx = pending interrupts
                                                ; enabled by lower IRQL.
        jne     short rqsl80                    ; Dispatch pending interrupts.

        popfd                                   ; restore interrupt state

        fstRET  KeReleaseQueuedSpinLock

ifndef NT_UP

        ; Another processor is waiting on this lock.   Hand the lock
        ; to that processor by getting the address of its LockQueue
        ; entry, turning ON its owner bit and OFF its wait bit.

rqsl40: xor     [ebx].LqLock, (LOCK_QUEUE_OWNER+LOCK_QUEUE_WAIT)

        ; Done, the other processor now owns the lock, clear the next
        ; field in my LockQueue entry (to preserve the order for entering
        ; the queue again) and proceed to lower IRQL and return.

        mov     [eax].LqNext, 0
        jmp     short rqsl20


        ; We get here if another processor is attempting to acquire
        ; the lock but had not yet updated the next field in this
        ; processor's Queued Lock Next field.   Wait for the next
        ; field to be updated.

rqsl60: mov     ebx, [eax].LqNext
        test    ebx, ebx                        ; check if still 0
        jnz     short rqsl40                    ; jif Next field now set.
        YIELD                                   ; wait a bit
        jmp     short rqsl60                    ; continue waiting

endif

cPublicFpo 0,1
rqsl80: bsr     ecx, edx                        ; (ecx) = Pending irq level
        cmp     ecx, DISPATCH_LEVEL             ; if new int at dispatch level
        jle     short @f                        ; no need to change 8259 masks

        mov     eax, PCR[PcIDR]                 ; Clear all the interrupt
        SET_8259_MASK                           ; masks
@@:
        mov     edx, 1
        shl     edx, cl
        xor     PCR[PcIRR], edx                 ; clear bit in IRR
        call    SWInterruptHandlerTable[ecx*4]  ; Dispatch the pending int.
        popfd

cPublicFpo 0, 0
        fstRet  KfReleaseSpinLock               ; all done

ifndef NT_UP

if DBG

cPublicFpo 0,1

rqsl98: stdCall _KeBugCheckEx,<SPIN_LOCK_NOT_OWNED,ecx,eax,0,1>
        int     3                               ; so stacktrace works

endif

endif

fstENDP KeReleaseQueuedSpinLock

        page    ,132
        subttl  "Try to Acquire Queued SpinLock"

;++
;
; LOGICAL
; KeTryToAcquireQueuedSpinLock (
;     IN  KSPIN_LOCK_QUEUE_NUMBER Number,
;     OUT PKIRQL OldIrql
;     )
;
; LOGICAL
; KeTryToAcquireQueuedSpinLockRaiseToSynch (
;     IN  KSPIN_LOCK_QUEUE_NUMBER Number,
;     OUT PKIRQL OldIrql
;     )
;
; Routine Description:
;
;    This function raises the current IRQL to DISPATCH/SYNCH level
;    and attempts to acquire the specified queued spinlock.  If the
;    spinlock is already owned by another thread, IRQL is restored
;    to its previous value and FALSE is returned.
;
; Arguments:
;
;    Number  (ecx) - Supplies the queued spinlock number.
;    OldIrql (edx) - A pointer to the variable to receive the old
;                    IRQL.
;
; Return Value:
;
;    TRUE if the lock was acquired, FALSE otherwise.
;    N.B. ZF is set if FALSE returned, clear otherwise.
;
;--


align 16
cPublicFastCall KeTryToAcquireQueuedSpinLockRaiseToSynch,2
cPublicFpo 0,0

        mov     eax, SYNCH_LEVEL                ; raise to SYNCH
        jmp     short taqsl10                   ; continue in common code

fstENDP KeTryToAcquireQueuedSpinLockRaiseToSynch


cPublicFastCall KeTryToAcquireQueuedSpinLock,2
cPublicFpo 0,0

        mov     eax, DISPATCH_LEVEL             ; raise to DPC level

        ; Attempt to get the lock with interrupts disabled, raising
        ; the priority in the interrupt controller only if acquisition
        ; is successful.
taqsl10:

if DBG
        cmp     al, PCR[PcIrql]
        jb      short taqsl98
endif

ifndef NT_UP

        push    edx                             ; save address of OldIrql
        pushfd                                  ; save interrupt state
        cli                                     ; disable interrupts

        ; Get address of Lock Queue entry

        mov     edx, PCR[PcPrcb]                ; get address of PRCB
        lea     edx, [edx+ecx*8].PbLockQueue    ; get &PRCB->LockQueue[Number]

        ; Get address of the actual lock.

        mov     ecx, [edx].LqLock

if DBG

        test    ecx, LOCK_QUEUE_OWNER+LOCK_QUEUE_WAIT
        jnz     short taqsl99                   ; jiff lock already held (or
                                                ; this proc already waiting).
endif

        cmp     dword ptr [ecx], 0              ; check if already taken
        push    eax                             ; save new IRQL
        jnz     taqsl60                         ; jif already taken
        xor     eax, eax                        ; comparison value (not locked)


        ; Store the Lock Queue entry address in the lock ONLY if the
        ; current lock value is 0.

        lock cmpxchg [ecx], edx
        jnz     short taqsl60

        ; Lock has been acquired.

        ; note: the actual lock address will be word aligned, we use
        ; the bottom two bits as indicators, bit 0 is LOCK_QUEUE_WAIT,
        ; bit 1 is LOCK_QUEUE_OWNER.

        or      ecx, LOCK_QUEUE_OWNER           ; mark self as lock owner
        mov     [edx].LqLock, ecx

        pop     eax

endif

        ; Raise IRQL and return success.

        mov     ecx, PCR[PcIrql]                ; ecx = OldIrql
        mov     dword ptr PCR[PcIrql], eax      ; set new IRQL

ifndef NT_UP

        popfd                                   ; restore interrupt state
        pop     edx

endif

        mov     [edx], cl                       ; *OldIrql = OldIrql
        xor     eax, eax
        or      eax, 1                          ; return TRUE

        fstRET  KeTryToAcquireQueuedSpinLock

ifndef NT_UP

taqsl60:
        ; The lock is already held by another processor.  Indicate
        ; failure to the caller.

        pop     eax                             ; pop new IRQL off stack
        popfd                                   ; restore interrupt state
        pop     edx                             ; pop saved OldIrql address
        xor     eax, eax                        ; return FALSE
        fstRET  KeTryToAcquireQueuedSpinLock

endif

if DBG

taqsl98: stdCall _KeBugCheckEx,<IRQL_NOT_GREATER_OR_EQUAL,PCR[PcIrql],eax,ecx,9>
taqsl99: stdCall _KeBugCheckEx,<SPIN_LOCK_ALREADY_OWNED,ecx,edx,0,0>
        ; never returns (help the debugger back-trace)
        int     3

endif

fstENDP KeTryToAcquireQueuedSpinLock
_TEXT$01   ends

        end