windows-server-2003/base/ntdll/ia64/critsect.s

//++
//
// Copyright (c) 1989  Microsoft Corporation
//
// Module Name:
//
//     critsect.s
//
// Abstract:
//
//    This module implements functions to support user mode critical sections.
//
// Author:
//
//    William K. Cheung (wcheung) 18-Sep-95
//
// Revision History:
//
//    07-Jul-97  bl    Updated to EAS2.3
//
//    08-Feb-96        Updated to EAS2.1
//
//--

#include "ksia64.h"

        .file    "critsect.s"

        //
        // intra-module functions to be called.
        //

        PublicFunction(RtlpWaitForCriticalSection)
        PublicFunction(RtlpUnWaitCriticalSection)


//++
//
// NTSTATUS
// RtlEnterCriticalSection(
//     IN PRTL_CRITICAL_SECTION CriticalSection
//     )
//
// Routine Description:
//
//     This function enters a critical section.
//
// Arguments:
//
//     CriticalSection (a0) - supplies a pointer to a critical section.
//
// Return Value:
//
//     STATUS_SUCCESS or raises an exception if an error occured.
//
// Algorithm in C:
//
// NTSTATUS
// RtlEnterCriticalSection(
//     IN PRTL_CRITICAL_SECTION CriticalSection
//     )
// {
//     PTEB     Teb;
//     LONG     OriginalValue;
//     HANDLE   CurrentThreadId;
//     DWORD    SpinCount;
//
//     Teb = NtCurrentTeb();
//     CurrentThreadId = Teb->ClientId.UniqueThread;
//     SpinCount = CriticalSection->SpinCount;
//
//     if (SpinCount == 0) {
// nospin:
//           OriginalValue = AtomicIncrement(CriticalSection->LockCount);
//      
//           if (OriginalValue != -1) {
//               if (CriticalSection->OwningThread != CurrentThreadId) {
//                   RtlpWaitForCriticalSection(CriticalSection);
//                   CriticalSection->OwningThread = CurrentThreadId;
//                   CriticalSection->RecursionCount = 0;
//               } else {
//                   CriticalSection->RecursionCount++;
//               }
//           } else {
//               CriticalSection->OwningThread = CurrentThreadId;
//               CriticalSection->RecursionCount = 0;
//           }
//       
//           return STATUS_SUCCESS;
//
//     } else { /* spin count not 0 */
//
//           if (CriticalSection->OwningThread == CurrentThread) {
//               AtomicIncrement(CriticalSection->LockCount);
//               CriticalSection->RecursionCount++;
//               return STATUS_SUCCESS;
//           } else {
//             do {
//               if (!CmpXchg(CriticalSection->LockCount,0,-1)) {
//                  Lock acquired
//                  CriticalSection->OwningThread = CurrentThreadId;
//                  CriticalSection->RecursionCount = 0;
//                  return STATUS_SUCCESS;
//               }
//               if (CriticalSection->LockCount >= 1) goto nospin;
//
//               while (CriticalSection->LockCount == 0) {
//                  if (!--SpinCount) goto nospin;
//               }
//             } while (1) // CriticalSection->LockCount == -1,  not owned
//           }
//     } 
// }
//
//--



        NESTED_ENTRY(RtlEnterCriticalSection)

        //
        // register aliases
        //

        rpThreadId=t3
        rOldLockCount=t4
        rpLockCount=t5
        rRecursionCount=t6
        rOwnerId=t7
        rpSpinCount=t9
        rSpinCount=t10
        rT1=t11
        rT2=t12
        
        pSpin=pt0
        pNoSpin=pt1
        pNotOwn=pt2
        pNotAcq=pt3
        pFree=pt5
        pHeld=pt6
        pGo=pt7
        pWait=pt8

//
// alloc regs and save brp & pfs
//

        NESTED_SETUP(1,5,1,0)
        add         rpLockCount = CsLockCount, a0
        ;;

        //
        // more register aliases
        //

        rThreadId   = loc2
        rpOwner     = loc3
        rpCsRecursion = loc4

        PROLOGUE_END

//
// Swizzle pointers to address the RecursionCount and
// LockCount fields in the critical section structure.
//

        lfetch.nt1  [rpLockCount]
        nop.f       0
        add         rpSpinCount = CsSpinCount, a0

        add         rpCsRecursion = CsRecursionCount, a0
        nop.f       0
        add         rpThreadId = TeClientId+CidUniqueThread, teb
        ;;

//
// Load the id of the currently running thread, anticipating
// that it may be needed.
//

        ld8         rThreadId = [rpThreadId]
        ld4         rSpinCount = [rpSpinCount]
        add         rpOwner = CsOwningThread, a0
        ;;
        
//
// Branch out if spin count is non-zero
//      
        cmp4.ne     pSpin, pNoSpin = rSpinCount, zero
        mov         v0 = STATUS_SUCCESS
(pSpin) br.spnt     RecsSpin

//
// Atomically increment the lock count at location (rpLockCount)
// and put the old value in register "rOldLockCount".
// Swizzle a pointer to address the thread id field in teb structure.
//

RecsNoSpin:
        fetchadd4.acq rOldLockCount = [rpLockCount], 1
        ld4.nt1     rRecursionCount = [rpCsRecursion]
        ;;
        
//
// Check the original value of lock count to determine if the
// lock is free.  If the value is -1, it is free.
//

        cmp4.eq     pFree, pHeld = -1, rOldLockCount
        ;;

//
// if lock is not free, get the thread id of its current owner
// otherwise, save the currently running thread's id.
//

(pHeld) ld8         rOwnerId = [rpOwner]
(pFree) st8         [rpOwner] = rThreadId
(pFree) st4         [rpCsRecursion] = zero
(pFree) br.ret.sptk.clr brp                     // return
        ;;

//
// if lock is not free, compare the owner id of the critical section against 
// that of the thread to determine if this thread is the owner.
// otherwise, return to caller.
//

        cmp.eq      pGo, pWait = rThreadId, rOwnerId
        mov         out0 = a0
        add         rRecursionCount = 1, rRecursionCount
        ;;

//
// if the thread has already owned the lock, save the updated 
// recursion count and return.
// otherwise, wait for the critical section to be released.
//

(pGo)   st4         [rpCsRecursion] = rRecursionCount  
(pGo)   br.ret.sptk brp
(pWait) br.call.spnt.many brp = RtlpWaitForCriticalSection
        ;;

        st8.rel     [rpOwner] = rThreadId
        st4         [rpCsRecursion] = zero
        mov         v0 = STATUS_SUCCESS

        NESTED_RETURN

// A nonzero spin count is specified
//

RecsSpin:

        ld8         rOwnerId = [rpOwner]
        mov         rT1 = -1
        ;;
        zxt4        rT1 = rT1                   // zero extend for compare with
        ;;                                      // 4 byte lock value

        mov         ar.ccv = rT1                // compare value
        cmp.ne      pNotOwn = rOwnerId, rThreadId        
(pNotOwn) br.spnt   RecsNotOwn

//
// The critical section is owned by the current thread. Increment the lock
// count and the recursion count.
//
        
        ld4         rRecursionCount = [rpCsRecursion]
        fetchadd4.acq rOldLockCount = [rpLockCount], 1
        ;;
        add         rRecursionCount = 1, rRecursionCount
        ;;

        st4         [rpCsRecursion] = rRecursionCount
        br.ret.sptk.clr brp                     // return

//
// A nonzero spin count is specified and the current thread is not the owner.
//

RecsNotOwn:
        cmpxchg4.acq rT1 = [rpLockCount], zero  // try to acquire lock
        ;;
        
        cmp4.ne     pNotAcq = -1, rT1
(pNotAcq) br.spnt   RecsNotAcq

        
//
// The critical section has been acquired. Set the owning thread and the initial
// recursion count and return success.
//

        st8         [rpOwner] = rThreadId
        st4         [rpCsRecursion] = zero
        br.ret.sptk.clr brp                     // return

//
// The critical section is currently owned. Spin until it is either unowned
// or the spin count has reached zero. 
//
// If LockCount > 0, then there are waiters. Don't spin because
// the lock will not free.
//

RecsNotAcq:
        YIELD        
        ld4         rOldLockCount = [rpLockCount]
        ;;

        cmp4.eq     pNotOwn = -1, rOldLockCount
        cmp4.gt     pNoSpin = rOldLockCount, zero
(pNoSpin) br.spnt   RecsNoSpin
(pNotOwn) br.spnt   RecsNotOwn

        add         rSpinCount = -1, rSpinCount
        ;;

        cmp4.eq     pNoSpin, pSpin = zero, rSpinCount
(pNoSpin) br.spnt   RecsNoSpin
(pSpin) br.sptk     RecsNotAcq
        ;;

        NESTED_EXIT(RtlEnterCriticalSection)

//++
//
// NTSTATUS
// RtlLeaveCriticalSection(
//    IN PRTL_CRITICAL_SECTION CriticalSection
//    )
//
// Routine Description:
//
//    This function enters a critical section.
//
//    N.B. This function is duplicated in the runtime library.
//
// Arguments:
//
//    CriticalSection (a0) - Supplies a pointer to a critical section.
//
// Return Value:
//
//    STATUS_SUCCESS is returned as the function value.
//
// Algorithm in C:
//
// NTSTATUS
// RtlLeaveCriticalSection(
//     IN PRTL_CRITICAL_SECTION CriticalSection
//     )
// {
//     LONG NewRecursionCount;
//     LONG OldLockCount;
//     BOOL ToRelease;
// 
//     ASSERT(CriticalSection->RecursionCount >= 0)
//
//     if (CriticalSection->RecursionCount != 0) {
//         CriticalSection->RecursionCount -= 1;
//         AtomicDecrement(CriticalSection->LockCount);
//         return STATUS_SUCCESS;
//     }
// 
//     CriticalSection->OwningThread = 0; 
//     OldLockCount = AtomicDecrement(CriticalSection->LockCount);
// 
//     if ( OldLockCount != 0 ) {
//         RtlpUnWaitCriticalSection(CriticalSection);
//     }
// 
//     return STATUS_SUCCESS;
// }
// 
//--

//
// register aliases
//


        NESTED_ENTRY(RtlLeaveCriticalSection)

        rpOwner=t0
        rOldLockCount=t1
        rRecursionCount=t2
        rpLockCount=t5
        rpCsRecursion=t9

        pHold=pt0
        pRel=pt1
        pGo=pt7
        pWait=pt8

        NESTED_SETUP(1,2,1,0)
        add         rpCsRecursion = CsRecursionCount, a0
        ;;

        PROLOGUE_END

//
// load recursion count
// swizzle pointers to address the LockCount and OwningThread
// fields in the critical section structure.
//

        ld4         rRecursionCount = [rpCsRecursion]
        add         rpOwner = CsOwningThread, a0
        add         rpLockCount = CsLockCount, a0
        ;;

//
// check if the original value of the recursion count to determine
// if the lock is to be released.
//
// decrement the register copy of recursion count by 1 and save
// the new value in temp register
//

        cmp.ne      pHold, pRel = zero, rRecursionCount
        add         rRecursionCount = -1, rRecursionCount
        add         v0 = STATUS_SUCCESS, zero   // return STATUS_SUCCESS
        ;;

//
// save the updated recursion count into the critical section structure.
//
// atomically decrement the lock count.
//
// if lock is still held, return to caller.
//
// Note: An atomic fetch & add with release form is used here
//       all previous memory accesses are visible at this point.
//
// Note: All updates to the Critical Section structure MUST be complete
//       prior to the lock count being decremented which releases the
//       lock.
//

(pHold) st4         [rpCsRecursion] = rRecursionCount
(pRel)  st8         [rpOwner] = zero            // clear the owner field
        ;;
        fetchadd4.rel rOldLockCount = [rpLockCount], -1
(pHold) br.ret.sptk.clr brp                     // return to caller
        ;;

//
// The lock is now free, check the original value of the lock count to
// determine if any other thread is waiting for this critical section.
// If no thread is waiting, return to caller immediately.
// 

        cmp4.ge     pGo, pWait = zero, rOldLockCount
  (pGo) br.ret.sptk.clr brp                     // return to caller

        mov         out0 = a0
(pWait) br.call.spnt.many brp = RtlpUnWaitCriticalSection
 
        mov         v0 = STATUS_SUCCESS         // return STATUS_SUCCESS

        NESTED_RETURN
        ;;

        NESTED_EXIT(RtlLeaveCriticalSection)


//++
//
// BOOL
// RtlTryEnterCriticalSection(
//    IN PRTL_CRITICAL_SECTION CriticalSection
//    )
//
// Routine Description:
//
//    This function attempts to enter a critical section without blocking.
//
// Arguments:
//
//    CriticalSection (a0) - Supplies a pointer to a critical section.
//
// Return Value:
//
//    If the critical section was successfully entered, then a value of TRUE
//    is returned as the function value. Otherwise, a value of FALSE is returned
//
//--

        LEAF_ENTRY(RtlTryEnterCriticalSection)

        //
        // register aliases
        //

        rT0=t0
        rT1=t1
        rpThreadId=t3
        rOldLockCount=t4
        rpLockCount=t5
        rRecursionCount=t6
        rOwnerId=t7
        rpCsRecursion=t8
        rThreadId=t9
        rpOwner=t10
 
        pFree=pt5
        pHeld=pt6
        pOwn=pt7
        pFail=pt8

        alloc       rT0 = ar.pfs, 1, 0, 0, 0
        movl        rT1 = 0xffffffff
        ;;

        mov         ar.ccv = rT1
        add         rpOwner = CsOwningThread, a0
        add         rpLockCount = CsLockCount, a0
        ;;

        cmpxchg4.acq rOldLockCount = [rpLockCount], r0, ar.ccv
        add         rpCsRecursion = CsRecursionCount, a0
        add         rpThreadId = TeClientId+CidUniqueThread, teb
        ;;
 
        ld8         rOwnerId = [rpOwner]
        cmp4.eq     pFree, pHeld = rT1, rOldLockCount
        ;;


        ld8.nt1     rThreadId = [rpThreadId]
(pHeld) ld4.nt1     rRecursionCount = [rpCsRecursion]
        mov         v0 = TRUE
        ;;

(pFree) st4         [rpCsRecursion] = zero
(pFree) st8         [rpOwner] = rThreadId
(pHeld) cmp.eq      pOwn, pFail = rThreadId, rOwnerId
(pFree) br.ret.sptk.clr brp
        ;;

(pOwn)  fetchadd4.acq rT0 = [rpLockCount], 1
(pOwn)  add         rRecursionCount = 1, rRecursionCount
        nop.i       0
        ;;

(pOwn)  st4.rel     [rpCsRecursion] = rRecursionCount  
(pFail) mov         v0 = FALSE
        br.ret.sptk.clr brp

        LEAF_EXIT(RtlTryEnterCriticalSection)