//++ // // Copyright (c) 1989 Microsoft Corporation // // Module Name: // // critsect.s // // Abstract: // // This module implements functions to support user mode critical sections. // // Author: // // William K. Cheung (wcheung) 18-Sep-95 // // Revision History: // // 07-Jul-97 bl Updated to EAS2.3 // // 08-Feb-96 Updated to EAS2.1 // //-- #include "ksia64.h" .file "critsect.s" // // intra-module functions to be called. // PublicFunction(RtlpWaitForCriticalSection) PublicFunction(RtlpUnWaitCriticalSection) //++ // // NTSTATUS // RtlEnterCriticalSection( // IN PRTL_CRITICAL_SECTION CriticalSection // ) // // Routine Description: // // This function enters a critical section. // // Arguments: // // CriticalSection (a0) - supplies a pointer to a critical section. // // Return Value: // // STATUS_SUCCESS or raises an exception if an error occured. // // Algorithm in C: // // NTSTATUS // RtlEnterCriticalSection( // IN PRTL_CRITICAL_SECTION CriticalSection // ) // { // PTEB Teb; // LONG OriginalValue; // HANDLE CurrentThreadId; // DWORD SpinCount; // // Teb = NtCurrentTeb(); // CurrentThreadId = Teb->ClientId.UniqueThread; // SpinCount = CriticalSection->SpinCount; // // if (SpinCount == 0) { // nospin: // OriginalValue = AtomicIncrement(CriticalSection->LockCount); // // if (OriginalValue != -1) { // if (CriticalSection->OwningThread != CurrentThreadId) { // RtlpWaitForCriticalSection(CriticalSection); // CriticalSection->OwningThread = CurrentThreadId; // CriticalSection->RecursionCount = 0; // } else { // CriticalSection->RecursionCount++; // } // } else { // CriticalSection->OwningThread = CurrentThreadId; // CriticalSection->RecursionCount = 0; // } // // return STATUS_SUCCESS; // // } else { /* spin count not 0 */ // // if (CriticalSection->OwningThread == CurrentThread) { // AtomicIncrement(CriticalSection->LockCount); // CriticalSection->RecursionCount++; // return STATUS_SUCCESS; // } else { // do { // if (!CmpXchg(CriticalSection->LockCount,0,-1)) { // Lock acquired // CriticalSection->OwningThread = CurrentThreadId; // CriticalSection->RecursionCount = 0; // return STATUS_SUCCESS; // } // if (CriticalSection->LockCount >= 1) goto nospin; // // while (CriticalSection->LockCount == 0) { // if (!--SpinCount) goto nospin; // } // } while (1) // CriticalSection->LockCount == -1, not owned // } // } // } // //-- NESTED_ENTRY(RtlEnterCriticalSection) // // register aliases // rpThreadId=t3 rOldLockCount=t4 rpLockCount=t5 rRecursionCount=t6 rOwnerId=t7 rpSpinCount=t9 rSpinCount=t10 rT1=t11 rT2=t12 pSpin=pt0 pNoSpin=pt1 pNotOwn=pt2 pNotAcq=pt3 pFree=pt5 pHeld=pt6 pGo=pt7 pWait=pt8 // // alloc regs and save brp & pfs // NESTED_SETUP(1,5,1,0) add rpLockCount = CsLockCount, a0 ;; // // more register aliases // rThreadId = loc2 rpOwner = loc3 rpCsRecursion = loc4 PROLOGUE_END // // Swizzle pointers to address the RecursionCount and // LockCount fields in the critical section structure. // lfetch.nt1 [rpLockCount] nop.f 0 add rpSpinCount = CsSpinCount, a0 add rpCsRecursion = CsRecursionCount, a0 nop.f 0 add rpThreadId = TeClientId+CidUniqueThread, teb ;; // // Load the id of the currently running thread, anticipating // that it may be needed. // ld8 rThreadId = [rpThreadId] ld4 rSpinCount = [rpSpinCount] add rpOwner = CsOwningThread, a0 ;; // // Branch out if spin count is non-zero // cmp4.ne pSpin, pNoSpin = rSpinCount, zero mov v0 = STATUS_SUCCESS (pSpin) br.spnt RecsSpin // // Atomically increment the lock count at location (rpLockCount) // and put the old value in register "rOldLockCount". // Swizzle a pointer to address the thread id field in teb structure. // RecsNoSpin: fetchadd4.acq rOldLockCount = [rpLockCount], 1 ld4.nt1 rRecursionCount = [rpCsRecursion] ;; // // Check the original value of lock count to determine if the // lock is free. If the value is -1, it is free. // cmp4.eq pFree, pHeld = -1, rOldLockCount ;; // // if lock is not free, get the thread id of its current owner // otherwise, save the currently running thread's id. // (pHeld) ld8 rOwnerId = [rpOwner] (pFree) st8 [rpOwner] = rThreadId (pFree) st4 [rpCsRecursion] = zero (pFree) br.ret.sptk.clr brp // return ;; // // if lock is not free, compare the owner id of the critical section against // that of the thread to determine if this thread is the owner. // otherwise, return to caller. // cmp.eq pGo, pWait = rThreadId, rOwnerId mov out0 = a0 add rRecursionCount = 1, rRecursionCount ;; // // if the thread has already owned the lock, save the updated // recursion count and return. // otherwise, wait for the critical section to be released. // (pGo) st4 [rpCsRecursion] = rRecursionCount (pGo) br.ret.sptk brp (pWait) br.call.spnt.many brp = RtlpWaitForCriticalSection ;; st8.rel [rpOwner] = rThreadId st4 [rpCsRecursion] = zero mov v0 = STATUS_SUCCESS NESTED_RETURN // A nonzero spin count is specified // RecsSpin: ld8 rOwnerId = [rpOwner] mov rT1 = -1 ;; zxt4 rT1 = rT1 // zero extend for compare with ;; // 4 byte lock value mov ar.ccv = rT1 // compare value cmp.ne pNotOwn = rOwnerId, rThreadId (pNotOwn) br.spnt RecsNotOwn // // The critical section is owned by the current thread. Increment the lock // count and the recursion count. // ld4 rRecursionCount = [rpCsRecursion] fetchadd4.acq rOldLockCount = [rpLockCount], 1 ;; add rRecursionCount = 1, rRecursionCount ;; st4 [rpCsRecursion] = rRecursionCount br.ret.sptk.clr brp // return // // A nonzero spin count is specified and the current thread is not the owner. // RecsNotOwn: cmpxchg4.acq rT1 = [rpLockCount], zero // try to acquire lock ;; cmp4.ne pNotAcq = -1, rT1 (pNotAcq) br.spnt RecsNotAcq // // The critical section has been acquired. Set the owning thread and the initial // recursion count and return success. // st8 [rpOwner] = rThreadId st4 [rpCsRecursion] = zero br.ret.sptk.clr brp // return // // The critical section is currently owned. Spin until it is either unowned // or the spin count has reached zero. // // If LockCount > 0, then there are waiters. Don't spin because // the lock will not free. // RecsNotAcq: YIELD ld4 rOldLockCount = [rpLockCount] ;; cmp4.eq pNotOwn = -1, rOldLockCount cmp4.gt pNoSpin = rOldLockCount, zero (pNoSpin) br.spnt RecsNoSpin (pNotOwn) br.spnt RecsNotOwn add rSpinCount = -1, rSpinCount ;; cmp4.eq pNoSpin, pSpin = zero, rSpinCount (pNoSpin) br.spnt RecsNoSpin (pSpin) br.sptk RecsNotAcq ;; NESTED_EXIT(RtlEnterCriticalSection) //++ // // NTSTATUS // RtlLeaveCriticalSection( // IN PRTL_CRITICAL_SECTION CriticalSection // ) // // Routine Description: // // This function enters a critical section. // // N.B. This function is duplicated in the runtime library. // // Arguments: // // CriticalSection (a0) - Supplies a pointer to a critical section. // // Return Value: // // STATUS_SUCCESS is returned as the function value. // // Algorithm in C: // // NTSTATUS // RtlLeaveCriticalSection( // IN PRTL_CRITICAL_SECTION CriticalSection // ) // { // LONG NewRecursionCount; // LONG OldLockCount; // BOOL ToRelease; // // ASSERT(CriticalSection->RecursionCount >= 0) // // if (CriticalSection->RecursionCount != 0) { // CriticalSection->RecursionCount -= 1; // AtomicDecrement(CriticalSection->LockCount); // return STATUS_SUCCESS; // } // // CriticalSection->OwningThread = 0; // OldLockCount = AtomicDecrement(CriticalSection->LockCount); // // if ( OldLockCount != 0 ) { // RtlpUnWaitCriticalSection(CriticalSection); // } // // return STATUS_SUCCESS; // } // //-- // // register aliases // NESTED_ENTRY(RtlLeaveCriticalSection) rpOwner=t0 rOldLockCount=t1 rRecursionCount=t2 rpLockCount=t5 rpCsRecursion=t9 pHold=pt0 pRel=pt1 pGo=pt7 pWait=pt8 NESTED_SETUP(1,2,1,0) add rpCsRecursion = CsRecursionCount, a0 ;; PROLOGUE_END // // load recursion count // swizzle pointers to address the LockCount and OwningThread // fields in the critical section structure. // ld4 rRecursionCount = [rpCsRecursion] add rpOwner = CsOwningThread, a0 add rpLockCount = CsLockCount, a0 ;; // // check if the original value of the recursion count to determine // if the lock is to be released. // // decrement the register copy of recursion count by 1 and save // the new value in temp register // cmp.ne pHold, pRel = zero, rRecursionCount add rRecursionCount = -1, rRecursionCount add v0 = STATUS_SUCCESS, zero // return STATUS_SUCCESS ;; // // save the updated recursion count into the critical section structure. // // atomically decrement the lock count. // // if lock is still held, return to caller. // // Note: An atomic fetch & add with release form is used here // all previous memory accesses are visible at this point. // // Note: All updates to the Critical Section structure MUST be complete // prior to the lock count being decremented which releases the // lock. // (pHold) st4 [rpCsRecursion] = rRecursionCount (pRel) st8 [rpOwner] = zero // clear the owner field ;; fetchadd4.rel rOldLockCount = [rpLockCount], -1 (pHold) br.ret.sptk.clr brp // return to caller ;; // // The lock is now free, check the original value of the lock count to // determine if any other thread is waiting for this critical section. // If no thread is waiting, return to caller immediately. // cmp4.ge pGo, pWait = zero, rOldLockCount (pGo) br.ret.sptk.clr brp // return to caller mov out0 = a0 (pWait) br.call.spnt.many brp = RtlpUnWaitCriticalSection mov v0 = STATUS_SUCCESS // return STATUS_SUCCESS NESTED_RETURN ;; NESTED_EXIT(RtlLeaveCriticalSection) //++ // // BOOL // RtlTryEnterCriticalSection( // IN PRTL_CRITICAL_SECTION CriticalSection // ) // // Routine Description: // // This function attempts to enter a critical section without blocking. // // Arguments: // // CriticalSection (a0) - Supplies a pointer to a critical section. // // Return Value: // // If the critical section was successfully entered, then a value of TRUE // is returned as the function value. Otherwise, a value of FALSE is returned // //-- LEAF_ENTRY(RtlTryEnterCriticalSection) // // register aliases // rT0=t0 rT1=t1 rpThreadId=t3 rOldLockCount=t4 rpLockCount=t5 rRecursionCount=t6 rOwnerId=t7 rpCsRecursion=t8 rThreadId=t9 rpOwner=t10 pFree=pt5 pHeld=pt6 pOwn=pt7 pFail=pt8 alloc rT0 = ar.pfs, 1, 0, 0, 0 movl rT1 = 0xffffffff ;; mov ar.ccv = rT1 add rpOwner = CsOwningThread, a0 add rpLockCount = CsLockCount, a0 ;; cmpxchg4.acq rOldLockCount = [rpLockCount], r0, ar.ccv add rpCsRecursion = CsRecursionCount, a0 add rpThreadId = TeClientId+CidUniqueThread, teb ;; ld8 rOwnerId = [rpOwner] cmp4.eq pFree, pHeld = rT1, rOldLockCount ;; ld8.nt1 rThreadId = [rpThreadId] (pHeld) ld4.nt1 rRecursionCount = [rpCsRecursion] mov v0 = TRUE ;; (pFree) st4 [rpCsRecursion] = zero (pFree) st8 [rpOwner] = rThreadId (pHeld) cmp.eq pOwn, pFail = rThreadId, rOwnerId (pFree) br.ret.sptk.clr brp ;; (pOwn) fetchadd4.acq rT0 = [rpLockCount], 1 (pOwn) add rRecursionCount = 1, rRecursionCount nop.i 0 ;; (pOwn) st4.rel [rpCsRecursion] = rRecursionCount (pFail) mov v0 = FALSE br.ret.sptk.clr brp LEAF_EXIT(RtlTryEnterCriticalSection)