|
|
// TITLE("Interlocked Support")//++//// Copyright (c) 1998 Intel Corporation// Copyright (c) 1998 Microsoft Corporation//// Module Name://// slist.s//// Abstract://// This module implements functions to support interlocked S_List// operations.//// Author://// William K. Cheung (v-wcheung) 10-Mar-1998//// Environment://// User mode.//// Revision History:////--
#include "ksia64.h"
//// Define how big the depth field. This is the start of the word//#define SLIST_DEPTH_BITS_SIZE 16
//// Define the start of the pointer in an slist//#define SLIST_ADR_BITS_START 25
//// Define the number of alignment bits for an address//
#define SLIST_ADR_ALIGMENT 4
//// Define were the sequence bits start and how many there are//
#define SLIST_SEQ_BITS_START 16
#define SLIST_SEQ_BITS_SIZE 9
//++//// PSINGLE_LIST_ENTRY// RtlpInterlockedFlushSList (// IN PSLIST_HEADER ListHead// )//// Routine Description://// This function flushes the entire list of entries on a sequenced singly// linked list so that access to the list is synchronized in a MP system.// If there are no entries in the list, then a value of NULL is returned.// Otherwise, the address of the 1st entry on the list is returned as the// function value.//// Arguments://// ListHead (a0) - Supplies a pointer to the sequenced listhead from which// an current list is to be removed.//// Return Value://// The address of the 1st entry on the list, or NULL if the list is// empty.////--
LEAF_ENTRY(RtlpInterlockedFlushSList) ALTERNATE_ENTRY(ExpInterlockedFlushSList)
ld8.acq.nt1 t0 = [a0] // load next entry & sequence#if defined (NTOS_KERNEL_RUNTIME)
add t1 = 8, a0 // Get address of region cell#endif
;;
#if defined (NTOS_KERNEL_RUNTIME)
ld8 t1 = [t1] // Load region bits#endif
mov ar.ccv = t0 shr.u v0 = t0, SLIST_ADR_BITS_START ;;
Efls10: cmp.eq pt1 = v0, r0 // if eq, list is empty extr.u t5 = t0, SLIST_SEQ_BITS_START, \ SLIST_SEQ_BITS_SIZE // extract the sequence number (pt1) br.ret.spnt.clr brp // return if the list is null ;;
#if defined (NTOS_KERNEL_RUNTIME)
shladd v0 = v0, SLIST_ADR_ALIGMENT, t1 // merge region & va bits#else
shl v0 = v0, SLIST_ADR_ALIGMENT // shift address into place#endif
shl t5 = t5, SLIST_SEQ_BITS_START // shift sequence number into place
;;
cmpxchg8.rel t3 = [a0], t5, ar.ccv // perform the pop ;;
cmp.eq pt1, pt2 = t3, t0 // if eq, cmpxchg8 succeeded ;;
(pt2) mov ar.ccv = t3 (pt1) br.ret.sptk brp ;;
mov t0 = t3 shr.u v0 = t3, SLIST_ADR_BITS_START br Efls10 // try again
LEAF_EXIT(RtlpInterlockedFlushSList) SBTTL("Interlocked Pop Entry Sequenced List")//++//// PSINGLE_LIST_ENTRY// RtlpInterlockedPopEntrySList (// IN PSLIST_HEADER ListHead// )//// Routine Description://// This function removes an entry from the front of a sequenced singly// linked list so that access to the list is synchronized in a MP system.// If there are no entries in the list, then a value of NULL is returned.// Otherwise, the address of the entry that is removed is returned as the// function value.//// Arguments://// ListHead (a0) - Supplies a pointer to the sequenced listhead from which// an entry is to be removed.//// Return Value://// The address of the entry removed from the list, or NULL if the list is// empty.////--
LEAF_ENTRY(RtlpInterlockedPopEntrySList) ALTERNATE_ENTRY(ExpInterlockedPopEntrySList) ALTERNATE_ENTRY(ExpInterlockedPopEntrySListResume)
ld8.acq.nt1 t0 = [a0] // load next entry & sequence#if defined (NTOS_KERNEL_RUNTIME)
add t1 = 8, a0 // Generate address region bits#endif
;;
Epop10:
#if defined (NTOS_KERNEL_RUNTIME)
ld8 t1 = [t1] // capture region bits#endif
mov ar.ccv = t0 shr.u t2 = t0, SLIST_ADR_BITS_START ;;
#if defined (NTOS_KERNEL_RUNTIME)
shladd v0 = t2, SLIST_ADR_ALIGMENT, t1 // merge region & va bits#else
shl v0 = t2, SLIST_ADR_ALIGMENT // shift va into place#endif
;;
#if defined (NTOS_KERNEL_RUNTIME)
cmp.eq pt2, pt1 = t1, v0 // if eq, list is empty#else
cmp.eq pt2, pt1 = r0, v0 // if eq, list is empty#endif
sub t4 = t0, zero, 1 // adjust depth ;;
//// N.B. It is possible for the following instruction to fault in the rare// case where the first entry in the list is allocated on another// processor and free between the time the free pointer is read above// and the following instruction. When this happens, the access fault// code continues execution by skipping the following instruction slot.// This results in the compare failing and the entire operation is// retried.//
ALTERNATE_ENTRY(ExpInterlockedPopEntrySListFault)
(pt1) ld8.acq t5 = [v0] // get addr of successor entry extr.u t4 = t4, 0, SLIST_DEPTH_BITS_SIZE + \ SLIST_SEQ_BITS_SIZE // extract depth & sequence ;;
(pt1) shl t5 = t5, SLIST_ADR_BITS_START - \ SLIST_ADR_ALIGMENT // shift va into position dropping alignment bits (pt2) add v0 = 0, zero // The list is empty clear retur value ;;
(pt1) ld8.nt1 t3 = [a0] // reload next entry & sequence (pt1) or t5 = t4, t5 // merge va, depth & sequence (pt2) br.ret.spnt.clr brp // return if the list is null ;;
cmp.eq.unc pt4, pt1 = t3, t0 ;;
ALTERNATE_ENTRY(ExpInterlockedPopEntrySListEnd)
(pt4) cmpxchg8.rel t3 = [a0], t5, ar.ccv // perform the pop nop.i 0 ;;
(pt4) cmp.eq.unc pt3 = t3, t0 // if eq, cmpxchg8 succeeded#if defined (NTOS_KERNEL_RUNTIME)
add t1 = 8, a0#endif
mov t0 = t3 (pt3) br.ret.sptk brp
br Epop10 // try again
LEAF_EXIT(RtlpInterlockedPopEntrySList) //++//// PSINGLE_LIST_ENTRY// RtlpInterlockedPushEntrySList (// IN PSLIST_HEADER ListHead,// IN PSINGLE_LIST_ENTRY ListEntry// )//// Routine Description://// This function inserts an entry at the head of a sequenced singly linked// list so that access to the list is synchronized in an MP system.//// Arguments://// ListHead (a0) - Supplies a pointer to the sequenced listhead into which// an entry is to be inserted.//// ListEntry (a1) - Supplies a pointer to the entry to be inserted at the// head of the list.//// Return Value://// Previous contents of ListHead. NULL implies list went from empty// to not empty.////--
LEAF_ENTRY(RtlpInterlockedPushEntrySList) ALTERNATE_ENTRY(ExpInterlockedPushEntrySList)
ld8.acq.nt1 t0 = [a0] // load next entry & sequence mov t6 = (1<<SLIST_SEQ_BITS_START)+1 // Increment sequence and depth ldf.fill ft0 = [a1] // Force alignment check. shl t5 = a1, SLIST_ADR_BITS_START - \ SLIST_ADR_ALIGMENT // shift va into position ;;
#if DBG==1
and t3 = (1<<SLIST_ADR_ALIGMENT)-1, a1 add t2 = 8, a0 dep t1 = 0, a1, 0, 61 // capture region bits ;;
cmp.eq pt2, pt1 = t3, r0#if defined (NTOS_KERNEL_RUNTIME)
ld8 t2 = [t2] ;;
(pt2) cmp.eq pt2, pt1 = t2, t1#endif
;;
(pt1) break.i 0x80016#endif
Epush10: mov ar.ccv = t0 // set the comparand add t4 = t0, t6 shr.u v0 = t0, SLIST_ADR_BITS_START
;;
#if defined (NTOS_KERNEL_RUNTIME)
dep t1 = 0, a1, 0, 61 // capture region bits#endif
cmp.ne pt3 = zero, v0 // if ne, list not empty ;;
#if defined (NTOS_KERNEL_RUNTIME)
(pt3) shladd v0 = v0, SLIST_ADR_ALIGMENT, t1 // merge region & va bits#else
(pt3) shl v0 = v0, SLIST_ADR_ALIGMENT // shift address into place#endif
extr.u t4 = t4, 0, SLIST_DEPTH_BITS_SIZE + \ SLIST_SEQ_BITS_SIZE // extract depth & sequence ;;
st8 [a1] = v0 or t5 = t4, t5 // merge va, depth & sequence ;;
cmpxchg8.rel t3 = [a0], t5, ar.ccv ;;
cmp.eq pt2, pt1 = t0, t3 dep t5 = 0, t5, 0, SLIST_DEPTH_BITS_SIZE + \ SLIST_SEQ_BITS_SIZE // Zero depth a sequence ;;
(pt2) br.ret.sptk brp // if equal, return (pt1) mov t0 = t3 (pt1) br.spnt Epush10 // retry ;;
LEAF_EXIT(RtlpInterlockedPushEntrySList)
//++//// SINGLE_LIST_ENTRY// FASTCALL// InterlockedPushListSList (// IN PSLIST_HEADER ListHead,// IN PSINGLE_LIST_ENTRY List,// IN PSINGLE_LIST_ENTRY ListEnd,// IN ULONG Count// )//// Routine Description://// This function will push multiple entries onto an SList at once//// Arguments://// ListHead - List head to push the list to.//// List - The list to add to the front of the SList// ListEnd - The last element in the chain// Count - The number of items in the chain//// Return Value://// PSINGLE_LIST_ENTRY - The old header pointer is returned////--
LEAF_ENTRY(InterlockedPushListSList)
ld8.acq.nt1 t0 = [a0] // load next entry & sequence mov t6 = 0x10000 // Generate literal for adjusting depth and count ldf.fill ft0 = [a1] // Force alignment check. shl t5 = a1, SLIST_ADR_BITS_START - \ SLIST_ADR_ALIGMENT // shift va into position ;;
#if DBG==1
and t3 = (1<<SLIST_ADR_ALIGMENT)-1, a1 add t2 = 8, a0 dep t1 = 0, a1, 0, 61 // capture region bits ;;
cmp.eq pt2, pt1 = t3, r0#if defined (NTOS_KERNEL_RUNTIME)
ld8 t2 = [t2] ;;
(pt2) cmp.eq pt2, pt1 = t2, t1#endif
;;
(pt1) break.i 0x80016#endif
Epushl10: mov ar.ccv = t0 // set the comparand add t4 = t0, t6 shr.u v0 = t0, SLIST_ADR_BITS_START ;;
#if defined (NTOS_KERNEL_RUNTIME)
dep t1 = 0, a1, 0, 61 // capture region bits#endif
cmp.ne pt3 = zero, v0 // if ne, list not empty add t4 = t4, a3 // Add in count ;;
#if defined (NTOS_KERNEL_RUNTIME)
(pt3) shladd v0 = v0, SLIST_ADR_ALIGMENT, t1 // merge region & va bits#else
(pt3) shl v0 = v0, SLIST_ADR_ALIGMENT // shift address into place#endif
extr.u t4 = t4, 0, SLIST_DEPTH_BITS_SIZE + \ SLIST_SEQ_BITS_SIZE // extract depth & sequence ;;
st8 [a2] = v0 or t5 = t4, t5 // merge va, depth & sequence ;;
cmpxchg8.rel t3 = [a0], t5, ar.ccv ;;
cmp.eq pt2, pt1 = t0, t3 dep t5 = 0, t5, 0, SLIST_DEPTH_BITS_SIZE + \ SLIST_SEQ_BITS_SIZE // zero depth and sequence
mov t0 = t3 (pt2) br.ret.sptk brp // if equal, return (pt1) br.spnt Epushl10 // retry ;;
LEAF_EXIT(InterlockedPushListSList)
//++//// PSINGLE_LIST_ENTRY// FirstEntrySList (// IN PSLIST_HEADER SListHead// )//// Routine Description://// This function returns the address of the fisrt entry in the SLIST or// NULL.//// Arguments://// ListHead - Supplies a pointer to the sequenced listhead from// which the first entry address is to be computed.//// Return Value://// The address of the first entry is the specified, or NULL if the list is// empty.////-- LEAF_ENTRY(FirstEntrySList)
ld8 t0 = [a0] // load next entry & sequence#if defined (NTOS_KERNEL_RUNTIME)
add t1 = 8, a0 // Generate address region bits#endif
;;
#if defined (NTOS_KERNEL_RUNTIME)
ld8 t1 = [t1] // capture region bits#endif
shr.u t2 = t0, SLIST_ADR_BITS_START ;;
#if defined (NTOS_KERNEL_RUNTIME)
cmp.eq pt0 = t2, r0 shladd v0 = t2, SLIST_ADR_ALIGMENT, t1 // merge region & va bits#else
shl v0 = t2, SLIST_ADR_ALIGMENT // shift va into place#endif
;;
#if defined (NTOS_KERNEL_RUNTIME)
(pt0) mov v0 = r0#endif
br.ret.sptk brp // return function value
LEAF_EXIT(FirstEntrySList)
|
