You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
468 lines
14 KiB
468 lines
14 KiB
// TITLE("Interlocked Support")
|
|
//++
|
|
//
|
|
// Copyright (c) 1998 Intel Corporation
|
|
// Copyright (c) 1998 Microsoft Corporation
|
|
//
|
|
// Module Name:
|
|
//
|
|
// slist.s
|
|
//
|
|
// Abstract:
|
|
//
|
|
// This module implements functions to support interlocked S_List
|
|
// operations.
|
|
//
|
|
// Author:
|
|
//
|
|
// William K. Cheung (v-wcheung) 10-Mar-1998
|
|
//
|
|
// Environment:
|
|
//
|
|
// User mode.
|
|
//
|
|
// Revision History:
|
|
//
|
|
//--
|
|
|
|
#include "ksia64.h"
|
|
|
|
//
|
|
// Define how big the depth field. This is the start of the word
|
|
//
|
|
#define SLIST_DEPTH_BITS_SIZE 16
|
|
|
|
//
|
|
// Define the start of the pointer in an slist
|
|
//
|
|
#define SLIST_ADR_BITS_START 25
|
|
|
|
//
|
|
// Define the number of alignment bits for an address
|
|
//
|
|
|
|
#define SLIST_ADR_ALIGMENT 4
|
|
|
|
//
|
|
// Define were the sequence bits start and how many there are
|
|
//
|
|
|
|
#define SLIST_SEQ_BITS_START 16
|
|
#define SLIST_SEQ_BITS_SIZE 9
|
|
|
|
|
|
//++
|
|
//
|
|
// PSINGLE_LIST_ENTRY
|
|
// RtlpInterlockedFlushSList (
|
|
// IN PSLIST_HEADER ListHead
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// This function flushes the entire list of entries on a sequenced singly
|
|
// linked list so that access to the list is synchronized in a MP system.
|
|
// If there are no entries in the list, then a value of NULL is returned.
|
|
// Otherwise, the address of the 1st entry on the list is returned as the
|
|
// function value.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// ListHead (a0) - Supplies a pointer to the sequenced listhead from which
|
|
// an current list is to be removed.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// The address of the 1st entry on the list, or NULL if the list is
|
|
// empty.
|
|
//
|
|
//--
|
|
|
|
LEAF_ENTRY(RtlpInterlockedFlushSList)
|
|
ALTERNATE_ENTRY(ExpInterlockedFlushSList)
|
|
|
|
ld8.acq.nt1 t0 = [a0] // load next entry & sequence
|
|
#if defined (NTOS_KERNEL_RUNTIME)
|
|
add t1 = 8, a0 // Get address of region cell
|
|
#endif
|
|
;;
|
|
|
|
#if defined (NTOS_KERNEL_RUNTIME)
|
|
ld8 t1 = [t1] // Load region bits
|
|
#endif
|
|
mov ar.ccv = t0
|
|
shr.u v0 = t0, SLIST_ADR_BITS_START
|
|
;;
|
|
Efls10:
|
|
cmp.eq pt1 = v0, r0 // if eq, list is empty
|
|
extr.u t5 = t0, SLIST_SEQ_BITS_START, \
|
|
SLIST_SEQ_BITS_SIZE // extract the sequence number
|
|
(pt1) br.ret.spnt.clr brp // return if the list is null
|
|
;;
|
|
#if defined (NTOS_KERNEL_RUNTIME)
|
|
shladd v0 = v0, SLIST_ADR_ALIGMENT, t1 // merge region & va bits
|
|
#else
|
|
shl v0 = v0, SLIST_ADR_ALIGMENT // shift address into place
|
|
#endif
|
|
shl t5 = t5, SLIST_SEQ_BITS_START // shift sequence number into place
|
|
|
|
;;
|
|
cmpxchg8.rel t3 = [a0], t5, ar.ccv // perform the pop
|
|
;;
|
|
|
|
cmp.eq pt1, pt2 = t3, t0 // if eq, cmpxchg8 succeeded
|
|
;;
|
|
(pt2) mov ar.ccv = t3
|
|
(pt1) br.ret.sptk brp
|
|
;;
|
|
mov t0 = t3
|
|
shr.u v0 = t3, SLIST_ADR_BITS_START
|
|
br Efls10 // try again
|
|
|
|
|
|
LEAF_EXIT(RtlpInterlockedFlushSList)
|
|
|
|
SBTTL("Interlocked Pop Entry Sequenced List")
|
|
//++
|
|
//
|
|
// PSINGLE_LIST_ENTRY
|
|
// RtlpInterlockedPopEntrySList (
|
|
// IN PSLIST_HEADER ListHead
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// This function removes an entry from the front of a sequenced singly
|
|
// linked list so that access to the list is synchronized in a MP system.
|
|
// If there are no entries in the list, then a value of NULL is returned.
|
|
// Otherwise, the address of the entry that is removed is returned as the
|
|
// function value.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// ListHead (a0) - Supplies a pointer to the sequenced listhead from which
|
|
// an entry is to be removed.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// The address of the entry removed from the list, or NULL if the list is
|
|
// empty.
|
|
//
|
|
//--
|
|
|
|
LEAF_ENTRY(RtlpInterlockedPopEntrySList)
|
|
ALTERNATE_ENTRY(ExpInterlockedPopEntrySList)
|
|
ALTERNATE_ENTRY(ExpInterlockedPopEntrySListResume)
|
|
|
|
|
|
ld8.acq.nt1 t0 = [a0] // load next entry & sequence
|
|
#if defined (NTOS_KERNEL_RUNTIME)
|
|
add t1 = 8, a0 // Generate address region bits
|
|
#endif
|
|
;;
|
|
|
|
Epop10:
|
|
|
|
#if defined (NTOS_KERNEL_RUNTIME)
|
|
ld8 t1 = [t1] // capture region bits
|
|
#endif
|
|
mov ar.ccv = t0
|
|
shr.u t2 = t0, SLIST_ADR_BITS_START
|
|
;;
|
|
|
|
#if defined (NTOS_KERNEL_RUNTIME)
|
|
shladd v0 = t2, SLIST_ADR_ALIGMENT, t1 // merge region & va bits
|
|
#else
|
|
shl v0 = t2, SLIST_ADR_ALIGMENT // shift va into place
|
|
#endif
|
|
;;
|
|
|
|
#if defined (NTOS_KERNEL_RUNTIME)
|
|
cmp.eq pt2, pt1 = t1, v0 // if eq, list is empty
|
|
#else
|
|
cmp.eq pt2, pt1 = r0, v0 // if eq, list is empty
|
|
#endif
|
|
sub t4 = t0, zero, 1 // adjust depth
|
|
;;
|
|
|
|
//
|
|
// N.B. It is possible for the following instruction to fault in the rare
|
|
// case where the first entry in the list is allocated on another
|
|
// processor and free between the time the free pointer is read above
|
|
// and the following instruction. When this happens, the access fault
|
|
// code continues execution by skipping the following instruction slot.
|
|
// This results in the compare failing and the entire operation is
|
|
// retried.
|
|
//
|
|
|
|
ALTERNATE_ENTRY(ExpInterlockedPopEntrySListFault)
|
|
|
|
(pt1) ld8.acq t5 = [v0] // get addr of successor entry
|
|
extr.u t4 = t4, 0, SLIST_DEPTH_BITS_SIZE + \
|
|
SLIST_SEQ_BITS_SIZE // extract depth & sequence
|
|
;;
|
|
(pt1) shl t5 = t5, SLIST_ADR_BITS_START - \
|
|
SLIST_ADR_ALIGMENT // shift va into position dropping alignment bits
|
|
(pt2) add v0 = 0, zero // The list is empty clear retur value
|
|
;;
|
|
|
|
(pt1) ld8.nt1 t3 = [a0] // reload next entry & sequence
|
|
(pt1) or t5 = t4, t5 // merge va, depth & sequence
|
|
(pt2) br.ret.spnt.clr brp // return if the list is null
|
|
;;
|
|
cmp.eq.unc pt4, pt1 = t3, t0
|
|
;;
|
|
|
|
ALTERNATE_ENTRY(ExpInterlockedPopEntrySListEnd)
|
|
|
|
(pt4) cmpxchg8.rel t3 = [a0], t5, ar.ccv // perform the pop
|
|
nop.i 0
|
|
;;
|
|
|
|
(pt4) cmp.eq.unc pt3 = t3, t0 // if eq, cmpxchg8 succeeded
|
|
#if defined (NTOS_KERNEL_RUNTIME)
|
|
add t1 = 8, a0
|
|
#endif
|
|
mov t0 = t3
|
|
(pt3) br.ret.sptk brp
|
|
|
|
br Epop10 // try again
|
|
|
|
LEAF_EXIT(RtlpInterlockedPopEntrySList)
|
|
|
|
//++
|
|
//
|
|
// PSINGLE_LIST_ENTRY
|
|
// RtlpInterlockedPushEntrySList (
|
|
// IN PSLIST_HEADER ListHead,
|
|
// IN PSINGLE_LIST_ENTRY ListEntry
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// This function inserts an entry at the head of a sequenced singly linked
|
|
// list so that access to the list is synchronized in an MP system.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// ListHead (a0) - Supplies a pointer to the sequenced listhead into which
|
|
// an entry is to be inserted.
|
|
//
|
|
// ListEntry (a1) - Supplies a pointer to the entry to be inserted at the
|
|
// head of the list.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// Previous contents of ListHead. NULL implies list went from empty
|
|
// to not empty.
|
|
//
|
|
//--
|
|
|
|
LEAF_ENTRY(RtlpInterlockedPushEntrySList)
|
|
ALTERNATE_ENTRY(ExpInterlockedPushEntrySList)
|
|
|
|
|
|
ld8.acq.nt1 t0 = [a0] // load next entry & sequence
|
|
mov t6 = (1<<SLIST_SEQ_BITS_START)+1 // Increment sequence and depth
|
|
ldf.fill ft0 = [a1] // Force alignment check.
|
|
shl t5 = a1, SLIST_ADR_BITS_START - \
|
|
SLIST_ADR_ALIGMENT // shift va into position
|
|
;;
|
|
|
|
#if DBG==1
|
|
and t3 = (1<<SLIST_ADR_ALIGMENT)-1, a1
|
|
add t2 = 8, a0
|
|
dep t1 = 0, a1, 0, 61 // capture region bits
|
|
;;
|
|
cmp.eq pt2, pt1 = t3, r0
|
|
#if defined (NTOS_KERNEL_RUNTIME)
|
|
ld8 t2 = [t2]
|
|
;;
|
|
(pt2) cmp.eq pt2, pt1 = t2, t1
|
|
#endif
|
|
;;
|
|
(pt1) break.i 0x80016
|
|
#endif
|
|
|
|
Epush10:
|
|
mov ar.ccv = t0 // set the comparand
|
|
add t4 = t0, t6
|
|
shr.u v0 = t0, SLIST_ADR_BITS_START
|
|
|
|
;;
|
|
#if defined (NTOS_KERNEL_RUNTIME)
|
|
dep t1 = 0, a1, 0, 61 // capture region bits
|
|
#endif
|
|
cmp.ne pt3 = zero, v0 // if ne, list not empty
|
|
;;
|
|
#if defined (NTOS_KERNEL_RUNTIME)
|
|
(pt3) shladd v0 = v0, SLIST_ADR_ALIGMENT, t1 // merge region & va bits
|
|
#else
|
|
(pt3) shl v0 = v0, SLIST_ADR_ALIGMENT // shift address into place
|
|
#endif
|
|
extr.u t4 = t4, 0, SLIST_DEPTH_BITS_SIZE + \
|
|
SLIST_SEQ_BITS_SIZE // extract depth & sequence
|
|
;;
|
|
|
|
st8 [a1] = v0
|
|
or t5 = t4, t5 // merge va, depth & sequence
|
|
;;
|
|
|
|
cmpxchg8.rel t3 = [a0], t5, ar.ccv
|
|
;;
|
|
cmp.eq pt2, pt1 = t0, t3
|
|
dep t5 = 0, t5, 0, SLIST_DEPTH_BITS_SIZE + \
|
|
SLIST_SEQ_BITS_SIZE // Zero depth a sequence
|
|
;;
|
|
(pt2) br.ret.sptk brp // if equal, return
|
|
(pt1) mov t0 = t3
|
|
(pt1) br.spnt Epush10 // retry
|
|
;;
|
|
|
|
LEAF_EXIT(RtlpInterlockedPushEntrySList)
|
|
|
|
|
|
//++
|
|
//
|
|
// SINGLE_LIST_ENTRY
|
|
// FASTCALL
|
|
// InterlockedPushListSList (
|
|
// IN PSLIST_HEADER ListHead,
|
|
// IN PSINGLE_LIST_ENTRY List,
|
|
// IN PSINGLE_LIST_ENTRY ListEnd,
|
|
// IN ULONG Count
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// This function will push multiple entries onto an SList at once
|
|
//
|
|
// Arguments:
|
|
//
|
|
// ListHead - List head to push the list to.
|
|
//
|
|
// List - The list to add to the front of the SList
|
|
// ListEnd - The last element in the chain
|
|
// Count - The number of items in the chain
|
|
//
|
|
// Return Value:
|
|
//
|
|
// PSINGLE_LIST_ENTRY - The old header pointer is returned
|
|
//
|
|
//--
|
|
|
|
LEAF_ENTRY(InterlockedPushListSList)
|
|
|
|
|
|
ld8.acq.nt1 t0 = [a0] // load next entry & sequence
|
|
mov t6 = 0x10000 // Generate literal for adjusting depth and count
|
|
ldf.fill ft0 = [a1] // Force alignment check.
|
|
shl t5 = a1, SLIST_ADR_BITS_START - \
|
|
SLIST_ADR_ALIGMENT // shift va into position
|
|
;;
|
|
|
|
#if DBG==1
|
|
and t3 = (1<<SLIST_ADR_ALIGMENT)-1, a1
|
|
add t2 = 8, a0
|
|
dep t1 = 0, a1, 0, 61 // capture region bits
|
|
;;
|
|
cmp.eq pt2, pt1 = t3, r0
|
|
#if defined (NTOS_KERNEL_RUNTIME)
|
|
ld8 t2 = [t2]
|
|
;;
|
|
(pt2) cmp.eq pt2, pt1 = t2, t1
|
|
#endif
|
|
;;
|
|
(pt1) break.i 0x80016
|
|
#endif
|
|
|
|
|
|
Epushl10:
|
|
mov ar.ccv = t0 // set the comparand
|
|
add t4 = t0, t6
|
|
shr.u v0 = t0, SLIST_ADR_BITS_START
|
|
;;
|
|
#if defined (NTOS_KERNEL_RUNTIME)
|
|
dep t1 = 0, a1, 0, 61 // capture region bits
|
|
#endif
|
|
cmp.ne pt3 = zero, v0 // if ne, list not empty
|
|
add t4 = t4, a3 // Add in count
|
|
;;
|
|
#if defined (NTOS_KERNEL_RUNTIME)
|
|
(pt3) shladd v0 = v0, SLIST_ADR_ALIGMENT, t1 // merge region & va bits
|
|
#else
|
|
(pt3) shl v0 = v0, SLIST_ADR_ALIGMENT // shift address into place
|
|
#endif
|
|
extr.u t4 = t4, 0, SLIST_DEPTH_BITS_SIZE + \
|
|
SLIST_SEQ_BITS_SIZE // extract depth & sequence
|
|
;;
|
|
|
|
st8 [a2] = v0
|
|
or t5 = t4, t5 // merge va, depth & sequence
|
|
;;
|
|
|
|
cmpxchg8.rel t3 = [a0], t5, ar.ccv
|
|
;;
|
|
cmp.eq pt2, pt1 = t0, t3
|
|
dep t5 = 0, t5, 0, SLIST_DEPTH_BITS_SIZE + \
|
|
SLIST_SEQ_BITS_SIZE // zero depth and sequence
|
|
|
|
mov t0 = t3
|
|
(pt2) br.ret.sptk brp // if equal, return
|
|
(pt1) br.spnt Epushl10 // retry
|
|
;;
|
|
|
|
LEAF_EXIT(InterlockedPushListSList)
|
|
|
|
//++
|
|
//
|
|
// PSINGLE_LIST_ENTRY
|
|
// FirstEntrySList (
|
|
// IN PSLIST_HEADER SListHead
|
|
// )
|
|
//
|
|
// Routine Description:
|
|
//
|
|
// This function returns the address of the fisrt entry in the SLIST or
|
|
// NULL.
|
|
//
|
|
// Arguments:
|
|
//
|
|
// ListHead - Supplies a pointer to the sequenced listhead from
|
|
// which the first entry address is to be computed.
|
|
//
|
|
// Return Value:
|
|
//
|
|
// The address of the first entry is the specified, or NULL if the list is
|
|
// empty.
|
|
//
|
|
//--
|
|
LEAF_ENTRY(FirstEntrySList)
|
|
|
|
ld8 t0 = [a0] // load next entry & sequence
|
|
#if defined (NTOS_KERNEL_RUNTIME)
|
|
add t1 = 8, a0 // Generate address region bits
|
|
#endif
|
|
;;
|
|
|
|
#if defined (NTOS_KERNEL_RUNTIME)
|
|
ld8 t1 = [t1] // capture region bits
|
|
#endif
|
|
shr.u t2 = t0, SLIST_ADR_BITS_START
|
|
;;
|
|
|
|
#if defined (NTOS_KERNEL_RUNTIME)
|
|
cmp.eq pt0 = t2, r0
|
|
shladd v0 = t2, SLIST_ADR_ALIGMENT, t1 // merge region & va bits
|
|
#else
|
|
shl v0 = t2, SLIST_ADR_ALIGMENT // shift va into place
|
|
#endif
|
|
;;
|
|
|
|
#if defined (NTOS_KERNEL_RUNTIME)
|
|
(pt0) mov v0 = r0
|
|
#endif
|
|
br.ret.sptk brp // return function value
|
|
|
|
|
|
LEAF_EXIT(FirstEntrySList)
|