#include "ksia64.h"
LEAF_ENTRY(memmove)
.prologue
.regstk 3,7,0,8
alloc t17 = ar.pfs,3,31,0,32
.save pr, r64
mov r64 = pr
and t3 = -32, a1
;;
lfetch [t3], 32 //0
.save ar.lc, r65
mov.i r65 = ar.lc
and t1 = 7, a1
;;
.body
lfetch [t3], 32 //32
mov v0 = a0
and t0 = 7, a0
;;
add t21 = a1, a2
cmp.gtu pt0 = a0, a1
or t2 = t0, t1
;;
(pt0) cmp.ltu.unc pt0 = a0, t21
cmp.eq pt1 = zero, a2
(pt1) br.ret.spnt brp
lfetch [t3], 32 //64
cmp.lt pt2 = 16, a2
(pt0) br.cond.spnt CopyDown
;;
lfetch [t3], 32 //96
cmp.lt pt6 = 127, a2
cmp.le pt4 = 8, a2
;;
(pt6) lfetch [t3], 32 //128
(pt4) cmp.eq.unc pt3 = 0, t2
(pt4) cmp.eq.unc pt5 = t0, t1
(pt3) br.cond.sptk QwordMoveUp
(pt5) br.cond.spnt AlignedMove
(pt2) br.cond.sptk UnalignedMove
ByteMoveUpLoop:
ld1 t10 = [a1], 1
nop.f 0
add a2 = -1, a2
;;
st1 [a0] = t10, 1
cmp.ne pt1 = zero, a2
(pt1) br.cond.sptk ByteMoveUpLoop
nop.m 0
nop.f 0
br.ret.sptk brp
UnalignedMove:
cmp.eq pt0 = 0, t1
sub t1 = 8, t1
(pt0) br.cond.spnt SkipUnalignedMoveByteLoop
;;
UnalignedMoveByteLoop:
ld1 t10 = [a1], 1
add t1 = -1, t1
add a2 = -1, a2
;;
st1 [a0] = t10, 1
cmp.eq p0, pt1 = zero, t1
(pt1) br.cond.sptk UnalignedMoveByteLoop
;;
SkipUnalignedMoveByteLoop:
and t0 = 7, a0
mov pr.rot = 3<<16
or t1 = a1, r0
;;
add t2 = a2, t0
mov.i ar.ec = 32
sub t21 = 8, t0
;;
sub t4 = a0, t0
shr t10 = t2, 3
shl t21 = t21, 3
;;
ld8 r33 = [t4], 0
add t10 = -1,t10
and t2 = 7, t2
;;
cmp.eq pt0 = 2, t0
cmp.eq pt3 = 4, t0
cmp.eq pt5 = 6, t0
;;
nop.m 0
shl r33 = r33,t21 // Prime r39
mov.i ar.lc = t10
(pt0) br.cond.spnt SpecialLoop2
(pt3) br.cond.spnt SpecialLoop4
(pt5) br.cond.spnt SpecialLoop6
cmp.eq pt1 = 3, t0
cmp.eq pt4 = 5, t0
cmp.eq pt6 = 7, t0
(pt1) br.cond.spnt SpecialLoop3
(pt4) br.cond.spnt SpecialLoop5
(pt6) br.cond.spnt SpecialLoop7
;;
SpecialLoop1:
(p16) ld8 r32 = [t1], 8
nop.f 0
brp.sptk.imp SpecialLoop1E, SpecialLoop1
SpecialLoop1E:
(p48) st8 [t4] = r10, 8
(p47) shrp r10 = r62,r63,56
br.ctop.sptk.many SpecialLoop1
br UnalignedByteDone
SpecialLoop2:
(p16) ld8 r32 = [t1], 8
nop.f 0
brp.sptk.imp SpecialLoop2E, SpecialLoop2
SpecialLoop2E:
(p48) st8 [t4] = r10, 8
(p47) shrp r10 = r62,r63,48
br.ctop.sptk.many SpecialLoop2
br UnalignedByteDone
SpecialLoop3:
(p16) ld8 r32 = [t1], 8
nop.f 0
brp.sptk.imp SpecialLoop3E, SpecialLoop3
SpecialLoop3E:
(p48) st8 [t4] = r10, 8
(p47) shrp r10 = r62,r63,40
br.ctop.sptk.many SpecialLoop3
br UnalignedByteDone
SpecialLoop4:
(p16) ld8 r32 = [t1], 8
nop.f 0
brp.sptk.imp SpecialLoop4E, SpecialLoop4
SpecialLoop4E:
(p48) st8 [t4] = r10, 8
(p47) shrp r10 = r62,r63,32
br.ctop.sptk.many SpecialLoop4
br UnalignedByteDone
SpecialLoop5:
(p16) ld8 r32 = [t1], 8
nop.f 0
brp.sptk.imp SpecialLoop5E, SpecialLoop5
SpecialLoop5E:
(p48) st8 [t4] = r10, 8
(p47) shrp r10 = r62,r63,24
br.ctop.sptk.many SpecialLoop5
br UnalignedByteDone
SpecialLoop6:
(p16) ld8 r32 = [t1], 8
nop.f 0
brp.sptk.imp SpecialLoop6E, SpecialLoop6
SpecialLoop6E:
(p48) st8 [t4] = r10, 8
(p47) shrp r10 = r62,r63,16
br.ctop.sptk.many SpecialLoop6
br UnalignedByteDone
SpecialLoop7:
(p16) ld8 r32 = [t1], 8
nop.f 0
brp.sptk.imp SpecialLoop7E, SpecialLoop7
SpecialLoop7E:
(p48) st8 [t4] = r10, 8
(p47) shrp r10 = r62,r63,8
br.ctop.sptk.many SpecialLoop7;;
UnalignedByteDone:
sub t1 = t1, t0
mov pr = r64
mov.i ar.lc = r65
;;
cmp.eq pt0 = zero, t2
(pt0) br.ret.spnt brp
UnAlignedByteDoneLoop:
ld1 t10 = [t1], 1
add t2 = -1, t2
;;
cmp.ne pt1 = zero, t2
st1 [t4] = t10, 1
(pt1) br.cond.sptk UnAlignedByteDoneLoop
br.ret.spnt brp
AlignedMove:
add t4 = 64, t3
(pt6) lfetch [t3], 32 //160
sub t22 = 8, t0
;;
(pt6) lfetch [t3], 64 //192
(pt6) lfetch [t4], 96 //224
sub a2 = a2, t22
;;
AlignedMoveByteLoop:
ld1 t10 = [a1], 1
nop.f 0
add t22 = -1, t22
;;
st1 [a0] = t10, 1
cmp.ne pt1 = zero, t22
(pt1) br.cond.sptk AlignedMoveByteLoop
;;
(pt6) lfetch [t3], 32 //256
cmp.eq.unc pt0 = zero, a2
cmp.gt pt2 = 8, a2
(pt6) lfetch [t4], 128 //320
(pt0) br.ret.spnt brp
(pt2) br.cond.sptk ByteMoveUpLoop
;;
//
// both src & dest are now 8-byte aligned
//
QwordMoveUp:
add t3 = 128, a1
add t4 = 288, a1
add t7 = 8, a1
add t8 = 8, a0
cmp.gt pt3 = 64, a2
(pt3) br.cond.spnt QwordMoveUpLoop
;;
UnrolledQwordMoveUpLoop:
ld8 t10 = [a1], 16
ld8 t11 = [t7], 16
add a2 = -64, a2
;;
ld8 t12 = [a1], 16
ld8 t13 = [t7], 16
cmp.le pt3 = 128, a2
;;
ld8 t14 = [a1], 16
ld8 t15 = [t7], 16
cmp.gt pt2 = 8, a2
;;
ld8 t16 = [a1], 16
ld8 t17 = [t7], 16
;;
(pt3) lfetch [t3], 64
(pt3) lfetch [t4], 64
st8 [a0] = t10, 16
st8 [t8] = t11, 16
;;
st8 [a0] = t12, 16
st8 [t8] = t13, 16
;;
st8 [a0] = t14, 16
st8 [t8] = t15, 16
;;
st8 [a0] = t16, 16
st8 [t8] = t17, 16
(pt3) br.cond.dptk UnrolledQwordMoveUpLoop
(pt2) br.cond.spnt ByteMoveUp
;;
QwordMoveUpLoop:
ld8 t10 = [a1], 8
add a2 = -8, a2
;;
cmp.le pt1 = 8, a2
st8 [a0] = t10, 8
(pt1) br.cond.sptk QwordMoveUpLoop
;;
ByteMoveUp:
cmp.eq pt0 = zero, a2
(pt0) br.ret.spnt brp
;;
AlignedByteDoneLoop:
ld1 t10 = [a1], 1
add a2 = -1, a2
;;
cmp.ne pt1 = zero, a2
st1 [a0] = t10, 1
(pt1) br.cond.sptk AlignedByteDoneLoop
br.ret.spnt brp
;;
CopyDown:
cmp.eq pt0 = zero, a2
cmp.ne pt6 = t0, t1
(pt0) br.ret.spnt brp // return if length is zero
cmp.gt pt4 = 16, a2
add t20 = a2, a0
add t21 = a2, a1
nop.m 0
(pt4) br.cond.sptk ByteMoveDown // less than 16 bytes to copy
(pt6) br.cond.spnt UnalignedMoveDown // incompatible alignment
;;
nop.m 0
nop.m 0
and t22 = 0x7, t21
;;
add t20 = -1, t20
add t21 = -1, t21
sub a2 = a2, t22
;;
TailMove:
cmp.eq pt0, pt1 = zero, t22
;;
(pt1) ld1 t10 = [t21], -1
(pt1) add t22 = -1, t22
;;
(pt1) st1 [t20] = t10, -1
(pt1) br.cond.sptk TailMove
Block8Move:
nop.m 0
add t20 = -7, t20
add t21 = -7, t21
;;
Block8MoveLoop:
cmp.gt pt5, pt6 = 8, a2
;;
(pt6) ld8 t10 = [t21], -8
(pt6) add a2 = -8, a2
;;
(pt6) st8 [t20] = t10, -8
(pt6) br.cond.sptk Block8MoveLoop
add t20 = 8, t20 // adjust dest
add t21 = 8, t21 // adjust source
br.cond.sptk ByteMoveDown
;;
UnalignedMoveDown:
and t1 = 7, t21
;;
cmp.eq pt0 = 0, t1
(pt0) br.cond.spnt SkipUnalignedMoveDownByteLoop
;;
add t20 = -1, t20
add t21 = -1, t21
;;
UnalignedMoveDownByteLoop:
ld1 t10 = [t21], -1
add t1 = -1, t1
add a2 = -1, a2
;;
st1 [t20] = t10, -1
cmp.eq p0, pt1 = zero, t1
(pt1) br.cond.sptk UnalignedMoveDownByteLoop
;;
add t20 = 1, t20
add t21 = 1, t21
;;
SkipUnalignedMoveDownByteLoop:
add t21 = -8, t21
;;
and t0 = 7, t20
mov pr.rot = 3<<16
or t1 = t21, r0
;;
sub t7 = 8, t0
;;
add t2 = a2, t7
mov.i ar.ec = 32
;;
sub t4 = t20, t0
shr t10 = t2, 3
shl t6 = t0, 3
;;
ld8 r33 = [t4], 0
add t10 = -1,t10
and t2 = 7, t2
;;
cmp.eq pt0 = 2, t0
cmp.eq pt3 = 4, t0
cmp.eq pt5 = 6, t0
;;
shr r33 = r33,t6 // Prime r39
mov.i ar.lc = t10
(pt0) br.cond.spnt SpecialLoopDown2
(pt3) br.cond.spnt SpecialLoopDown4
(pt5) br.cond.spnt SpecialLoopDown6
cmp.eq pt1 = 3, t0
cmp.eq pt4 = 5, t0
cmp.eq pt6 = 7, t0
(pt1) br.cond.spnt SpecialLoopDown3
(pt4) br.cond.spnt SpecialLoopDown5
(pt6) br.cond.spnt SpecialLoopDown7
;;
SpecialLoopDown1:
(p16) ld8 r32 = [t1], -8
nop.f 0
brp.sptk.imp SpecialLoopDown1E, SpecialLoopDown1
SpecialLoopDown1E:
(p48) st8 [t4] = r10, -8
(p47) shrp r10 = r63,r62,56
br.ctop.sptk.many SpecialLoopDown1
br UnalignedByteDownDone
SpecialLoopDown2:
(p16) ld8 r32 = [t1], -8
nop.f 0
brp.sptk.imp SpecialLoopDown2E, SpecialLoopDown2
SpecialLoopDown2E:
(p48) st8 [t4] = r10, -8
(p47) shrp r10 = r63,r62,48
br.ctop.sptk.many SpecialLoopDown2
br UnalignedByteDownDone
SpecialLoopDown3:
(p16) ld8 r32 = [t1], -8
nop.f 0
brp.sptk.imp SpecialLoopDown3E, SpecialLoopDown3
SpecialLoopDown3E:
(p48) st8 [t4] = r10, -8
(p47) shrp r10 = r63,r62,40
br.ctop.sptk.many SpecialLoopDown3
br UnalignedByteDownDone
SpecialLoopDown4:
(p16) ld8 r32 = [t1], -8
nop.f 0
brp.sptk.imp SpecialLoopDown4E, SpecialLoopDown4
SpecialLoopDown4E:
(p48) st8 [t4] = r10, -8
(p47) shrp r10 = r63,r62,32
br.ctop.sptk.many SpecialLoopDown4
br UnalignedByteDownDone
SpecialLoopDown5:
(p16) ld8 r32 = [t1], -8
nop.f 0
brp.sptk.imp SpecialLoopDown5E, SpecialLoopDown5
SpecialLoopDown5E:
(p48) st8 [t4] = r10, -8
(p47) shrp r10 = r63,r62,24
br.ctop.sptk.many SpecialLoopDown5
br UnalignedByteDownDone
SpecialLoopDown6:
(p16) ld8 r32 = [t1], -8
nop.f 0
brp.sptk.imp SpecialLoopDown6E, SpecialLoopDown6
SpecialLoopDown6E:
(p48) st8 [t4] = r10, -8
(p47) shrp r10 = r63,r62,16
br.ctop.sptk.many SpecialLoopDown6
br UnalignedByteDownDone
SpecialLoopDown7:
(p16) ld8 r32 = [t1], -8
nop.f 0
brp.sptk.imp SpecialLoopDown7E, SpecialLoopDown7
SpecialLoopDown7E:
(p48) st8 [t4] = r10, -8
(p47) shrp r10 = r63,r62,8
br.ctop.sptk.many SpecialLoopDown7;;
UnalignedByteDownDone:
add t1 = 7, t1
add t4 = 7, t4
;;
add t1 = t1, t7
mov pr = r64
mov.i ar.lc = r65
;;
cmp.eq pt0 = zero, t2
(pt0) br.ret.spnt brp
;;
UnAlignedByteDoneDownLoop:
ld1 t10 = [t1], -1
add t2 = -1, t2
;;
cmp.ne pt1 = zero, t2
st1 [t4] = t10, -1
(pt1) br.cond.sptk UnAlignedByteDoneDownLoop
br.ret.spnt brp
ByteMoveDown:
nop.m 0
add t20 = -1, t20 // adjust source
add t21 = -1, t21 // adjust destination
;;
ByteMoveDownLoop:
cmp.ne pt1 = zero, a2
;;
(pt1) ld1 t10 = [t21], -1
(pt1) add a2 = -1, a2
;;
(pt1) st1 [t20] = t10, -1
(pt1) br.cond.sptk ByteMoveDownLoop
br.ret.spnt brp
;;
LEAF_EXIT(memmove)