#include "kxia64.h"
#include "regia64.h"
 
//++
//
// VOID
// KiZeroPages (
//    IN PVOID PageBase,
//    IN SIZE_T NumberOfBytes
//    )
//
//--

//
// Based on the original code assumption, NumberOfBytes >= 2048 and
// is a multiple of 128.
//
// This code is optimized for McKinley CPU.
//

     LEAF_ENTRY(KiZeroPages)
     .prologue
     .regstk                 2,0,0,0

//
// Note: Do not delete the nop bundle below. It seemed to improve the performance
// by 150 cycles with this extra bundle. But the reason for it is unexplanable at
// this time... we're in the process of investigating it.
//

{    .mmi
     nop.m                   0
     nop.m                   0
     nop.i                   0
}

//
// Do 16 lfetch.fault.excl.nt1 to ensure that the L2 cache line is ready to receive the store data.
// The .fault is to ensure that the data enters into the cache hierarchy.
// The .nt1 is to ensure that the data will not displace data residing in the L1D.
// The .excl is to ensure that the data is ready to be modified.
// 16 lfetches seemed to be an optimal value for McKinley.
//

     .save                   ar.lc, r31

{    .mmi
     add                     r14 = r0, r32			// pointer to 0th cache line
     add                     r15 = 0x400, r32			// pointer to 8th cache line
     mov.i                   r31 = ar.lc			// save ar.lc; to be restored at the end
     ;;
}
{    .mmi
     lfetch.fault.excl.nt1   [r14], 0x80			// Note: lfetch increment must be in
     lfetch.fault.excl.nt1   [r15], 0x80			// this range (-256 to 255).
     add                     r16 = r0, r32                      // r16 == 1st store pointer
     ;;
}
{    .mmi
     lfetch.fault.excl.nt1   [r14], 0x80
     lfetch.fault.excl.nt1   [r15], 0x80
     add                     r17 = 0x10, r32                    // r17 == 2nd store pointer
     ;;
}
{    .mmi
     lfetch.fault.excl.nt1   [r14], 0x80
     lfetch.fault.excl.nt1   [r15], 0x80
     shr.u                   r18 = r33, 7			// number of 128-byte blocks
     ;;
}
{    .mmi
     lfetch.fault.excl.nt1   [r14], 0x80
     lfetch.fault.excl.nt1   [r15], 0x80
     adds                    r18 = -1, r18			// Loop Count
     ;;
}
{    .mmi
     lfetch.fault.excl.nt1   [r14], 0x80
     lfetch.fault.excl.nt1   [r15], 0x80
     mov.i                   ar.lc = r18
     ;;
}
{    .mmi
     lfetch.fault.excl.nt1   [r14], 0x80
     lfetch.fault.excl.nt1   [r15], 0x80
     add                     r19 = r32, r33                     // r19 == lfetch stop address
     ;;
}
{    .mmi
     lfetch.fault.excl.nt1   [r14], 0x80
     lfetch.fault.excl.nt1   [r15], 0x80
     nop.i                   0
     ;;
}
{    .mmi
     lfetch.fault.excl.nt1   [r14], 0x80
     lfetch.fault.excl.nt1   [r15], 0x80			// r15 will continue to be used for lfetch below
     nop.i                   0
     ;;
}

Mizp10:
{    .mmi
     stf.spill.nta           [r16] = f0, 0x20			// store 16 bytes at 1st pointer
     stf.spill.nta           [r17] = f0, 0x20			// store 16 bytes at 2nd pointer
     cmp.lt                  p8, p0 = r15, r19			// if r15 >= r32+r33, don't lfetch
     ;;
}
{    .mmi
     stf.spill.nta           [r16] = f0, 0x20
     stf.spill.nta           [r17] = f0, 0x20
     nop.i                   0
     ;;
}
{    .mmi
     stf.spill.nta           [r16] = f0, 0x20
     stf.spill.nta           [r17] = f0, 0x20
     nop.i                   0
     ;;
}
{    .mmi
     stf.spill.nta           [r16] = f0, 0x20
     stf.spill.nta           [r17] = f0, 0x20
     nop.i                   0
}

//
// Note: On McKinley, this added lfetch instruction below does not add any extra cycle.
// Since the bundle above and this bundle can be issued in one cycle (since no stop bits
// in between). Without the lfetch, the br instr could be combined with the above bundle,
// but only one bundle can be issued in this case.
//
{    .mib
(p8) lfetch.fault.excl.nt1   [r15], 0x80
     nop.i                   0
     br.cloop.sptk           Mizp10
     ;;
}
{    .mib
     nop.m                   0
     mov.i                   ar.lc = r31			// restore ar.lc for the caller
     br.ret.sptk             b0
}
     LEAF_EXIT(KiZeroPages)