title "Compute Checksum"
;/*++
;
; Copyright (c) Microsoft Corporation. All rights reserved.
;
; Module Name:
;
; xsum.x86
;
; Abstract:
;
; This module implements a function to compute the checksum of a buffer.
;
; Author:
;
; David N. Cutler (davec) 27-Jan-1992
;
; Revision History:
;
; Who When What
; -------- -------- ----------------------------------------------
; mikeab 01-22-94 Pentium optimization
;
; Environment:
;
; Any mode.
;
; Revision History:
;
;--*/
LOOP_UNROLLING_BITS equ 5
LOOP_UNROLLING equ (1 SHL LOOP_UNROLLING_BITS)
.386
.model small,c
assume cs:FLAT,ds:FLAT,es:FLAT,ss:FLAT
assume fs:nothing,gs:nothing
.xlist
include callconv.inc
include ks386.inc
.list
.code
;++
;
; ULONG
; tcpxsum(
; IN ULONG cksum,
; IN PUCHAR buf,
; IN ULONG len
; )
;
; Routine Description:
;
; This function computes the checksum of the specified buffer.
;
; Arguments:
;
; cksum - Suppiles the initial checksum value, in 16-bit form,
; with the high word set to 0.
;
; buf - Supplies a pointer to the buffer to the checksum buffer.
;
; len - Supplies the length of the buffer in bytes.
;
; Return Value:
;
; The computed checksum in 32-bit two-partial-accumulators form, added to
; the initial checksum, is returned as the function value.
;
;--
cksum equ 12 ; stack offset to initial checksum
buf equ 16 ; stack offset to source address
len equ 20 ; stack offset to length in words
to_checksum_last_word:
jmp checksum_last_word
to_checksum_done:
jmp checksum_done
to_checksum_dword_loop_done:
jmp checksum_dword_loop_done
cPublicProc tcpxsum,3
push ebx ; save nonvolatile register
push esi ; save nonvolatile register
mov ecx,[esp + len] ; get length in bytes
sub eax,eax ; clear computed checksum
test ecx,ecx ; any bytes to checksum at all?
jz short to_checksum_done ; no bytes to checksum
;
; if the checksum buffer is not word aligned, then add the first byte of
; the buffer to the input checksum.
;
mov esi,[esp + buf] ; get source address
sub edx,edx ; set up to load word into EDX below
test esi,1 ; check if buffer word aligned
jz short checksum_word_aligned ; if zf, buffer word aligned
mov ah,[esi] ; get first byte (we know we'll have
; to swap at the end)
inc esi ; increment buffer address
dec ecx ; decrement number of bytes
jz short to_checksum_done ; if zf set, no more bytes
;
; If the buffer is not an even number of of bytes, then initialize
; the computed checksum with the last byte of the buffer.
;
checksum_word_aligned: ;
shr ecx,1 ; convert to word count
jnc short checksum_start ; if nc, even number of bytes
mov al,[esi+ecx*2] ; initialize the computed checksum
jz short to_checksum_done ; if zf set, no more bytes
;
; Compute checksum in large blocks of dwords, with one partial word up front if
; necessary to get dword alignment, and another partial word at the end if
; needed.
;
;
; Compute checksum on the leading word, if that's necessary to get dword
; alignment.
;
checksum_start: ;
test esi,02h ; check if source dword aligned
jz short checksum_dword_aligned ; source is already dword aligned
mov dx,[esi] ; get first word to checksum
add esi,2 ; update source address
add eax,edx ; update partial checksum
; (no carry is possible, because EAX
; and EDX are both 16-bit values)
dec ecx ; count off this word (zero case gets
; picked up below)
;
; Checksum as many words as possible by processing a dword at a time.
;
checksum_dword_aligned:
push ecx ; so we can tell if there's a trailing
; word later
shr ecx,1 ; # of dwords to checksum
jz short to_checksum_last_word ; no dwords to checksum
mov edx,[esi] ; preload the first dword
add esi,4 ; point to the next dword
dec ecx ; count off the dword we just loaded
jz short to_checksum_dword_loop_done
; skip the loop if that was the only
; dword
mov ebx,ecx ; EBX = # of dwords left to checksum
add ecx,LOOP_UNROLLING-1 ; round up loop count
shr ecx,LOOP_UNROLLING_BITS ; convert from word count to unrolled
; loop count
and ebx,LOOP_UNROLLING-1 ; # of partial dwords to do in first
; loop
jz short checksum_dword_loop ; special-case when no partial loop,
; because fixup below doesn't work
; in that case (carry flag is
; cleared at this point, as required
; at loop entry)
lea esi,[esi+ebx*4-(LOOP_UNROLLING*4)]
; adjust buffer pointer back to
; compensate for hardwired displacement
; at loop entry point
; ***doesn't change carry flag***
jmp loop_entry[ebx*4] ; enter the loop to do the first,
; partial iteration, after which we can
; just do 64-word blocks
; ***doesn't change carry flag***
checksum_dword_loop:
DEFLAB macro pre,suf
pre&suf:
endm
TEMP=0
REPT LOOP_UNROLLING
deflab loop_entry_,%TEMP
adc eax,edx
mov edx,[esi + TEMP]
TEMP=TEMP+4
ENDM
checksum_dword_loop_end:
lea esi,[esi + LOOP_UNROLLING * 4] ; update source address
; ***doesn't change carry flag***
dec ecx ; count off unrolled loop iteration
; ***doesn't change carry flag***
jnz checksum_dword_loop ; do more blocks
checksum_dword_loop_done label proc
adc eax,edx ; finish dword checksum
mov edx,0 ; prepare to load trailing word
adc eax,edx
;
; Compute checksum on the trailing word, if there is one.
; High word of EDX = 0 at this point
; Carry flag set iff there's a trailing word to do at this point
;
checksum_last_word label proc ; "proc" so not scoped to function
pop ecx ; get back word count
test ecx,1 ; is there a trailing word?
jz short checksum_done ; no trailing word
add ax,[esi] ; add in the trailing word
adc eax,0 ;
checksum_done label proc ; "proc" so not scoped to function
mov ecx,eax ; fold the checksum to 16 bits
ror ecx,16
add eax,ecx
mov ebx,[esp + buf]
shr eax,16
test ebx,1 ; check if buffer word aligned
jz short checksum_combine ; if zf set, buffer word aligned
ror ax,8 ; byte aligned--swap bytes back
checksum_combine label proc ; "proc" so not scoped to function
add ax,word ptr [esp + cksum] ; combine checksums
pop esi ; restore nonvolatile register
adc eax,0 ;
pop ebx ; restore nonvolatile register
stdRET tcpxsum
REFLAB macro pre,suf
dd pre&suf
endm
align 4
loop_entry label dword
dd 0
TEMP=LOOP_UNROLLING*4
REPT LOOP_UNROLLING-1
TEMP=TEMP-4
reflab loop_entry_,%TEMP
ENDM
stdENDP tcpxsum
ifndef NO_XMMI
LOOP_UNROLLING_BITS_XMMI equ 4
LOOP_UNROLLING_XMMI equ (1 SHL LOOP_UNROLLING_BITS_XMMI)
;VRSTEST EQU 0
ifdef VRSTEST
;
; Test tcpxsum_xmmi for correctness.
tcksum equ 8[ebp] ; stack offset to initial checksum
tbuf equ 12[ebp] ; stack offset to source address
tlen equ 16[ebp] ; stack offset to length in words
align
cPublicProc tcpxsum_xmmi,3
;int 3
push ebp
mov ebp, esp
push ebx
push esi
mov ebx, offset tcpxsum
mov esi, offset tcpxsum_xmmi1
; Get a "random" number
.586p
rdtsc
.386p
and eax, 10H
jz old_then_new
; Swap which routine is called first
push ebx
mov ebx, esi
pop esi
old_then_new:
; Call the first routine
push tlen
push tbuf
push tcksum
call ebx
; Save the answer
push eax
; Call the second routine
push tlen
push tbuf
push tcksum
call esi
; Check the answer
cmp eax, [esp]
jnz different_xsum
; Same answer, we are done
pop eax
pop esi
pop ebx
pop ebp
stdRET tcpxsum_xmmi
align
; Different answers, need to debug the problem
different_xsum:
; Get both checksums onto the stack
push eax
; ... and bugcheck
;EXTRNP _KeBugCheck,1,IMPORT
;stdCall _KeBugCheck, <0>
again:
int 3
jmp again
stdENDP tcpxsum_xmmi
endif
;++
;
; ULONG
; tcpxsum_xmmi(
; IN ULONG cksum,
; IN PUCHAR buf,
; IN ULONG len
; )
;
; Routine Description:
;
; This function computes the checksum of the specified buffer.
; It uses Processor's prefetch instruction.
;
; Arguments:
;
; cksum - Suppiles the initial checksum value, in 16-bit form,
; with the high word set to 0.
;
; buf - Supplies a pointer to the buffer to the checksum buffer.
;
; len - Supplies the length of the buffer in bytes.
;
; Return Value:
;
; The computed checksum in 32-bit two-partial-accumulators form, added to
; the initial checksum, is returned as the function value.
;
;--
cksum equ 12 ; stack offset to initial checksum
buf equ 16 ; stack offset to source address
len equ 20 ; stack offset to length in words
to_checksum_last_word_xmmi:
jmp checksum_last_word_xmmi
to_checksum_done_xmmi:
jmp checksum_done_xmmi
to_checksum_dword_loop_done_xmmi:
jmp checksum_dword_loop_done_xmmi
ifdef VRSTEST
cPublicProc tcpxsum_xmmi1,3
else
cPublicProc tcpxsum_xmmi,3
endif
push ebx ; save nonvolatile register
push esi ; save nonvolatile register
mov ecx,[esp + len] ; get length in bytes
sub eax,eax ; clear computed checksum
test ecx,ecx ; any bytes to checksum at all?
jz short to_checksum_done_xmmi ; no bytes to checksum
;
; if the checksum buffer is not word aligned, then add the first byte of
; the buffer to the input checksum.
;
mov esi,[esp + buf] ; get source address
sub edx,edx ; set up to load word into EDX below
test esi,1 ; check if buffer word aligned
jz short checksum_word_aligned ; if zf, buffer word aligned
mov ah,[esi] ; get first byte (we know we'll have
; to swap at the end)
inc esi ; increment buffer address
dec ecx ; decrement number of bytes
jz short to_checksum_done_xmmi ; if zf set, no more bytes
;
; If the buffer is not an even number of of bytes, then initialize
; the computed checksum with the last byte of the buffer.
;
checksum_word_aligned: ;
shr ecx,1 ; convert to word count
jnc short checksum_start ; if nc, even number of bytes
mov al,[esi+ecx*2] ; initialize the computed checksum
jz short to_checksum_done_xmmi ; if zf set, no more bytes
;
; Compute checksum in large blocks of dwords, with one partial word up front if
; necessary to get dword alignment, and another partial word at the end if
; needed.
;
;
; Compute checksum on the leading word, if that's necessary to get dword
; alignment.
;
checksum_start: ;
test esi,02h ; check if source dword aligned
jz short checksum_dword_aligned ; source is already dword aligned
mov dx,[esi] ; get first word to checksum
add esi,2 ; update source address
add eax,edx ; update partial checksum
; (no carry is possible, because EAX
; and EDX are both 16-bit values)
dec ecx ; count off this word (zero case gets
; picked up below)
;
; Checksum as many words as possible by processing a dword at a time.
;
checksum_dword_aligned:
push ecx ; so we can tell if there's a trailing
; word later
shr ecx,1 ; # of dwords to checksum
jz short to_checksum_last_word_xmmi ; no dwords to checksum
mov edx,[esi] ; preload the first dword
add esi,4 ; point to the next dword
dec ecx ; count off the dword we just loaded
jz short to_checksum_dword_loop_done_xmmi
; skip the loop if that was the only
; dword
mov ebx,ecx ; EBX = # of dwords left to checksum
add ecx,LOOP_UNROLLING_XMMI-1 ; round up loop count
shr ecx,LOOP_UNROLLING_BITS_XMMI ; convert from word count to unrolled
; loop count
and ebx,LOOP_UNROLLING_XMMI-1 ; # of partial dwords to do in first
; loop
jz short checksum_dword_loop ; special-case when no partial loop,
; because fixup below doesn't work
; in that case (carry flag is
; cleared at this point, as required
; at loop entry)
lea esi,[esi+ebx*4-(LOOP_UNROLLING_XMMI*4)]
; adjust buffer pointer back to
; compensate for hardwired displacement
; at loop entry point
; ***doesn't change carry flag***
jmp xmmi_loop_entry[ebx*4] ; enter the loop to do the first,
; partial iteration, after which we can
; just do 64-word blocks
; ***doesn't change carry flag***
checksum_dword_loop:
; prefetch the 32-byte cache line from [esi+0]
db 0fH
db 18H
db 46H
db 00H
; prefetch the 32-byte cache line from [esi+20h]
db 0fH
db 18H
db 46H
db 20H
; prefetch the 32-byte cache line from [esi+40h]
db 0fH
db 18H
db 46H
db 40H
; prefetch the 32-byte cache line from [esi+60h]
db 0fH
db 18H
db 46H
db 60H
DEFLAB macro pre,suf
pre&suf:
endm
TEMP=0
REPT LOOP_UNROLLING_XMMI
deflab xmmi_loop_entry_,%TEMP
adc eax,edx
mov edx,[esi + TEMP]
TEMP=TEMP+4
ENDM
checksum_dword_loop_end:
lea esi,[esi + LOOP_UNROLLING_XMMI * 4] ; update source address
; ***doesn't change carry flag***
dec ecx ; count off unrolled loop iteration
; ***doesn't change carry flag***
jnz checksum_dword_loop ; do more blocks
checksum_dword_loop_done_xmmi label proc
adc eax,edx ; finish dword checksum
mov edx,0 ; prepare to load trailing word
adc eax,edx
;
; Compute checksum on the trailing word, if there is one.
; High word of EDX = 0 at this point
; Carry flag set iff there's a trailing word to do at this point
;
checksum_last_word_xmmi label proc ; "proc" so not scoped to function
pop ecx ; get back word count
test ecx,1 ; is there a trailing word?
jz short checksum_done_xmmi; no trailing word
add ax,[esi] ; add in the trailing word
adc eax,0 ;
checksum_done_xmmi label proc ; "proc" so not scoped to function
mov ecx,eax ; fold the checksum to 16 bits
ror ecx,16
add eax,ecx
mov ebx,[esp + buf]
shr eax,16
test ebx,1 ; check if buffer word aligned
jz short checksum_combine_xmmi ; if zf set, buffer word aligned
ror ax,8 ; byte aligned--swap bytes back
checksum_combine_xmmi label proc ; "proc" so not scoped to function
add ax,word ptr [esp + cksum] ; combine checksums
pop esi ; restore nonvolatile register
adc eax,0 ;
pop ebx ; restore nonvolatile register
stdRET tcpxsum
REFLAB macro pre,suf
dd pre&suf
endm
align 4
xmmi_loop_entry label dword
dd 0
TEMP=LOOP_UNROLLING_XMMI*4
REPT LOOP_UNROLLING_XMMI-1
TEMP=TEMP-4
reflab xmmi_loop_entry_,%TEMP
ENDM
ifdef VRSTEST
stdENDP tcpxsum_xmmi1
else
stdENDP tcpxsum_xmmi
endif
endif ; NO_XMMI
ifndef NO_OLD_FLUSHSLIST
;++
;
; PSINGLE_LIST_ENTRY
; FASTCALL
; InterlockedFlushSList (
; IN PSINGLE_LIST_ENTRY ListHead
; )
;
; Routine Description:
;
; This function removes the entire list from a sequenced singly
; linked list so that access to the list is synchronized in an MP system.
; If there are no entries in the list, then a value of NULL is returned.
; Otherwise, the address of the entry at the top of the list is removed
; and returned as the function value and the list header is set to point
; to NULL.
;
; N.B. The cmpxchg8b instruction is only supported on some processors.
; If the host processor does not support this instruction, then
; then following code is patched to contain a jump to the normal
; pop entry code which has a compatible calling sequence and data
; structure.
;
; Arguments:
;
; (ecx) = ListHead - Supplies a pointer to the sequenced listhead from
; which the list is to be flushed.
;
; Return Value:
;
; The address of the entire current list, or NULL if the list is
; empty.
;
;--
cPublicProc InterlockedFlushSList, 1
;
; Save nonvolatile registers and read the listhead sequence number followed
; by the listhead next link.
;
; N.B. These two dwords MUST be read exactly in this order.
;
push ecx
push ebx ; save nonvolatile registers
push ebp ;
mov ecx, [esp+16]
mov ebp, ecx ; save listhead address
mov edx, [ebp] + 4 ; get current sequence number
mov eax, [ebp] + 0 ; get current next link
;
; N.B. The following code is the retry code should the compare
; part of the compare exchange operation fail
;
; If the list is empty, then there is nothing that can be removed.
;
Efls10: or eax, eax ; check if list is empty
jz short Efls20 ; if z set, list is empty
mov ecx, 0 ; clear sequence number and depth
mov ebx, 0 ; clear successor entry pointer
.586
ifndef NT_UP
lock cmpxchg8b qword ptr [ebp] ; compare and exchange
else
cmpxchg8b qword ptr [ebp] ; compare and exchange
endif
.386
jnz short Efls10 ; if z clear, exchange failed
;
; Restore nonvolatile registers and return result.
;
Efls20: pop ebp ; restore nonvolatile registers
pop ebx ;
pop ecx
stdRET InterlockedFlushSList
stdENDP InterlockedFlushSList
endif ; NO_OLD_FLUSHSLIST
end