dnl AMD K7 mpn_rshift -- mpn right shift. dnl Copyright 1999, 2000, 2001, 2002 Free Software Foundation, Inc. dnl dnl This file is part of the GNU MP Library. dnl dnl The GNU MP Library is free software; you can redistribute it and/or dnl modify it under the terms of the GNU Lesser General Public License as dnl published by the Free Software Foundation; either version 2.1 of the dnl License, or (at your option) any later version. dnl dnl The GNU MP Library is distributed in the hope that it will be useful, dnl but WITHOUT ANY WARRANTY; without even the implied warranty of dnl MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU dnl Lesser General Public License for more details. dnl dnl You should have received a copy of the GNU Lesser General Public dnl License along with the GNU MP Library; see the file COPYING.LIB. If dnl not, write to the Free Software Foundation, Inc., 51 Franklin Street, dnl Fifth Floor, Boston, MA 02110-1301, USA. include(`../config.m4') C K7: 1.21 cycles/limb (at 16 limbs/loop). dnl K7: UNROLL_COUNT cycles/limb dnl 4 1.51 dnl 8 1.26 dnl 16 1.21 dnl 32 1.2 dnl Maximum possible with the current code is 64. deflit(UNROLL_COUNT, 16) C mp_limb_t mpn_rshift (mp_ptr dst, mp_srcptr src, mp_size_t size, C unsigned shift); C C Shift src,size right by shift many bits and store the result in dst,size. C Zeros are shifted in at the left. The bits shifted out at the right are C the return value. C C This code uses 64-bit MMX operations, which makes it possible to handle C two limbs at a time, for a theoretical 1.0 cycles/limb. Plain integer C code, on the other hand, suffers from shrd being a vector path decode and C running at 3 cycles back-to-back. C C Full speed depends on source and destination being aligned, and some hairy C setups and finish-ups are done to arrange this for the loop. ifdef(`PIC',` deflit(UNROLL_THRESHOLD, 10) ',` deflit(UNROLL_THRESHOLD, 10) ') defframe(PARAM_SHIFT,16) defframe(PARAM_SIZE, 12) defframe(PARAM_SRC, 8) defframe(PARAM_DST, 4) defframe(SAVE_EDI, -4) defframe(SAVE_ESI, -8) defframe(SAVE_EBX, -12) deflit(SAVE_SIZE, 12) TEXT ALIGN(32) PROLOGUE(mpn_rshift) deflit(`FRAME',0) movl PARAM_SIZE, %eax movl PARAM_SRC, %edx subl $SAVE_SIZE, %esp deflit(`FRAME',SAVE_SIZE) movl PARAM_SHIFT, %ecx movl %edi, SAVE_EDI movl PARAM_DST, %edi decl %eax jnz L(more_than_one_limb) movl (%edx), %edx C src limb shrdl( %cl, %edx, %eax) C eax was decremented to zero shrl %cl, %edx movl %edx, (%edi) C dst limb movl SAVE_EDI, %edi addl $SAVE_SIZE, %esp ret C ----------------------------------------------------------------------------- L(more_than_one_limb): C eax size-1 C ebx C ecx shift C edx src C esi C edi dst C ebp movd PARAM_SHIFT, %mm6 C rshift movd (%edx), %mm5 C src low limb cmp $UNROLL_THRESHOLD-1, %eax jae L(unroll) leal (%edx,%eax,4), %edx C &src[size-1] leal -4(%edi,%eax,4), %edi C &dst[size-2] movd (%edx), %mm4 C src high limb negl %eax L(simple_top): C eax loop counter, limbs, negative C ebx C ecx shift C edx carry C edx &src[size-1] C edi &dst[size-2] C ebp C C mm0 scratch C mm4 src high limb C mm5 src low limb C mm6 shift movq (%edx,%eax,4), %mm0 incl %eax psrlq %mm6, %mm0 movd %mm0, (%edi,%eax,4) jnz L(simple_top) psllq $32, %mm5 psrlq %mm6, %mm4 psrlq %mm6, %mm5 movd %mm4, 4(%edi) C dst high limb movd %mm5, %eax C return value movl SAVE_EDI, %edi addl $SAVE_SIZE, %esp emms ret C ----------------------------------------------------------------------------- ALIGN(16) L(unroll): C eax size-1 C ebx C ecx shift C edx src C esi C edi dst C ebp C C mm5 src low limb C mm6 rshift testb $4, %dl movl %esi, SAVE_ESI movl %ebx, SAVE_EBX psllq $32, %mm5 jz L(start_src_aligned) C src isn't aligned, process low limb separately (marked xxx) and C step src and dst by one limb, making src aligned. C C source edx C --+-------+-------+-------+ C | xxx | C --+-------+-------+-------+ C 4mod8 0mod8 4mod8 C C dest edi C --+-------+-------+ C | | xxx | C --+-------+-------+ movq (%edx), %mm0 C src low two limbs addl $4, %edx movl %eax, PARAM_SIZE C size-1 addl $4, %edi decl %eax C size-2 is new size-1 psrlq %mm6, %mm0 movl %edi, PARAM_DST C new dst movd %mm0, -4(%edi) L(start_src_aligned): movq (%edx), %mm1 C src low two limbs decl %eax C size-2, two last limbs handled at end testl $4, %edi psrlq %mm6, %mm5 jz L(start_dst_aligned) C dst isn't aligned, add 4 to make it so, and pretend the shift is C 32 bits extra. Low limb of dst (marked xxx) handled here separately. C C source edx C --+-------+-------+ C | mm1 | C --+-------+-------+ C 4mod8 0mod8 C C dest edi C --+-------+-------+-------+ C | xxx | C --+-------+-------+-------+ C 4mod8 0mod8 4mod8 movq %mm1, %mm0 psrlq %mm6, %mm1 addl $32, %ecx C shift+32 movd %mm1, (%edi) movq %mm0, %mm1 addl $4, %edi C new dst movd %ecx, %mm6 L(start_dst_aligned): movq %mm1, %mm2 C copy of src low two limbs negl %ecx andl $-2, %eax C round size down to even movl %eax, %ebx negl %eax addl $64, %ecx andl $UNROLL_MASK, %eax decl %ebx shll %eax movd %ecx, %mm7 C lshift = 64-rshift ifdef(`PIC',` call L(pic_calc) L(here): ',` leal L(entry) (%eax,%eax,4), %esi negl %eax ') shrl $UNROLL_LOG2, %ebx C loop counter leal ifelse(UNROLL_BYTES,256,128+) 8(%edx,%eax,2), %edx leal ifelse(UNROLL_BYTES,256,128) (%edi,%eax,2), %edi movl PARAM_SIZE, %eax C for use at end jmp *%esi ifdef(`PIC',` L(pic_calc): C See mpn/x86/README about old gas bugs leal (%eax,%eax,4), %esi addl $L(entry)-L(here), %esi addl (%esp), %esi negl %eax ret_internal ') C ----------------------------------------------------------------------------- ALIGN(64) L(top): C eax size, for use at end C ebx loop counter C ecx lshift C edx src C esi was computed jump C edi dst C ebp C C mm0 scratch C mm1 \ carry (alternating) C mm2 / C mm6 rshift C mm7 lshift C C 10 code bytes/limb C C The two chunks differ in whether mm1 or mm2 hold the carry. C The computed jump puts the initial carry in both mm1 and mm2. L(entry): deflit(CHUNK_COUNT, 4) forloop(i, 0, UNROLL_COUNT/CHUNK_COUNT-1, ` deflit(`disp0', eval(i*CHUNK_COUNT*4 ifelse(UNROLL_BYTES,256,-128))) deflit(`disp1', eval(disp0 + 8)) Zdisp( movq, disp0,(%edx), %mm0) psrlq %mm6, %mm2 movq %mm0, %mm1 psllq %mm7, %mm0 por %mm2, %mm0 Zdisp( movq, %mm0, disp0,(%edi)) Zdisp( movq, disp1,(%edx), %mm0) psrlq %mm6, %mm1 movq %mm0, %mm2 psllq %mm7, %mm0 por %mm1, %mm0 Zdisp( movq, %mm0, disp1,(%edi)) ') addl $UNROLL_BYTES, %edx addl $UNROLL_BYTES, %edi decl %ebx jns L(top) deflit(`disp0', ifelse(UNROLL_BYTES,256,-128)) deflit(`disp1', eval(disp0-0 + 8)) testb $1, %al psrlq %mm6, %mm2 C wanted rshifted in all cases below movl SAVE_ESI, %esi movd %mm5, %eax C return value movl SAVE_EBX, %ebx jz L(end_even) C Size odd, destination was aligned. C C source C edx C +-------+---------------+-- C | | mm2 | C +-------+---------------+-- C C dest edi C +-------+---------------+---------------+-- C | | | written | C +-------+---------------+---------------+-- C C mm6 = shift C mm7 = ecx = 64-shift C Size odd, destination was unaligned. C C source C edx C +-------+---------------+-- C | | mm2 | C +-------+---------------+-- C C dest edi C +---------------+---------------+-- C | | written | C +---------------+---------------+-- C C mm6 = shift+32 C mm7 = ecx = 64-(shift+32) C In both cases there's one extra limb of src to fetch and combine C with mm2 to make a qword to store, and in the aligned case there's C a further extra limb of dst to be formed. movd disp0(%edx), %mm0 movq %mm0, %mm1 psllq %mm7, %mm0 testb $32, %cl por %mm2, %mm0 psrlq %mm6, %mm1 movq %mm0, disp0(%edi) jz L(finish_odd_unaligned) movd %mm1, disp1(%edi) L(finish_odd_unaligned): movl SAVE_EDI, %edi addl $SAVE_SIZE, %esp emms ret L(end_even): C Size even, destination was aligned. C C source C +---------------+-- C | mm2 | C +---------------+-- C C dest edi C +---------------+---------------+-- C | | mm3 | C +---------------+---------------+-- C C mm6 = shift C mm7 = ecx = 64-shift C Size even, destination was unaligned. C C source C +---------------+-- C | mm2 | C +---------------+-- C C dest edi C +-------+---------------+-- C | | mm3 | C +-------+---------------+-- C C mm6 = shift+32 C mm7 = 64-(shift+32) C The movd for the unaligned case is the same data as the movq for C the aligned case, it's just a choice between whether one or two C limbs should be written. testb $32, %cl movd %mm2, disp0(%edi) jz L(end_even_unaligned) movq %mm2, disp0(%edi) L(end_even_unaligned): movl SAVE_EDI, %edi addl $SAVE_SIZE, %esp emms ret EPILOGUE()