dnl AMD K7 mpn_mul_1 -- mpn by limb multiply. dnl Copyright 1999, 2000, 2001, 2002, 2005 Free Software Foundation, Inc. dnl dnl This file is part of the GNU MP Library. dnl dnl The GNU MP Library is free software; you can redistribute it and/or dnl modify it under the terms of the GNU Lesser General Public License as dnl published by the Free Software Foundation; either version 2.1 of the dnl License, or (at your option) any later version. dnl dnl The GNU MP Library is distributed in the hope that it will be useful, dnl but WITHOUT ANY WARRANTY; without even the implied warranty of dnl MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU dnl Lesser General Public License for more details. dnl dnl You should have received a copy of the GNU Lesser General Public dnl License along with the GNU MP Library; see the file COPYING.LIB. If dnl not, write to the Free Software Foundation, Inc., 51 Franklin Street, dnl Fifth Floor, Boston, MA 02110-1301, USA. include(`../config.m4') C cycles/limb C P5: C P6 model 0-8,10-12) 4.95 C P6 model 9 (Banias) C P6 model 13 (Dothan) 4.79 C P4 model 0 (Willamette) C P4 model 1 (?) C P4 model 2 (Northwood) C P4 model 3 (Prescott) C P4 model 4 (Nocona) C K6: C K7: 3.4 dnl K7: UNROLL_COUNT cycles/limb dnl 8 3.9 dnl 16 3.4 dnl 32 3.4 dnl 64 3.35 dnl Maximum possible with the current code is 64. deflit(UNROLL_COUNT, 16) C mp_limb_t mpn_mul_1 (mp_ptr dst, mp_srcptr src, mp_size_t size, C mp_limb_t multiplier); C mp_limb_t mpn_mul_1c (mp_ptr dst, mp_srcptr src, mp_size_t size, C mp_limb_t multiplier, mp_limb_t carry); C C Multiply src,size by mult and store the result in dst,size. C Return the carry limb from the top of the result. C C mpn_mul_1c() accepts an initial carry for the calculation, it's added into C the low limb of the destination. C C Variations on the unrolled loop have been tried, with the current C registers or with the counter on the stack to free up ecx. The current C code is the fastest found. C C An interesting effect is that removing the stores "movl %ebx, disp0(%edi)" C from the unrolled loop actually slows it down to 5.0 cycles/limb. Code C with this change can be tested on sizes of the form UNROLL_COUNT*n+1 C without having to change the computed jump. There's obviously something C fishy going on, perhaps with what execution units the mul needs. defframe(PARAM_CARRY, 20) defframe(PARAM_MULTIPLIER,16) defframe(PARAM_SIZE, 12) defframe(PARAM_SRC, 8) defframe(PARAM_DST, 4) defframe(SAVE_EBP, -4) defframe(SAVE_EDI, -8) defframe(SAVE_ESI, -12) defframe(SAVE_EBX, -16) deflit(STACK_SPACE, 16) dnl Must have UNROLL_THRESHOLD >= 2, since the unrolled loop can't handle 1. ifdef(`PIC',` deflit(UNROLL_THRESHOLD, 7) ',` deflit(UNROLL_THRESHOLD, 5) ') TEXT ALIGN(32) PROLOGUE(mpn_mul_1c) deflit(`FRAME',0) movl PARAM_CARRY, %edx jmp L(start_nc) EPILOGUE() PROLOGUE(mpn_mul_1) deflit(`FRAME',0) xorl %edx, %edx C initial carry L(start_nc): movl PARAM_SIZE, %ecx subl $STACK_SPACE, %esp deflit(`FRAME', STACK_SPACE) movl %edi, SAVE_EDI movl %ebx, SAVE_EBX movl %edx, %ebx movl %esi, SAVE_ESI movl PARAM_SRC, %esi cmpl $UNROLL_THRESHOLD, %ecx movl PARAM_DST, %edi movl %ebp, SAVE_EBP jae L(unroll) leal (%esi,%ecx,4), %esi leal (%edi,%ecx,4), %edi negl %ecx movl PARAM_MULTIPLIER, %ebp L(simple): C eax scratch C ebx carry C ecx counter (negative) C edx scratch C esi src C edi dst C ebp multiplier movl (%esi,%ecx,4), %eax mull %ebp addl %ebx, %eax movl %eax, (%edi,%ecx,4) movl $0, %ebx adcl %edx, %ebx incl %ecx jnz L(simple) movl %ebx, %eax movl SAVE_EBX, %ebx movl SAVE_ESI, %esi movl SAVE_EDI, %edi movl SAVE_EBP, %ebp addl $STACK_SPACE, %esp ret C ----------------------------------------------------------------------------- C The mov to load the next source limb is done well ahead of the mul, this C is necessary for full speed. It leads to one limb handled separately C after the loop. C C When unrolling to 32 or more, an offset of +4 is used on the src pointer, C to avoid having an 0x80 displacement in the code for the last limb in the C unrolled loop. This is for a fair comparison between 16 and 32 unrolling. ifelse(eval(UNROLL_COUNT >= 32),1,` deflit(SRC_OFFSET,4) ',` deflit(SRC_OFFSET,) ') C this is offset 0x62, so close enough to aligned L(unroll): C eax C ebx initial carry C ecx size C edx C esi src C edi dst C ebp deflit(`FRAME', STACK_SPACE) leal -1(%ecx), %edx C one limb handled at end leal -2(%ecx), %ecx C and ecx is one less than edx movl %ebp, SAVE_EBP negl %edx shrl $UNROLL_LOG2, %ecx C unrolled loop counter movl (%esi), %eax C src low limb andl $UNROLL_MASK, %edx movl PARAM_DST, %edi movl %edx, %ebp shll $4, %edx C 17 code bytes per limb ifdef(`PIC',` call L(add_eip_to_edx) L(here): ',` leal L(entry) (%edx,%ebp), %edx ') negl %ebp leal ifelse(UNROLL_BYTES,256,128+) SRC_OFFSET(%esi,%ebp,4), %esi leal ifelse(UNROLL_BYTES,256,128) (%edi,%ebp,4), %edi movl PARAM_MULTIPLIER, %ebp jmp *%edx ifdef(`PIC',` L(add_eip_to_edx): C See mpn/x86/README about old gas bugs leal (%edx,%ebp), %edx addl $L(entry)-L(here), %edx addl (%esp), %edx ret_internal ') C ---------------------------------------------------------------------------- ALIGN(32) L(top): C eax next src limb C ebx carry C ecx counter C edx scratch C esi src+4 C edi dst C ebp multiplier C C 17 code bytes per limb processed L(entry): forloop(i, 0, UNROLL_COUNT-1, ` deflit(`disp_dst', eval(i*4 ifelse(UNROLL_BYTES,256,-128))) deflit(`disp_src', eval(disp_dst + 4-(SRC_OFFSET-0))) mull %ebp addl %eax, %ebx Zdisp( movl, disp_src,(%esi), %eax) Zdisp( movl, %ebx, disp_dst,(%edi)) movl $0, %ebx adcl %edx, %ebx ') decl %ecx leal UNROLL_BYTES(%esi), %esi leal UNROLL_BYTES(%edi), %edi jns L(top) deflit(`disp0', ifelse(UNROLL_BYTES,256,-128)) mull %ebp addl %eax, %ebx movl $0, %eax movl SAVE_ESI, %esi movl %ebx, disp0(%edi) movl SAVE_EBX, %ebx movl SAVE_EDI, %edi adcl %edx, %eax movl SAVE_EBP, %ebp addl $STACK_SPACE, %esp ret EPILOGUE()