; Copyright 2016 Jens Nurmann and Alexander Kruppa ; This file is part of the MPIR Library. ; The MPIR Library is free software; you can redistribute it and/or modify ; it under the terms of the GNU Lesser General Public License as published ; by the Free Software Foundation; either version 2.1 of the License, or (at ; your option) any later version. ; The MPIR Library is distributed in the hope that it will be useful, but ; WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY ; or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public ; License for more details. ; You should have received a copy of the GNU Lesser General Public License ; along with the MPIR Library; see the file COPYING.LIB. If not, write ; to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ; Boston, MA 02110-1301, USA. ; mp_limb_t mpn_lshift(mp_ptr Op2, mp_srcptr Op1, mp_size_t Size1, unsigned int Shift) ; Linux RAX RDI RSI RDX RCX ; Win7 RAX RCX RDX R8 R9 ; ; Description: ; The function shifts Op1 left by n bit, stores the result in Op2 (non- ; destructive shl) and hands back the shifted-out most significant bits of Op1. ; The function operates decreasing in memory supporting in-place operation. ; ; Result: ; - Op2[ Size1-1..0 ] := ( Op1[ Size1-1..0 ]:ShlIn ) << 1 ; - Op1[ 0 ] >> 63 ; ; Caveats: ; - caller must ensure that Shift is in [ 1..63 ]! ; - currently Linux64 support only! ; - the AVX version uses mnemonics only available on Haswell, Broadwell and ; Skylake cores ; - the behaviour of cache prefetching in combination with AVX shifting seems ; somewhat erratic ; - slight (a few clock cycles) degradation for 1/2 LD1$ sizes ; - slight (a few percent) improvement for full LD1$ sizes ; - substantial (>10%) improvement for 1/2 LD2$ sizes ; - slight (a few percent) improvement for full LD2$ sizes ; - slight (a few percent) degradation for 1/2 LD3$ sizes ; - substantial (around 10%) degradation for full LD3$ sizes ; ; Comments: ; - implemented, tested and benched on 31.03.2016 by jn ; - includes prefetching ; ============================================================================ %define USE_WIN64 %include 'yasm_mac.inc' BITS 64 %ifdef USE_WIN64 %define Op2 R11 %define Op1 RDX %define Size1 R8 %define Shift RCX %define Limb1 R9 %define Limb2 R10 %ifdef USE_PREFETCH %define Offs -512 ; No caller-saves regs left, use immediate %endif %define reg_save_list XMM, 6, 7 %else %define Op2 RDI %define Op1 RSI %define Size1 RDX %define Shift RCX %define Limb1 R8 %define Limb2 R9 %ifdef USE_PREFETCH %define OFFS_REG 1 %define Offs R10 %endif %endif %define ShlDL0 XMM2 ; Attn: this must match ShlQL0 definition %define ShrDL0 XMM3 ; Attn: this must match ShrQL0 definition %define ShlDLCnt XMM6 ; Attn: this must match ShlQlCnt definition %define ShrDLCnt XMM7 ; Attn: this must match ShrQlCnt definition %define QLimb0 YMM0 %define QLimb1 YMM1 %define ShlQL0 YMM2 %define ShrQL0 YMM3 %define ShlQL1 YMM4 %define ShrQL1 YMM5 %define ShlQLCnt YMM6 %define ShrQLCnt YMM7 align 32 FRAME_PROC mpn_lshift, 0, reg_save_list %ifdef USE_WIN64 mov r11, rcx mov rcx, r9 %endif xor EAX, EAX sub Size1, 1 jc .Exit ; Size1=0 => lea Op1, [Op1+8*Size1] lea Op2, [Op2+8*Size1] mov Limb1, [Op1] shld RAX, Limb1, CL or Size1, Size1 je .lShlEquPost ; Size1=1 => %ifdef USE_PREFETCH %ifdef OFFS_REG mov Offs, -512 %endif %endif cmp Size1, 8 jc .lShlEquFour ; AVX inefficient => ; first align Op2 to 32 bytes test Op2, 8 jne .lShlEquA16 mov Limb2, [Op1-8] shld Limb1, Limb2, CL mov [Op2], Limb1 mov Limb1, Limb2 sub Op1, 8 sub Op2, 8 sub Size1, 1 .lShlEquA16: test Op2, 16 jne .lShlEquAVX mov Limb2, [Op1-8] shld Limb1, Limb2, CL mov [Op2], Limb1 mov Limb1, [Op1-16] shld Limb2, Limb1, CL mov [Op2-8], Limb2 sub Op1, 16 sub Op2, 16 sub Size1, 2 .lShlEquAVX: ; initialize AVX shift counter vmovq ShlDLCnt, RCX neg RCX and RCX, 63 ; must do, as AVX shifts set result=0 if Shift>63! vmovq ShrDLCnt, RCX neg RCX and RCX, 63 ; must do, as AVX shifts set result=0 if Shift>63! vpbroadcastq ShlQLCnt, ShlDLCnt vpbroadcastq ShrQLCnt, ShrDLCnt ; pre-fetch first quad-limb vmovdqu QLimb0, [Op1-24] vpsrlvq ShrQL0, QLimb0, ShrQLCnt vpermq ShrQL0, ShrQL0, 10010011b sub Op1, 32 sub Size1, 4 jmp .lShlEquAVXCheck ; main loop (prefetching enabled; unloaded cache) ; - 0.60 cycles per limb in LD1$ ; - 0.60-0.70 cycles per limb in LD2$ ; - 0.70-0.90 cycles per limb in LD3$ align 16 .lShlEquAVXLoop: %ifdef USE_PREFETCH prefetchnta [Op1+Offs] %endif vmovdqu QLimb1, [Op1-24] vpsllvq ShlQL0, QLimb0, ShlQLCnt vmovdqu QLimb0, [Op1-56] vpsrlvq ShrQL1, QLimb1, ShrQLCnt vpermq ShrQL1, ShrQL1, 10010011b vpblendd ShrQL0, ShrQL0, ShrQL1, 00000011b vpor ShlQL0, ShlQL0, ShrQL0 vpsllvq ShlQL1, QLimb1, ShlQLCnt vpsrlvq ShrQL0, QLimb0, ShrQLCnt vpermq ShrQL0, ShrQL0, 10010011b vpblendd ShrQL1, ShrQL1, ShrQL0, 00000011b vmovdqa [Op2-24], ShlQL0 vpor ShlQL1, ShlQL1, ShrQL1 vmovdqa [Op2-56], ShlQL1 sub Op1, 64 sub Op2, 64 .lShlEquAVXCheck: sub Size1, 8 jnc .lShlEquAVXLoop mov Limb1, [Op1] xor Limb2, Limb2 shld Limb2, Limb1, CL %if 1 vmovq ShlDL0, Limb2 vpblendd ShrQL0, ShrQL0, ShlQL0, 3 %else ; I am mixing in a single SSE4.1 instruction into otherwise pure AVX2 ; this is generating stalls on Haswell & Broadwell architecture (Agner Fog) ; but it is only executed once and there is no AVX2 based alternative pinsrq ShrDL0, Limb2, 0 ; SSE4.1 %endif vpsllvq ShlQL0, QLimb0, ShlQLCnt vpor ShlQL0, ShlQL0, ShrQL0 vmovdqa [Op2-24], ShlQL0 sub Op2, 32 add Size1, 8 ; shift remaining max. 7 limbs with SHLD mnemonic .lShlEquFour: sub Op1, 8 test Size1, 4 je .lShlEquTwo mov Limb2, [Op1] shld Limb1, Limb2, CL mov [Op2], Limb1 mov Limb1, [Op1-8] shld Limb2, Limb1, CL mov [Op2-8], Limb2 mov Limb2, [Op1-16] shld Limb1, Limb2, CL mov [Op2-16], Limb1 mov Limb1, [Op1-24] shld Limb2, Limb1, CL mov [Op2-24], Limb2 sub Op1, 32 sub Op2, 32 .lShlEquTwo: test Size1, 2 je .lShlEquOne mov Limb2, [Op1] shld Limb1, Limb2, CL mov [Op2], Limb1 mov Limb1, [Op1-8] shld Limb2, Limb1, CL mov [Op2-8], Limb2 sub Op1, 16 sub Op2, 16 .lShlEquOne: test Size1, 1 je .lShlEquPost mov Limb2, [Op1] shld Limb1, Limb2, CL mov [Op2], Limb1 mov Limb1, Limb2 sub Op2, 8 .lShlEquPost: shl Limb1, CL mov [Op2], Limb1 .Exit: vzeroupper END_PROC reg_save_list .end: