161 lines
3.7 KiB
NASM
161 lines
3.7 KiB
NASM
|
|
; AMD64 mpn_add_n
|
|
; Copyright 2016 Alexander Kruppa
|
|
; This file is part of the MPIR Library.
|
|
; The MPIR Library is free software; you can redistribute it and/or modify
|
|
; it under the terms of the GNU Lesser General Public License as published
|
|
; by the Free Software Foundation; either version 2.1 of the License, or (at
|
|
; your option) any later version.
|
|
; The MPIR Library is distributed in the hope that it will be useful, but
|
|
; WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
|
|
; or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
|
|
; License for more details.
|
|
; You should have received a copy of the GNU Lesser General Public License
|
|
; along with the MPIR Library; see the file COPYING.LIB. If not, write
|
|
; to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
|
|
; Boston, MA 02110-1301, USA.
|
|
|
|
; (rdi,rcx) = (rsi,rcx)+(rdx,rcx)
|
|
; rax = carry
|
|
|
|
%define USE_WIN64
|
|
|
|
%include 'yasm_mac.inc'
|
|
|
|
%ifdef USE_WIN64
|
|
%define SumP rcx
|
|
%define Inp1P rdx
|
|
%define Inp2P r8
|
|
%define Size r9
|
|
%define SizeRest r11
|
|
%define LIMB1 rax
|
|
%define LIMB2 r10
|
|
%else
|
|
%define SumP rdi
|
|
%define Inp1P rsi
|
|
%define Inp2P rdx
|
|
%define Size rcx
|
|
%define SizeRest r11
|
|
%define LIMB1 rax
|
|
%define LIMB2 r8
|
|
%endif
|
|
|
|
%define ADDSUB add
|
|
%define ADCSBB adc
|
|
|
|
; Skylake has problems sustaining 2 read and 1 write per clock cycle.
|
|
; It sometimes gets into a "mode" (for the lack of a better word) where
|
|
; it does not fully utilize port 7, causing store uops to compete with
|
|
; the reads for ports 2,3. We try to alleviate the problem by turning
|
|
; some of the 64-bit writes into 128-bit writes, reducing the number of
|
|
; write instructions. Unfortunately, SSE2/AVX2 do not have particularly
|
|
; good instructions for assembling an SSE2 128-bit word from two GPR
|
|
; 64-bit words, so the instruction count is greatly inflated.
|
|
|
|
%macro STORE 1
|
|
mov [SumP %1], LIMB1
|
|
mov [SumP %1 + 8], LIMB2
|
|
%endmacro
|
|
|
|
%macro SSESTORE 1
|
|
movq xmm0, LIMB1
|
|
movq xmm1, LIMB2
|
|
vpermilpd xmm1, xmm1, 0
|
|
pblendw xmm0, xmm1, 0xf0
|
|
movaps [SumP %1], xmm0
|
|
%endmacro
|
|
|
|
|
|
BITS 64
|
|
|
|
LEAF_PROC mpn_add_n
|
|
; Make dest 16-bytes aligned
|
|
test SumP, 8
|
|
jz .aligned
|
|
dec Size
|
|
mov SizeRest, Size
|
|
and SizeRest, 7
|
|
shr Size, 3
|
|
; Unaligned and Size > 8: do one limb separately, then the normal loop
|
|
jnz .unaligned
|
|
; Unaligned and Size <= 8: do all with .rest loop
|
|
inc SizeRest
|
|
clc
|
|
jmp .rest ;ajs:notshortform
|
|
|
|
.aligned:
|
|
mov SizeRest, Size
|
|
and SizeRest, 7
|
|
shr Size, 3
|
|
clc
|
|
jz .rest ;ajs:notshortform
|
|
jmp .loop1
|
|
|
|
.unaligned:
|
|
mov LIMB1, [Inp1P]
|
|
ADDSUB LIMB1, [Inp2P]
|
|
mov [SumP], LIMB1
|
|
lea Inp1P, [Inp1P+8]
|
|
lea Inp2P, [Inp2P+8]
|
|
lea SumP, [SumP+8]
|
|
|
|
align 16
|
|
.loop1:
|
|
mov LIMB1, [Inp1P]
|
|
mov LIMB2, [Inp1P+8]
|
|
ADCSBB LIMB1, [Inp2P]
|
|
ADCSBB LIMB2, [Inp2P+8]
|
|
SSESTORE +0
|
|
mov LIMB1, [Inp1P+16]
|
|
mov LIMB2, [Inp1P+24]
|
|
ADCSBB LIMB1, [Inp2P+16]
|
|
ADCSBB LIMB2, [Inp2P+24]
|
|
STORE +16
|
|
mov LIMB1, [Inp1P+32]
|
|
mov LIMB2, [Inp1P+40]
|
|
ADCSBB LIMB1, [Inp2P+32]
|
|
ADCSBB LIMB2, [Inp2P+40]
|
|
STORE +32
|
|
mov LIMB1, [Inp1P+48]
|
|
mov LIMB2, [Inp1P+56]
|
|
ADCSBB LIMB1, [Inp2P+48]
|
|
ADCSBB LIMB2, [Inp2P+56]
|
|
STORE +48
|
|
lea Inp1P, [Inp1P+64]
|
|
lea Inp2P, [Inp2P+64]
|
|
lea SumP, [SumP+64]
|
|
dec Size
|
|
jnz .loop1
|
|
inc SizeRest
|
|
dec SizeRest
|
|
jz .end
|
|
.rest:
|
|
mov LIMB1, [Inp1P]
|
|
ADCSBB LIMB1, [Inp2P]
|
|
mov [SumP], LIMB1
|
|
dec SizeRest
|
|
jz .end
|
|
mov LIMB1, [Inp1P+8]
|
|
ADCSBB LIMB1, [Inp2P+8]
|
|
mov [SumP+8], LIMB1
|
|
dec SizeRest
|
|
jz .end
|
|
mov LIMB1, [Inp1P+16]
|
|
ADCSBB LIMB1, [Inp2P+16]
|
|
mov [SumP+16], LIMB1
|
|
dec SizeRest
|
|
jz .end
|
|
mov LIMB1, [Inp1P+24]
|
|
ADCSBB LIMB1, [Inp2P+24]
|
|
mov [SumP+24], LIMB1
|
|
dec SizeRest
|
|
jz .end
|
|
lea Inp1P, [Inp1P+32]
|
|
lea Inp2P, [Inp2P+32]
|
|
lea SumP, [SumP+32]
|
|
jmp .rest
|
|
.end:
|
|
mov eax, 0
|
|
adc eax, eax
|
|
ret
|