c8a32b25ec
We use 6 instructions only to calculate mask based on single bit value. Use only 3 now and calculate multiple masks in parallel. Also `VPSUB*` is faster than VPBROADCAST*, see https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html . ``` name old time/op new time/op delta Sum/AVX2Inline_digest-8 1.83ms ± 0% 1.62ms ± 1% -11.23% (p=0.000 n=46+42) name old speed new speed delta Sum/AVX2Inline_digest-8 54.7MB/s ± 0% 61.6MB/s ± 1% +12.65% (p=0.000 n=46+42) ``` Signed-off-by: Evgenii Stratonikov <evgeniy@nspcc.ru>
75 lines
1.7 KiB
ArmAsm
75 lines
1.7 KiB
ArmAsm
#include "textflag.h"
|
|
|
|
#define mask(bit, tmp, to) \
|
|
VPSRLW bit, Y10, tmp \
|
|
VPAND Y12, tmp, to \ // to2 = 0x000<bit>000<bit>...
|
|
VPSUBW to, Y13, to // to2 = 0xFFFF.. or 0x0000 depending on bit
|
|
|
|
#define mulBit(bit) \
|
|
VPSLLQ $1, Y0, Y1 \
|
|
VPALIGNR $8, Y1, Y0, Y2 \
|
|
VPSRLQ $63, Y2, Y2 \
|
|
VPXOR Y1, Y2, Y2 \
|
|
VPSRLQ $63, Y1, Y3 \
|
|
VPSLLQ $63, Y3, Y3 \
|
|
VPUNPCKHQDQ Y3, Y3, Y3 \
|
|
VPXOR Y2, Y3, Y3 \
|
|
mask(bit, Y11, Y2) \
|
|
VPXOR Y3, Y8, Y3 \
|
|
VPAND Y3, Y2, Y4 \
|
|
VPXOR Y4, Y0, Y8 \
|
|
VMOVDQA Y3, Y0
|
|
|
|
// func mulByteRightx2(c00c10, c01c11 *[4]uint64, b byte)
|
|
TEXT ·mulByteRightx2(SB),NOSPLIT,$0
|
|
MOVQ c00c10+0(FP), AX
|
|
VMOVDQA (AX), Y0
|
|
MOVQ c01c11+8(FP), BX
|
|
VMOVDQA (BX), Y8
|
|
|
|
VPXOR Y13, Y13, Y13 // Y13 = 0x0000...
|
|
VPCMPEQB Y12, Y12, Y12 // Y12 = 0xFFFF...
|
|
VPSUBW Y12, Y13, Y12 // Y12 = 0x00010001... (packed words of 1)
|
|
|
|
VPBROADCASTB b+16(FP), X10 // X10 = packed bytes of b.
|
|
VPMOVZXBW X10, Y10 // Extend with zeroes to packed words.
|
|
|
|
mulBit($7)
|
|
mulBit($6)
|
|
mulBit($5)
|
|
mulBit($4)
|
|
mulBit($3)
|
|
mulBit($2)
|
|
mulBit($1)
|
|
mulBit($0)
|
|
|
|
VMOVDQA Y8, (BX)
|
|
VMOVDQA Y0, (AX)
|
|
|
|
RET
|
|
|
|
// func mulBitRightx2(c00c10, c01c11 *[4]uint64, e *[2]uint64)
|
|
TEXT ·mulBitRightx2(SB),NOSPLIT,$0
|
|
MOVQ c00c10+0(FP), AX
|
|
VMOVDQA (AX), Y0
|
|
MOVQ c01c11+8(FP), BX
|
|
VMOVDQA (BX), Y8
|
|
|
|
VPSLLQ $1, Y0, Y1
|
|
VPALIGNR $8, Y1, Y0, Y2
|
|
VPSRLQ $63, Y2, Y2
|
|
VPXOR Y1, Y2, Y2
|
|
VPSRLQ $63, Y1, Y3
|
|
VPSLLQ $63, Y3, Y3
|
|
VPUNPCKHQDQ Y3, Y3, Y3
|
|
VPXOR Y2, Y3, Y3
|
|
|
|
MOVQ e+16(FP), CX
|
|
VBROADCASTI128 (CX), Y2
|
|
|
|
VPXOR Y3, Y8, Y3
|
|
VPAND Y3, Y2, Y4
|
|
VPXOR Y4, Y0, Y8
|
|
VMOVDQA Y8, (BX)
|
|
VMOVDQA Y3, (AX)
|
|
RET
|