// Code generated by command: go run count_pair_asm.go -pkg mem -out ../mem/count_pair_amd64.s -stubs ../mem/count_pair_amd64.go. DO NOT EDIT.
//go:build !purego
#include "textflag.h"
// func countPair1(b []byte) int
// Requires: AVX, AVX2, CMOV, POPCNT
TEXT ·countPair1(SB), NOSPLIT, $0-32
MOVQ b_base+0(FP), AX
MOVQ b_len+8(FP), CX
XORQ DX, DX
SUBQ $0x01, CX
BTL $0x08, github·comâsegmentioâasmâcpu·X86+0(SB)
JCS avx2
tail:
CMPQ CX, $0x00
JLE done
generic:
MOVQ DX, BX
INCQ BX
MOVB (AX), SI
CMPB SI, 1(AX)
CMOVQEQ BX, DX
ADDQ $0x01, AX
SUBQ $0x01, CX
CMPQ CX, $0x00
JG generic
done:
MOVQ DX, ret+24(FP)
RET
avx2:
CMPQ CX, $0x00000101
JL avx2_tail128
XORQ BX, BX
XORQ SI, SI
XORQ DI, DI
XORQ R8, R8
XORQ R9, R9
XORQ R10, R10
XORQ R11, R11
XORQ R12, R12
avx2_loop256:
VMOVDQU (AX), Y0
VMOVDQU 32(AX), Y2
VMOVDQU 64(AX), Y4
VMOVDQU 96(AX), Y6
VMOVDQU 128(AX), Y8
VMOVDQU 160(AX), Y10
VMOVDQU 192(AX), Y12
VMOVDQU 224(AX), Y14
VMOVDQU 1(AX), Y1
VMOVDQU 33(AX), Y3
VMOVDQU 65(AX), Y5
VMOVDQU 97(AX), Y7
VMOVDQU 129(AX), Y9
VMOVDQU 161(AX), Y11
VMOVDQU 193(AX), Y13
VMOVDQU 225(AX), Y15
VPCMPEQB Y0, Y1, Y1
VPMOVMSKB Y1, BX
VPCMPEQB Y2, Y3, Y3
VPMOVMSKB Y3, SI
VPCMPEQB Y4, Y5, Y5
VPMOVMSKB Y5, DI
VPCMPEQB Y6, Y7, Y7
VPMOVMSKB Y7, R8
VPCMPEQB Y8, Y9, Y9
VPMOVMSKB Y9, R9
VPCMPEQB Y10, Y11, Y11
VPMOVMSKB Y11, R10
VPCMPEQB Y12, Y13, Y13
VPMOVMSKB Y13, R11
VPCMPEQB Y14, Y15, Y15
VPMOVMSKB Y15, R12
POPCNTQ BX, BX
POPCNTQ SI, SI
POPCNTQ DI, DI
POPCNTQ R8, R8
POPCNTQ R9, R9
POPCNTQ R10, R10
POPCNTQ R11, R11
POPCNTQ R12, R12
ADDQ SI, BX
ADDQ R8, DI
ADDQ DI, BX
ADDQ R10, R9
ADDQ R12, R11
ADDQ R11, R9
ADDQ R9, BX
ADDQ BX, DX
ADDQ $0x00000100, AX
SUBQ $0x00000100, CX
CMPQ CX, $0x00000101
JGE avx2_loop256
avx2_tail128:
CMPQ CX, $0x81
JL avx2_tail64
VMOVDQU (AX), Y0
VMOVDQU 32(AX), Y2
VMOVDQU 64(AX), Y4
VMOVDQU 96(AX), Y6
VMOVDQU 1(AX), Y1
VMOVDQU 33(AX), Y3
VMOVDQU 65(AX), Y5
VMOVDQU 97(AX), Y7
VPCMPEQB Y0, Y1, Y1
VPMOVMSKB Y1, BX
VPCMPEQB Y2, Y3, Y3
VPMOVMSKB Y3, SI
VPCMPEQB Y4, Y5, Y5
VPMOVMSKB Y5, DI
VPCMPEQB Y6, Y7, Y7
VPMOVMSKB Y7, R8
POPCNTQ BX, BX
POPCNTQ SI, SI
POPCNTQ DI, DI
POPCNTQ R8, R8
ADDQ SI, BX
ADDQ R8, DI
ADDQ DI, BX
ADDQ BX, DX
ADDQ $0x00000080, AX
SUBQ $0x00000080, CX
avx2_tail64:
CMPQ CX, $0x41
JL avx2_tail32
VMOVDQU (AX), Y0
VMOVDQU 32(AX), Y2
VMOVDQU 1(AX), Y1
VMOVDQU 33(AX), Y3
VPCMPEQB Y0, Y1, Y1
VPMOVMSKB Y1, BX
VPCMPEQB Y2, Y3, Y3
VPMOVMSKB Y3, SI
POPCNTQ BX, BX
POPCNTQ SI, SI
ADDQ SI, BX
ADDQ BX, DX
ADDQ $0x00000040, AX
SUBQ $0x00000040, CX
avx2_tail32:
CMPQ CX, $0x21
JL avx2_tail16
VMOVDQU (AX), Y0
VMOVDQU 1(AX), Y1
VPCMPEQB Y0, Y1, Y1
VPMOVMSKB Y1, BX
POPCNTQ BX, BX
ADDQ BX, DX
ADDQ $0x00000020, AX
SUBQ $0x00000020, CX
avx2_tail16:
CMPQ CX, $0x11
JL avx2_tail
VMOVDQU (AX), X0
VMOVDQU 1(AX), X1
VPCMPEQB X0, X1, X1
VPMOVMSKB X1, BX
POPCNTQ BX, BX
ADDQ BX, DX
ADDQ $0x10, AX
SUBQ $0x10, CX
avx2_tail:
VZEROUPPER
JMP tail
// func countPair2(b []byte) int
// Requires: AVX, AVX2, CMOV, POPCNT
TEXT ·countPair2(SB), NOSPLIT, $0-32
MOVQ b_base+0(FP), AX
MOVQ b_len+8(FP), CX
XORQ DX, DX
SUBQ $0x02, CX
BTL $0x08, github·comâsegmentioâasmâcpu·X86+0(SB)
JCS avx2
tail:
CMPQ CX, $0x00
JLE done
generic:
MOVQ DX, BX
INCQ BX
MOVW (AX), SI
CMPW SI, 2(AX)
CMOVQEQ BX, DX
ADDQ $0x02, AX
SUBQ $0x02, CX
CMPQ CX, $0x00
JG generic
done:
MOVQ DX, ret+24(FP)
RET
avx2:
CMPQ CX, $0x00000102
JL avx2_tail128
XORQ BX, BX
XORQ SI, SI
XORQ DI, DI
XORQ R8, R8
XORQ R9, R9
XORQ R10, R10
XORQ R11, R11
XORQ R12, R12
avx2_loop256:
VMOVDQU (AX), Y0
VMOVDQU 32(AX), Y2
VMOVDQU 64(AX), Y4
VMOVDQU 96(AX), Y6
VMOVDQU 128(AX), Y8
VMOVDQU 160(AX), Y10
VMOVDQU 192(AX), Y12
VMOVDQU 224(AX), Y14
VMOVDQU 2(AX), Y1
VMOVDQU 34(AX), Y3
VMOVDQU 66(AX), Y5
VMOVDQU 98(AX), Y7
VMOVDQU 130(AX), Y9
VMOVDQU 162(AX), Y11
VMOVDQU 194(AX), Y13
VMOVDQU 226(AX), Y15
VPCMPEQW Y0, Y1, Y1
VPMOVMSKB Y1, BX
VPCMPEQW Y2, Y3, Y3
VPMOVMSKB Y3, SI
VPCMPEQW Y4, Y5, Y5
VPMOVMSKB Y5, DI
VPCMPEQW Y6, Y7, Y7
VPMOVMSKB Y7, R8
VPCMPEQW Y8, Y9, Y9
VPMOVMSKB Y9, R9
VPCMPEQW Y10, Y11, Y11
VPMOVMSKB Y11, R10
VPCMPEQW Y12, Y13, Y13
VPMOVMSKB Y13, R11
VPCMPEQW Y14, Y15, Y15
VPMOVMSKB Y15, R12
POPCNTQ BX, BX
POPCNTQ SI, SI
POPCNTQ DI, DI
POPCNTQ R8, R8
POPCNTQ R9, R9
POPCNTQ R10, R10
POPCNTQ R11, R11
POPCNTQ R12, R12
ADDQ SI, BX
ADDQ R8, DI
ADDQ DI, BX
ADDQ R10, R9
ADDQ R12, R11
ADDQ R11, R9
ADDQ R9, BX
ADDQ BX, DX
ADDQ $0x00000100, AX
SUBQ $0x00000100, CX
CMPQ CX, $0x00000102
JGE avx2_loop256
avx2_tail128:
CMPQ CX, $0x82
JL avx2_tail64
VMOVDQU (AX), Y0
VMOVDQU 32(AX), Y2
VMOVDQU 64(AX), Y4
VMOVDQU 96(AX), Y6
VMOVDQU 2(AX), Y1
VMOVDQU 34(AX), Y3
VMOVDQU 66(AX), Y5
VMOVDQU 98(AX), Y7
VPCMPEQW Y0, Y1, Y1
VPMOVMSKB Y1, BX
VPCMPEQW Y2, Y3, Y3
VPMOVMSKB Y3, SI
VPCMPEQW Y4, Y5, Y5
VPMOVMSKB Y5, DI
VPCMPEQW Y6, Y7, Y7
VPMOVMSKB Y7, R8
POPCNTQ BX, BX
POPCNTQ SI, SI
POPCNTQ DI, DI
POPCNTQ R8, R8
ADDQ SI, BX
ADDQ R8, DI
ADDQ DI, BX
ADDQ BX, DX
ADDQ $0x00000080, AX
SUBQ $0x00000080, CX
avx2_tail64:
CMPQ CX, $0x42
JL avx2_tail32
VMOVDQU (AX), Y0
VMOVDQU 32(AX), Y2
VMOVDQU 2(AX), Y1
VMOVDQU 34(AX), Y3
VPCMPEQW Y0, Y1, Y1
VPMOVMSKB Y1, BX
VPCMPEQW Y2, Y3, Y3
VPMOVMSKB Y3, SI
POPCNTQ BX, BX
POPCNTQ SI, SI
ADDQ SI, BX
ADDQ BX, DX
ADDQ $0x00000040, AX
SUBQ $0x00000040, CX
avx2_tail32:
CMPQ CX, $0x22
JL avx2_tail16
VMOVDQU (AX), Y0
VMOVDQU 2(AX), Y1
VPCMPEQW Y0, Y1, Y1
VPMOVMSKB Y1, BX
POPCNTQ BX, BX
ADDQ BX, DX
ADDQ $0x00000020, AX
SUBQ $0x00000020, CX
avx2_tail16:
CMPQ CX, $0x12
JL avx2_tail
VMOVDQU (AX), X0
VMOVDQU 2(AX), X1
VPCMPEQW X0, X1, X1
VPMOVMSKB X1, BX
POPCNTQ BX, BX
ADDQ BX, DX
ADDQ $0x10, AX
SUBQ $0x10, CX
avx2_tail:
VZEROUPPER
SHRQ $0x01, DX
JMP tail
// func countPair4(b []byte) int
// Requires: AVX, AVX2, CMOV, POPCNT
TEXT ·countPair4(SB), NOSPLIT, $0-32
MOVQ b_base+0(FP), AX
MOVQ b_len+8(FP), CX
XORQ DX, DX
SUBQ $0x04, CX
BTL $0x08, github·comâsegmentioâasmâcpu·X86+0(SB)
JCS avx2
tail:
CMPQ CX, $0x00
JLE done
generic:
MOVQ DX, BX
INCQ BX
MOVL (AX), SI
CMPL SI, 4(AX)
CMOVQEQ BX, DX
ADDQ $0x04, AX
SUBQ $0x04, CX
CMPQ CX, $0x00
JG generic
done:
MOVQ DX, ret+24(FP)
RET
avx2:
CMPQ CX, $0x00000104
JL avx2_tail128
XORQ BX, BX
XORQ SI, SI
XORQ DI, DI
XORQ R8, R8
XORQ R9, R9
XORQ R10, R10
XORQ R11, R11
XORQ R12, R12
avx2_loop256:
VMOVDQU (AX), Y0
VMOVDQU 32(AX), Y2
VMOVDQU 64(AX), Y4
VMOVDQU 96(AX), Y6
VMOVDQU 128(AX), Y8
VMOVDQU 160(AX), Y10
VMOVDQU 192(AX), Y12
VMOVDQU 224(AX), Y14
VMOVDQU 4(AX), Y1
VMOVDQU 36(AX), Y3
VMOVDQU 68(AX), Y5
VMOVDQU 100(AX), Y7
VMOVDQU 132(AX), Y9
VMOVDQU 164(AX), Y11
VMOVDQU 196(AX), Y13
VMOVDQU 228(AX), Y15
VPCMPEQD Y0, Y1, Y1
VPMOVMSKB Y1, BX
VPCMPEQD Y2, Y3, Y3
VPMOVMSKB Y3, SI
VPCMPEQD Y4, Y5, Y5
VPMOVMSKB Y5, DI
VPCMPEQD Y6, Y7, Y7
VPMOVMSKB Y7, R8
VPCMPEQD Y8, Y9, Y9
VPMOVMSKB Y9, R9
VPCMPEQD Y10, Y11, Y11
VPMOVMSKB Y11, R10
VPCMPEQD Y12, Y13, Y13
VPMOVMSKB Y13, R11
VPCMPEQD Y14, Y15, Y15
VPMOVMSKB Y15, R12
POPCNTQ BX, BX
POPCNTQ SI, SI
POPCNTQ DI, DI
POPCNTQ R8, R8
POPCNTQ R9, R9
POPCNTQ R10, R10
POPCNTQ R11, R11
POPCNTQ R12, R12
ADDQ SI, BX
ADDQ R8, DI
ADDQ DI, BX
ADDQ R10, R9
ADDQ R12, R11
ADDQ R11, R9
ADDQ R9, BX
ADDQ BX, DX
ADDQ $0x00000100, AX
SUBQ $0x00000100, CX
CMPQ CX, $0x00000104
JGE avx2_loop256
avx2_tail128:
CMPQ CX, $0x84
JL avx2_tail64
VMOVDQU (AX), Y0
VMOVDQU 32(AX), Y2
VMOVDQU 64(AX), Y4
VMOVDQU 96(AX), Y6
VMOVDQU 4(AX), Y1
VMOVDQU 36(AX), Y3
VMOVDQU 68(AX), Y5
VMOVDQU 100(AX), Y7
VPCMPEQD Y0, Y1, Y1
VPMOVMSKB Y1, BX
VPCMPEQD Y2, Y3, Y3
VPMOVMSKB Y3, SI
VPCMPEQD Y4, Y5, Y5
VPMOVMSKB Y5, DI
VPCMPEQD Y6, Y7, Y7
VPMOVMSKB Y7, R8
POPCNTQ BX, BX
POPCNTQ SI, SI
POPCNTQ DI, DI
POPCNTQ R8, R8
ADDQ SI, BX
ADDQ R8, DI
ADDQ DI, BX
ADDQ BX, DX
ADDQ $0x00000080, AX
SUBQ $0x00000080, CX
avx2_tail64:
CMPQ CX, $0x44
JL avx2_tail32
VMOVDQU (AX), Y0
VMOVDQU 32(AX), Y2
VMOVDQU 4(AX), Y1
VMOVDQU 36(AX), Y3
VPCMPEQD Y0, Y1, Y1
VPMOVMSKB Y1, BX
VPCMPEQD Y2, Y3, Y3
VPMOVMSKB Y3, SI
POPCNTQ BX, BX
POPCNTQ SI, SI
ADDQ SI, BX
ADDQ BX, DX
ADDQ $0x00000040, AX
SUBQ $0x00000040, CX
avx2_tail32:
CMPQ CX, $0x24
JL avx2_tail16
VMOVDQU (AX), Y0
VMOVDQU 4(AX), Y1
VPCMPEQD Y0, Y1, Y1
VPMOVMSKB Y1, BX
POPCNTQ BX, BX
ADDQ BX, DX
ADDQ $0x00000020, AX
SUBQ $0x00000020, CX
avx2_tail16:
CMPQ CX, $0x14
JL avx2_tail
VMOVDQU (AX), X0
VMOVDQU 4(AX), X1
VPCMPEQD X0, X1, X1
VPMOVMSKB X1, BX
POPCNTQ BX, BX
ADDQ BX, DX
ADDQ $0x10, AX
SUBQ $0x10, CX
avx2_tail:
VZEROUPPER
SHRQ $0x02, DX
JMP tail
// func countPair8(b []byte) int
// Requires: AVX, AVX2, CMOV, POPCNT
TEXT ·countPair8(SB), NOSPLIT, $0-32
MOVQ b_base+0(FP), AX
MOVQ b_len+8(FP), CX
XORQ DX, DX
SUBQ $0x08, CX
BTL $0x08, github·comâsegmentioâasmâcpu·X86+0(SB)
JCS avx2
tail:
CMPQ CX, $0x00
JLE done
generic:
MOVQ DX, BX
INCQ BX
MOVQ (AX), SI
CMPQ SI, 8(AX)
CMOVQEQ BX, DX
ADDQ $0x08, AX
SUBQ $0x08, CX
CMPQ CX, $0x00
JG generic
done:
MOVQ DX, ret+24(FP)
RET
avx2:
CMPQ CX, $0x00000108
JL avx2_tail128
XORQ BX, BX
XORQ SI, SI
XORQ DI, DI
XORQ R8, R8
XORQ R9, R9
XORQ R10, R10
XORQ R11, R11
XORQ R12, R12
avx2_loop256:
VMOVDQU (AX), Y0
VMOVDQU 32(AX), Y2
VMOVDQU 64(AX), Y4
VMOVDQU 96(AX), Y6
VMOVDQU 128(AX), Y8
VMOVDQU 160(AX), Y10
VMOVDQU 192(AX), Y12
VMOVDQU 224(AX), Y14
VMOVDQU 8(AX), Y1
VMOVDQU 40(AX), Y3
VMOVDQU 72(AX), Y5
VMOVDQU 104(AX), Y7
VMOVDQU 136(AX), Y9
VMOVDQU 168(AX), Y11
VMOVDQU 200(AX), Y13
VMOVDQU 232(AX), Y15
VPCMPEQQ Y0, Y1, Y1
VPMOVMSKB Y1, BX
VPCMPEQQ Y2, Y3, Y3
VPMOVMSKB Y3, SI
VPCMPEQQ Y4, Y5, Y5
VPMOVMSKB Y5, DI
VPCMPEQQ Y6, Y7, Y7
VPMOVMSKB Y7, R8
VPCMPEQQ Y8, Y9, Y9
VPMOVMSKB Y9, R9
VPCMPEQQ Y10, Y11, Y11
VPMOVMSKB Y11, R10
VPCMPEQQ Y12, Y13, Y13
VPMOVMSKB Y13, R11
VPCMPEQQ Y14, Y15, Y15
VPMOVMSKB Y15, R12
POPCNTQ BX, BX
POPCNTQ SI, SI
POPCNTQ DI, DI
POPCNTQ R8, R8
POPCNTQ R9, R9
POPCNTQ R10, R10
POPCNTQ R11, R11
POPCNTQ R12, R12
ADDQ SI, BX
ADDQ R8, DI
ADDQ DI, BX
ADDQ R10, R9
ADDQ R12, R11
ADDQ R11, R9
ADDQ R9, BX
ADDQ BX, DX
ADDQ $0x00000100, AX
SUBQ $0x00000100, CX
CMPQ CX, $0x00000108
JGE avx2_loop256
avx2_tail128:
CMPQ CX, $0x88
JL avx2_tail64
VMOVDQU (AX), Y0
VMOVDQU 32(AX), Y2
VMOVDQU 64(AX), Y4
VMOVDQU 96(AX), Y6
VMOVDQU 8(AX), Y1
VMOVDQU 40(AX), Y3
VMOVDQU 72(AX), Y5
VMOVDQU 104(AX), Y7
VPCMPEQQ Y0, Y1, Y1
VPMOVMSKB Y1, BX
VPCMPEQQ Y2, Y3, Y3
VPMOVMSKB Y3, SI
VPCMPEQQ Y4, Y5, Y5
VPMOVMSKB Y5, DI
VPCMPEQQ Y6, Y7, Y7
VPMOVMSKB Y7, R8
POPCNTQ BX, BX
POPCNTQ SI, SI
POPCNTQ DI, DI
POPCNTQ R8, R8
ADDQ SI, BX
ADDQ R8, DI
ADDQ DI, BX
ADDQ BX, DX
ADDQ $0x00000080, AX
SUBQ $0x00000080, CX
avx2_tail64:
CMPQ CX, $0x48
JL avx2_tail32
VMOVDQU (AX), Y0
VMOVDQU 32(AX), Y2
VMOVDQU 8(AX), Y1
VMOVDQU 40(AX), Y3
VPCMPEQQ Y0, Y1, Y1
VPMOVMSKB Y1, BX
VPCMPEQQ Y2, Y3, Y3
VPMOVMSKB Y3, SI
POPCNTQ BX, BX
POPCNTQ SI, SI
ADDQ SI, BX
ADDQ BX, DX
ADDQ $0x00000040, AX
SUBQ $0x00000040, CX
avx2_tail32:
CMPQ CX, $0x28
JL avx2_tail16
VMOVDQU (AX), Y0
VMOVDQU 8(AX), Y1
VPCMPEQQ Y0, Y1, Y1
VPMOVMSKB Y1, BX
POPCNTQ BX, BX
ADDQ BX, DX
ADDQ $0x00000020, AX
SUBQ $0x00000020, CX
avx2_tail16:
CMPQ CX, $0x18
JL avx2_tail
VMOVDQU (AX), X0
VMOVDQU 8(AX), X1
VPCMPEQQ X0, X1, X1
VPMOVMSKB X1, BX
POPCNTQ BX, BX
ADDQ BX, DX
ADDQ $0x10, AX
SUBQ $0x10, CX
avx2_tail:
VZEROUPPER
SHRQ $0x03, DX
JMP tail
// func countPair16(b []byte) int
// Requires: AVX, AVX2, CMOV, POPCNT, SSE2, SSE4.1
TEXT ·countPair16(SB), NOSPLIT, $0-32
MOVQ b_base+0(FP), AX
MOVQ b_len+8(FP), CX
XORQ DX, DX
SUBQ $0x10, CX
BTL $0x08, github·comâsegmentioâasmâcpu·X86+0(SB)
JCS avx2
tail:
CMPQ CX, $0x00
JLE done
generic:
MOVQ DX, BX
INCQ BX
MOVOU (AX), X0
MOVOU 16(AX), X1
PCMPEQQ X0, X1
PMOVMSKB X1, SI
CMPL SI, $0x0000ffff
CMOVQEQ BX, DX
ADDQ $0x10, AX
SUBQ $0x10, CX
CMPQ CX, $0x00
JG generic
done:
MOVQ DX, ret+24(FP)
RET
avx2:
CMPQ CX, $0x00000110
JL avx2_tail128
XORQ BX, BX
XORQ SI, SI
XORQ DI, DI
XORQ R8, R8
XORQ R9, R9
XORQ R10, R10
XORQ R11, R11
XORQ R12, R12
avx2_loop256:
VMOVDQU (AX), Y0
VMOVDQU 32(AX), Y2
VMOVDQU 64(AX), Y4
VMOVDQU 96(AX), Y6
VMOVDQU 128(AX), Y8
VMOVDQU 160(AX), Y10
VMOVDQU 192(AX), Y12
VMOVDQU 224(AX), Y14
VPERM2I128 $0x21, Y2, Y0, Y1
VPERM2I128 $0x21, Y4, Y2, Y3
VPERM2I128 $0x21, Y6, Y4, Y5
VPERM2I128 $0x21, Y8, Y6, Y7
VPERM2I128 $0x21, Y10, Y8, Y9
VPERM2I128 $0x21, Y12, Y10, Y11
VPERM2I128 $0x21, Y14, Y12, Y13
VMOVDQU 240(AX), Y15
VPCMPEQQ Y0, Y1, Y1
VPERMQ $0xb1, Y1, Y0
VPAND Y1, Y0, Y0
VPMOVMSKB Y0, BX
VPCMPEQQ Y2, Y3, Y3
VPERMQ $0xb1, Y3, Y2
VPAND Y3, Y2, Y2
VPMOVMSKB Y2, SI
VPCMPEQQ Y4, Y5, Y5
VPERMQ $0xb1, Y5, Y4
VPAND Y5, Y4, Y4
VPMOVMSKB Y4, DI
VPCMPEQQ Y6, Y7, Y7
VPERMQ $0xb1, Y7, Y6
VPAND Y7, Y6, Y6
VPMOVMSKB Y6, R8
VPCMPEQQ Y8, Y9, Y9
VPERMQ $0xb1, Y9, Y8
VPAND Y9, Y8, Y8
VPMOVMSKB Y8, R9
VPCMPEQQ Y10, Y11, Y11
VPERMQ $0xb1, Y11, Y10
VPAND Y11, Y10, Y10
VPMOVMSKB Y10, R10
VPCMPEQQ Y12, Y13, Y13
VPERMQ $0xb1, Y13, Y12
VPAND Y13, Y12, Y12
VPMOVMSKB Y12, R11
VPCMPEQQ Y14, Y15, Y15
VPERMQ $0xb1, Y15, Y14
VPAND Y15, Y14, Y14
VPMOVMSKB Y14, R12
POPCNTQ BX, BX
POPCNTQ SI, SI
POPCNTQ DI, DI
POPCNTQ R8, R8
POPCNTQ R9, R9
POPCNTQ R10, R10
POPCNTQ R11, R11
POPCNTQ R12, R12
ADDQ SI, BX
ADDQ R8, DI
ADDQ DI, BX
ADDQ R10, R9
ADDQ R12, R11
ADDQ R11, R9
ADDQ R9, BX
ADDQ BX, DX
ADDQ $0x00000100, AX
SUBQ $0x00000100, CX
CMPQ CX, $0x00000110
JGE avx2_loop256
avx2_tail128:
CMPQ CX, $0x90
JL avx2_tail64
VMOVDQU (AX), Y0
VMOVDQU 32(AX), Y2
VMOVDQU 64(AX), Y4
VMOVDQU 96(AX), Y6
VPERM2I128 $0x21, Y2, Y0, Y1
VPERM2I128 $0x21, Y4, Y2, Y3
VPERM2I128 $0x21, Y6, Y4, Y5
VMOVDQU 112(AX), Y7
VPCMPEQQ Y0, Y1, Y1
VPERMQ $0xb1, Y1, Y0
VPAND Y1, Y0, Y0
VPMOVMSKB Y0, BX
VPCMPEQQ Y2, Y3, Y3
VPERMQ $0xb1, Y3, Y2
VPAND Y3, Y2, Y2
VPMOVMSKB Y2, SI
VPCMPEQQ Y4, Y5, Y5
VPERMQ $0xb1, Y5, Y4
VPAND Y5, Y4, Y4
VPMOVMSKB Y4, DI
VPCMPEQQ Y6, Y7, Y7
VPERMQ $0xb1, Y7, Y6
VPAND Y7, Y6, Y6
VPMOVMSKB Y6, R8
POPCNTQ BX, BX
POPCNTQ SI, SI
POPCNTQ DI, DI
POPCNTQ R8, R8
ADDQ SI, BX
ADDQ R8, DI
ADDQ DI, BX
ADDQ BX, DX
ADDQ $0x00000080, AX
SUBQ $0x00000080, CX
avx2_tail64:
CMPQ CX, $0x50
JL avx2_tail32
VMOVDQU (AX), Y0
VMOVDQU 32(AX), Y2
VPERM2I128 $0x21, Y2, Y0, Y1
VMOVDQU 48(AX), Y3
VPCMPEQQ Y0, Y1, Y1
VPERMQ $0xb1, Y1, Y0
VPAND Y1, Y0, Y0
VPMOVMSKB Y0, BX
VPCMPEQQ Y2, Y3, Y3
VPERMQ $0xb1, Y3, Y2
VPAND Y3, Y2, Y2
VPMOVMSKB Y2, SI
POPCNTQ BX, BX
POPCNTQ SI, SI
ADDQ SI, BX
ADDQ BX, DX
ADDQ $0x00000040, AX
SUBQ $0x00000040, CX
avx2_tail32:
CMPQ CX, $0x30
JL avx2_tail16
VMOVDQU (AX), Y0
VMOVDQU 16(AX), Y1
VPCMPEQQ Y0, Y1, Y1
VPERMQ $0xb1, Y1, Y0
VPAND Y1, Y0, Y0
VPMOVMSKB Y0, BX
POPCNTQ BX, BX
ADDQ BX, DX
ADDQ $0x00000020, AX
SUBQ $0x00000020, CX
avx2_tail16:
VZEROUPPER
SHRQ $0x04, DX
JMP tail
// func countPair32(b []byte) int
// Requires: AVX, AVX2, CMOV, POPCNT, SSE2, SSE4.1
TEXT ·countPair32(SB), NOSPLIT, $0-32
MOVQ b_base+0(FP), AX
MOVQ b_len+8(FP), CX
XORQ DX, DX
SUBQ $0x20, CX
BTL $0x08, github·comâsegmentioâasmâcpu·X86+0(SB)
JCS avx2
tail:
CMPQ CX, $0x00
JLE done
generic:
MOVQ DX, BX
INCQ BX
MOVOU (AX), X0
MOVOU 16(AX), X1
MOVOU 32(AX), X2
MOVOU 48(AX), X3
PCMPEQQ X0, X2
PCMPEQQ X1, X3
PMOVMSKB X2, SI
PMOVMSKB X3, DI
ANDL DI, SI
CMPL SI, $0x0000ffff
CMOVQEQ BX, DX
ADDQ $0x20, AX
SUBQ $0x20, CX
CMPQ CX, $0x00
JG generic
done:
MOVQ DX, ret+24(FP)
RET
avx2:
CMPQ CX, $0x00000120
JL avx2_tail128
XORQ BX, BX
XORQ SI, SI
XORQ DI, DI
XORQ R8, R8
XORQ R9, R9
XORQ R10, R10
XORQ R11, R11
XORQ R12, R12
avx2_loop256:
VMOVDQU (AX), Y0
VMOVDQU 32(AX), Y2
VMOVDQU 64(AX), Y4
VMOVDQU 96(AX), Y5
VMOVDQU 128(AX), Y7
VMOVDQU 160(AX), Y8
VMOVDQU 192(AX), Y9
VMOVDQU 224(AX), Y10
VMOVDQU 256(AX), Y11
VPCMPEQQ Y0, Y2, Y1
VPMOVMSKB Y1, BX
VPCMPEQQ Y2, Y4, Y3
VPMOVMSKB Y3, SI
VPCMPEQQ Y4, Y5, Y0
VPMOVMSKB Y0, DI
VPCMPEQQ Y5, Y7, Y6
VPMOVMSKB Y6, R8
VPCMPEQQ Y7, Y8, Y0
VPMOVMSKB Y0, R9
VPCMPEQQ Y8, Y9, Y0
VPMOVMSKB Y0, R10
VPCMPEQQ Y9, Y10, Y0
VPMOVMSKB Y0, R11
VPCMPEQQ Y10, Y11, Y11
VPMOVMSKB Y11, R12
POPCNTQ BX, BX
SHRQ $0x05, BX
POPCNTQ SI, SI
SHRQ $0x05, SI
POPCNTQ DI, DI
SHRQ $0x05, DI
POPCNTQ R8, R8
SHRQ $0x05, R8
POPCNTQ R9, R9
SHRQ $0x05, R9
POPCNTQ R10, R10
SHRQ $0x05, R10
POPCNTQ R11, R11
SHRQ $0x05, R11
POPCNTQ R12, R12
SHRQ $0x05, R12
ADDQ SI, BX
ADDQ R8, DI
ADDQ DI, BX
ADDQ R10, R9
ADDQ R12, R11
ADDQ R11, R9
ADDQ R9, BX
ADDQ BX, DX
ADDQ $0x00000100, AX
SUBQ $0x00000100, CX
CMPQ CX, $0x00000120
JGE avx2_loop256
avx2_tail128:
CMPQ CX, $0xa0
JL avx2_tail64
VMOVDQU (AX), Y0
VMOVDQU 32(AX), Y2
VMOVDQU 64(AX), Y4
VMOVDQU 96(AX), Y5
VMOVDQU 128(AX), Y6
VPCMPEQQ Y0, Y2, Y1
VPMOVMSKB Y1, BX
VPCMPEQQ Y2, Y4, Y3
VPMOVMSKB Y3, SI
VPCMPEQQ Y4, Y5, Y0
VPMOVMSKB Y0, DI
VPCMPEQQ Y5, Y6, Y6
VPMOVMSKB Y6, R8
POPCNTQ BX, BX
SHRQ $0x05, BX
POPCNTQ SI, SI
SHRQ $0x05, SI
POPCNTQ DI, DI
SHRQ $0x05, DI
POPCNTQ R8, R8
SHRQ $0x05, R8
ADDQ SI, BX
ADDQ R8, DI
ADDQ DI, BX
ADDQ BX, DX
ADDQ $0x00000080, AX
SUBQ $0x00000080, CX
avx2_tail64:
CMPQ CX, $0x60
JL avx2_tail32
VMOVDQU (AX), Y0
VMOVDQU 32(AX), Y2
VMOVDQU 64(AX), Y3
VPCMPEQQ Y0, Y2, Y1
VPMOVMSKB Y1, BX
VPCMPEQQ Y2, Y3, Y3
VPMOVMSKB Y3, SI
POPCNTQ BX, BX
SHRQ $0x05, BX
POPCNTQ SI, SI
SHRQ $0x05, SI
ADDQ SI, BX
ADDQ BX, DX
ADDQ $0x00000040, AX
SUBQ $0x00000040, CX
avx2_tail32:
CMPQ CX, $0x40
JL avx2_tail16
VMOVDQU (AX), Y0
VMOVDQU 32(AX), Y1
VPCMPEQQ Y0, Y1, Y1
VPMOVMSKB Y1, BX
POPCNTQ BX, BX
SHRQ $0x05, BX
ADDQ BX, DX
ADDQ $0x00000020, AX
SUBQ $0x00000020, CX
avx2_tail16:
VZEROUPPER
JMP tail