// Code generated by command: go run index_pair_asm.go -pkg mem -out ../mem/index_pair_amd64.s -stubs ../mem/index_pair_amd64.go. DO NOT EDIT.
//go:build !purego
#include "textflag.h"
// func indexPair1(b []byte) int
// Requires: AVX, AVX2, BMI
TEXT ·indexPair1(SB), NOSPLIT, $0-32
MOVQ b_base+0(FP), AX
MOVQ b_len+8(FP), CX
MOVQ AX, DX
CMPQ CX, $0x00
JLE fail
SUBQ $0x01, CX
BTL $0x08, github·comâsegmentioâasmâcpu·X86+0(SB)
JCS avx2
tail:
CMPQ CX, $0x00
JE fail
generic:
MOVB (AX), BL
CMPB BL, 1(AX)
JE done
ADDQ $0x01, AX
SUBQ $0x01, CX
CMPQ CX, $0x00
JA generic
fail:
MOVQ $0xffffffffffffffff, AX
MOVQ AX, ret+24(FP)
RET
done:
SUBQ DX, AX
MOVQ AX, ret+24(FP)
RET
avx2:
CMPQ CX, $0x00000101
JB avx2_tail128
XORQ BX, BX
XORQ SI, SI
XORQ DI, DI
XORQ R8, R8
XORQ R9, R9
XORQ R10, R10
XORQ R11, R11
XORQ R12, R12
avx2_loop256:
VMOVDQU (AX), Y0
VMOVDQU 32(AX), Y2
VMOVDQU 64(AX), Y4
VMOVDQU 96(AX), Y6
VMOVDQU 128(AX), Y8
VMOVDQU 160(AX), Y10
VMOVDQU 192(AX), Y12
VMOVDQU 224(AX), Y14
VMOVDQU 1(AX), Y1
VMOVDQU 33(AX), Y3
VMOVDQU 65(AX), Y5
VMOVDQU 97(AX), Y7
VMOVDQU 129(AX), Y9
VMOVDQU 161(AX), Y11
VMOVDQU 193(AX), Y13
VMOVDQU 225(AX), Y15
VPCMPEQB Y0, Y1, Y1
VPCMPEQB Y2, Y3, Y3
VPCMPEQB Y4, Y5, Y5
VPCMPEQB Y6, Y7, Y7
VPCMPEQB Y8, Y9, Y9
VPCMPEQB Y10, Y11, Y11
VPCMPEQB Y12, Y13, Y13
VPCMPEQB Y14, Y15, Y15
VPMOVMSKB Y1, BX
VPMOVMSKB Y3, SI
VPMOVMSKB Y5, DI
VPMOVMSKB Y7, R8
VPMOVMSKB Y9, R9
VPMOVMSKB Y11, R10
VPMOVMSKB Y13, R11
VPMOVMSKB Y15, R12
XORQ R13, R13
ORQ BX, R13
ORQ SI, R13
ORQ DI, R13
ORQ R8, R13
ORQ R9, R13
ORQ R10, R13
ORQ R11, R13
ORQ R12, R13
CMPQ R13, $0x00
JNE avx2_done
ADDQ $0x00000100, AX
SUBQ $0x00000100, CX
CMPQ CX, $0x00000101
JAE avx2_loop256
avx2_tail128:
CMPQ CX, $0x81
JB avx2_tail64
VMOVDQU (AX), Y0
VMOVDQU 32(AX), Y2
VMOVDQU 64(AX), Y4
VMOVDQU 96(AX), Y6
VMOVDQU 1(AX), Y1
VMOVDQU 33(AX), Y3
VMOVDQU 65(AX), Y5
VMOVDQU 97(AX), Y7
VPCMPEQB Y0, Y1, Y1
VPCMPEQB Y2, Y3, Y3
VPCMPEQB Y4, Y5, Y5
VPCMPEQB Y6, Y7, Y7
VPMOVMSKB Y1, BX
VPMOVMSKB Y3, SI
VPMOVMSKB Y5, DI
VPMOVMSKB Y7, R8
XORQ R13, R13
ORQ BX, R13
ORQ SI, R13
ORQ DI, R13
ORQ R8, R13
CMPQ R13, $0x00
JNE avx2_done
ADDQ $0x00000080, AX
SUBQ $0x00000080, CX
avx2_tail64:
CMPQ CX, $0x41
JB avx2_tail32
VMOVDQU (AX), Y0
VMOVDQU 32(AX), Y2
VMOVDQU 1(AX), Y1
VMOVDQU 33(AX), Y3
VPCMPEQB Y0, Y1, Y1
VPCMPEQB Y2, Y3, Y3
VPMOVMSKB Y1, BX
VPMOVMSKB Y3, SI
XORQ R13, R13
ORQ BX, R13
ORQ SI, R13
CMPQ R13, $0x00
JNE avx2_done
ADDQ $0x00000040, AX
SUBQ $0x00000040, CX
avx2_tail32:
CMPQ CX, $0x21
JB avx2_tail16
VMOVDQU (AX), Y0
VMOVDQU 1(AX), Y1
VPCMPEQB Y0, Y1, Y1
VPMOVMSKB Y1, BX
CMPQ BX, $0x00
JNE avx2_done
ADDQ $0x00000020, AX
SUBQ $0x00000020, CX
avx2_tail16:
CMPQ CX, $0x11
JB avx2_tail
VMOVDQU (AX), X0
VMOVDQU 1(AX), X1
VPCMPEQB X0, X1, X1
VPMOVMSKB X1, BX
CMPQ BX, $0x00
JNE avx2_done
ADDQ $0x10, AX
SUBQ $0x10, CX
avx2_tail:
VZEROUPPER
JMP tail
avx2_done:
VZEROUPPER
CMPQ BX, $0x00
JNE avx2_done0
CMPQ SI, $0x00
JNE avx2_done1
CMPQ DI, $0x00
JNE avx2_done2
CMPQ R8, $0x00
JNE avx2_done3
CMPQ R9, $0x00
JNE avx2_done4
CMPQ R10, $0x00
JNE avx2_done5
CMPQ R11, $0x00
JNE avx2_done6
CMPQ R12, $0x00
JNE avx2_done7
avx2_done0:
TZCNTQ BX, BX
ADDQ BX, AX
SUBQ BX, CX
JMP done
avx2_done1:
ADDQ $0x00000020, AX
SUBQ $0x00000020, CX
TZCNTQ SI, SI
ADDQ SI, AX
SUBQ SI, CX
JMP done
avx2_done2:
ADDQ $0x00000040, AX
SUBQ $0x00000040, CX
TZCNTQ DI, DI
ADDQ DI, AX
SUBQ DI, CX
JMP done
avx2_done3:
ADDQ $0x00000060, AX
SUBQ $0x00000060, CX
TZCNTQ R8, R8
ADDQ R8, AX
SUBQ R8, CX
JMP done
avx2_done4:
ADDQ $0x00000080, AX
SUBQ $0x00000080, CX
TZCNTQ R9, R9
ADDQ R9, AX
SUBQ R9, CX
JMP done
avx2_done5:
ADDQ $0x000000a0, AX
SUBQ $0x000000a0, CX
TZCNTQ R10, R10
ADDQ R10, AX
SUBQ R10, CX
JMP done
avx2_done6:
ADDQ $0x000000c0, AX
SUBQ $0x000000c0, CX
TZCNTQ R11, R11
ADDQ R11, AX
SUBQ R11, CX
JMP done
avx2_done7:
ADDQ $0x000000e0, AX
SUBQ $0x000000e0, CX
TZCNTQ R12, R12
ADDQ R12, AX
SUBQ R12, CX
JMP done
// func indexPair2(b []byte) int
// Requires: AVX, AVX2, BMI
TEXT ·indexPair2(SB), NOSPLIT, $0-32
MOVQ b_base+0(FP), AX
MOVQ b_len+8(FP), CX
MOVQ AX, DX
CMPQ CX, $0x00
JLE fail
SUBQ $0x02, CX
BTL $0x08, github·comâsegmentioâasmâcpu·X86+0(SB)
JCS avx2
tail:
CMPQ CX, $0x00
JE fail
generic:
MOVW (AX), BX
CMPW BX, 2(AX)
JE done
ADDQ $0x02, AX
SUBQ $0x02, CX
CMPQ CX, $0x00
JA generic
fail:
MOVQ $0xffffffffffffffff, AX
MOVQ AX, ret+24(FP)
RET
done:
SUBQ DX, AX
MOVQ AX, ret+24(FP)
RET
avx2:
CMPQ CX, $0x00000102
JB avx2_tail128
XORQ BX, BX
XORQ SI, SI
XORQ DI, DI
XORQ R8, R8
XORQ R9, R9
XORQ R10, R10
XORQ R11, R11
XORQ R12, R12
avx2_loop256:
VMOVDQU (AX), Y0
VMOVDQU 32(AX), Y2
VMOVDQU 64(AX), Y4
VMOVDQU 96(AX), Y6
VMOVDQU 128(AX), Y8
VMOVDQU 160(AX), Y10
VMOVDQU 192(AX), Y12
VMOVDQU 224(AX), Y14
VMOVDQU 2(AX), Y1
VMOVDQU 34(AX), Y3
VMOVDQU 66(AX), Y5
VMOVDQU 98(AX), Y7
VMOVDQU 130(AX), Y9
VMOVDQU 162(AX), Y11
VMOVDQU 194(AX), Y13
VMOVDQU 226(AX), Y15
VPCMPEQW Y0, Y1, Y1
VPCMPEQW Y2, Y3, Y3
VPCMPEQW Y4, Y5, Y5
VPCMPEQW Y6, Y7, Y7
VPCMPEQW Y8, Y9, Y9
VPCMPEQW Y10, Y11, Y11
VPCMPEQW Y12, Y13, Y13
VPCMPEQW Y14, Y15, Y15
VPMOVMSKB Y1, BX
VPMOVMSKB Y3, SI
VPMOVMSKB Y5, DI
VPMOVMSKB Y7, R8
VPMOVMSKB Y9, R9
VPMOVMSKB Y11, R10
VPMOVMSKB Y13, R11
VPMOVMSKB Y15, R12
XORQ R13, R13
ORQ BX, R13
ORQ SI, R13
ORQ DI, R13
ORQ R8, R13
ORQ R9, R13
ORQ R10, R13
ORQ R11, R13
ORQ R12, R13
CMPQ R13, $0x00
JNE avx2_done
ADDQ $0x00000100, AX
SUBQ $0x00000100, CX
CMPQ CX, $0x00000102
JAE avx2_loop256
avx2_tail128:
CMPQ CX, $0x82
JB avx2_tail64
VMOVDQU (AX), Y0
VMOVDQU 32(AX), Y2
VMOVDQU 64(AX), Y4
VMOVDQU 96(AX), Y6
VMOVDQU 2(AX), Y1
VMOVDQU 34(AX), Y3
VMOVDQU 66(AX), Y5
VMOVDQU 98(AX), Y7
VPCMPEQW Y0, Y1, Y1
VPCMPEQW Y2, Y3, Y3
VPCMPEQW Y4, Y5, Y5
VPCMPEQW Y6, Y7, Y7
VPMOVMSKB Y1, BX
VPMOVMSKB Y3, SI
VPMOVMSKB Y5, DI
VPMOVMSKB Y7, R8
XORQ R13, R13
ORQ BX, R13
ORQ SI, R13
ORQ DI, R13
ORQ R8, R13
CMPQ R13, $0x00
JNE avx2_done
ADDQ $0x00000080, AX
SUBQ $0x00000080, CX
avx2_tail64:
CMPQ CX, $0x42
JB avx2_tail32
VMOVDQU (AX), Y0
VMOVDQU 32(AX), Y2
VMOVDQU 2(AX), Y1
VMOVDQU 34(AX), Y3
VPCMPEQW Y0, Y1, Y1
VPCMPEQW Y2, Y3, Y3
VPMOVMSKB Y1, BX
VPMOVMSKB Y3, SI
XORQ R13, R13
ORQ BX, R13
ORQ SI, R13
CMPQ R13, $0x00
JNE avx2_done
ADDQ $0x00000040, AX
SUBQ $0x00000040, CX
avx2_tail32:
CMPQ CX, $0x22
JB avx2_tail16
VMOVDQU (AX), Y0
VMOVDQU 2(AX), Y1
VPCMPEQW Y0, Y1, Y1
VPMOVMSKB Y1, BX
CMPQ BX, $0x00
JNE avx2_done
ADDQ $0x00000020, AX
SUBQ $0x00000020, CX
avx2_tail16:
CMPQ CX, $0x12
JB avx2_tail
VMOVDQU (AX), X0
VMOVDQU 2(AX), X1
VPCMPEQW X0, X1, X1
VPMOVMSKB X1, BX
CMPQ BX, $0x00
JNE avx2_done
ADDQ $0x10, AX
SUBQ $0x10, CX
avx2_tail:
VZEROUPPER
JMP tail
avx2_done:
VZEROUPPER
CMPQ BX, $0x00
JNE avx2_done0
CMPQ SI, $0x00
JNE avx2_done1
CMPQ DI, $0x00
JNE avx2_done2
CMPQ R8, $0x00
JNE avx2_done3
CMPQ R9, $0x00
JNE avx2_done4
CMPQ R10, $0x00
JNE avx2_done5
CMPQ R11, $0x00
JNE avx2_done6
CMPQ R12, $0x00
JNE avx2_done7
avx2_done0:
TZCNTQ BX, BX
ADDQ BX, AX
SUBQ BX, CX
JMP done
avx2_done1:
ADDQ $0x00000020, AX
SUBQ $0x00000020, CX
TZCNTQ SI, SI
ADDQ SI, AX
SUBQ SI, CX
JMP done
avx2_done2:
ADDQ $0x00000040, AX
SUBQ $0x00000040, CX
TZCNTQ DI, DI
ADDQ DI, AX
SUBQ DI, CX
JMP done
avx2_done3:
ADDQ $0x00000060, AX
SUBQ $0x00000060, CX
TZCNTQ R8, R8
ADDQ R8, AX
SUBQ R8, CX
JMP done
avx2_done4:
ADDQ $0x00000080, AX
SUBQ $0x00000080, CX
TZCNTQ R9, R9
ADDQ R9, AX
SUBQ R9, CX
JMP done
avx2_done5:
ADDQ $0x000000a0, AX
SUBQ $0x000000a0, CX
TZCNTQ R10, R10
ADDQ R10, AX
SUBQ R10, CX
JMP done
avx2_done6:
ADDQ $0x000000c0, AX
SUBQ $0x000000c0, CX
TZCNTQ R11, R11
ADDQ R11, AX
SUBQ R11, CX
JMP done
avx2_done7:
ADDQ $0x000000e0, AX
SUBQ $0x000000e0, CX
TZCNTQ R12, R12
ADDQ R12, AX
SUBQ R12, CX
JMP done
// func indexPair4(b []byte) int
// Requires: AVX, AVX2, BMI
TEXT ·indexPair4(SB), NOSPLIT, $0-32
MOVQ b_base+0(FP), AX
MOVQ b_len+8(FP), CX
MOVQ AX, DX
CMPQ CX, $0x00
JLE fail
SUBQ $0x04, CX
BTL $0x08, github·comâsegmentioâasmâcpu·X86+0(SB)
JCS avx2
tail:
CMPQ CX, $0x00
JE fail
generic:
MOVL (AX), BX
CMPL BX, 4(AX)
JE done
ADDQ $0x04, AX
SUBQ $0x04, CX
CMPQ CX, $0x00
JA generic
fail:
MOVQ $0xffffffffffffffff, AX
MOVQ AX, ret+24(FP)
RET
done:
SUBQ DX, AX
MOVQ AX, ret+24(FP)
RET
avx2:
CMPQ CX, $0x00000104
JB avx2_tail128
XORQ BX, BX
XORQ SI, SI
XORQ DI, DI
XORQ R8, R8
XORQ R9, R9
XORQ R10, R10
XORQ R11, R11
XORQ R12, R12
avx2_loop256:
VMOVDQU (AX), Y0
VMOVDQU 32(AX), Y2
VMOVDQU 64(AX), Y4
VMOVDQU 96(AX), Y6
VMOVDQU 128(AX), Y8
VMOVDQU 160(AX), Y10
VMOVDQU 192(AX), Y12
VMOVDQU 224(AX), Y14
VMOVDQU 4(AX), Y1
VMOVDQU 36(AX), Y3
VMOVDQU 68(AX), Y5
VMOVDQU 100(AX), Y7
VMOVDQU 132(AX), Y9
VMOVDQU 164(AX), Y11
VMOVDQU 196(AX), Y13
VMOVDQU 228(AX), Y15
VPCMPEQD Y0, Y1, Y1
VPCMPEQD Y2, Y3, Y3
VPCMPEQD Y4, Y5, Y5
VPCMPEQD Y6, Y7, Y7
VPCMPEQD Y8, Y9, Y9
VPCMPEQD Y10, Y11, Y11
VPCMPEQD Y12, Y13, Y13
VPCMPEQD Y14, Y15, Y15
VPMOVMSKB Y1, BX
VPMOVMSKB Y3, SI
VPMOVMSKB Y5, DI
VPMOVMSKB Y7, R8
VPMOVMSKB Y9, R9
VPMOVMSKB Y11, R10
VPMOVMSKB Y13, R11
VPMOVMSKB Y15, R12
XORQ R13, R13
ORQ BX, R13
ORQ SI, R13
ORQ DI, R13
ORQ R8, R13
ORQ R9, R13
ORQ R10, R13
ORQ R11, R13
ORQ R12, R13
CMPQ R13, $0x00
JNE avx2_done
ADDQ $0x00000100, AX
SUBQ $0x00000100, CX
CMPQ CX, $0x00000104
JAE avx2_loop256
avx2_tail128:
CMPQ CX, $0x84
JB avx2_tail64
VMOVDQU (AX), Y0
VMOVDQU 32(AX), Y2
VMOVDQU 64(AX), Y4
VMOVDQU 96(AX), Y6
VMOVDQU 4(AX), Y1
VMOVDQU 36(AX), Y3
VMOVDQU 68(AX), Y5
VMOVDQU 100(AX), Y7
VPCMPEQD Y0, Y1, Y1
VPCMPEQD Y2, Y3, Y3
VPCMPEQD Y4, Y5, Y5
VPCMPEQD Y6, Y7, Y7
VPMOVMSKB Y1, BX
VPMOVMSKB Y3, SI
VPMOVMSKB Y5, DI
VPMOVMSKB Y7, R8
XORQ R13, R13
ORQ BX, R13
ORQ SI, R13
ORQ DI, R13
ORQ R8, R13
CMPQ R13, $0x00
JNE avx2_done
ADDQ $0x00000080, AX
SUBQ $0x00000080, CX
avx2_tail64:
CMPQ CX, $0x44
JB avx2_tail32
VMOVDQU (AX), Y0
VMOVDQU 32(AX), Y2
VMOVDQU 4(AX), Y1
VMOVDQU 36(AX), Y3
VPCMPEQD Y0, Y1, Y1
VPCMPEQD Y2, Y3, Y3
VPMOVMSKB Y1, BX
VPMOVMSKB Y3, SI
XORQ R13, R13
ORQ BX, R13
ORQ SI, R13
CMPQ R13, $0x00
JNE avx2_done
ADDQ $0x00000040, AX
SUBQ $0x00000040, CX
avx2_tail32:
CMPQ CX, $0x24
JB avx2_tail16
VMOVDQU (AX), Y0
VMOVDQU 4(AX), Y1
VPCMPEQD Y0, Y1, Y1
VPMOVMSKB Y1, BX
CMPQ BX, $0x00
JNE avx2_done
ADDQ $0x00000020, AX
SUBQ $0x00000020, CX
avx2_tail16:
CMPQ CX, $0x14
JB avx2_tail
VMOVDQU (AX), X0
VMOVDQU 4(AX), X1
VPCMPEQD X0, X1, X1
VPMOVMSKB X1, BX
CMPQ BX, $0x00
JNE avx2_done
ADDQ $0x10, AX
SUBQ $0x10, CX
avx2_tail:
VZEROUPPER
JMP tail
avx2_done:
VZEROUPPER
CMPQ BX, $0x00
JNE avx2_done0
CMPQ SI, $0x00
JNE avx2_done1
CMPQ DI, $0x00
JNE avx2_done2
CMPQ R8, $0x00
JNE avx2_done3
CMPQ R9, $0x00
JNE avx2_done4
CMPQ R10, $0x00
JNE avx2_done5
CMPQ R11, $0x00
JNE avx2_done6
CMPQ R12, $0x00
JNE avx2_done7
avx2_done0:
TZCNTQ BX, BX
ADDQ BX, AX
SUBQ BX, CX
JMP done
avx2_done1:
ADDQ $0x00000020, AX
SUBQ $0x00000020, CX
TZCNTQ SI, SI
ADDQ SI, AX
SUBQ SI, CX
JMP done
avx2_done2:
ADDQ $0x00000040, AX
SUBQ $0x00000040, CX
TZCNTQ DI, DI
ADDQ DI, AX
SUBQ DI, CX
JMP done
avx2_done3:
ADDQ $0x00000060, AX
SUBQ $0x00000060, CX
TZCNTQ R8, R8
ADDQ R8, AX
SUBQ R8, CX
JMP done
avx2_done4:
ADDQ $0x00000080, AX
SUBQ $0x00000080, CX
TZCNTQ R9, R9
ADDQ R9, AX
SUBQ R9, CX
JMP done
avx2_done5:
ADDQ $0x000000a0, AX
SUBQ $0x000000a0, CX
TZCNTQ R10, R10
ADDQ R10, AX
SUBQ R10, CX
JMP done
avx2_done6:
ADDQ $0x000000c0, AX
SUBQ $0x000000c0, CX
TZCNTQ R11, R11
ADDQ R11, AX
SUBQ R11, CX
JMP done
avx2_done7:
ADDQ $0x000000e0, AX
SUBQ $0x000000e0, CX
TZCNTQ R12, R12
ADDQ R12, AX
SUBQ R12, CX
JMP done
// func indexPair8(b []byte) int
// Requires: AVX, AVX2, BMI
TEXT ·indexPair8(SB), NOSPLIT, $0-32
MOVQ b_base+0(FP), AX
MOVQ b_len+8(FP), CX
MOVQ AX, DX
CMPQ CX, $0x00
JLE fail
SUBQ $0x08, CX
BTL $0x08, github·comâsegmentioâasmâcpu·X86+0(SB)
JCS avx2
tail:
CMPQ CX, $0x00
JE fail
generic:
MOVQ (AX), BX
CMPQ BX, 8(AX)
JE done
ADDQ $0x08, AX
SUBQ $0x08, CX
CMPQ CX, $0x00
JA generic
fail:
MOVQ $0xffffffffffffffff, AX
MOVQ AX, ret+24(FP)
RET
done:
SUBQ DX, AX
MOVQ AX, ret+24(FP)
RET
avx2:
CMPQ CX, $0x00000108
JB avx2_tail128
XORQ BX, BX
XORQ SI, SI
XORQ DI, DI
XORQ R8, R8
XORQ R9, R9
XORQ R10, R10
XORQ R11, R11
XORQ R12, R12
avx2_loop256:
VMOVDQU (AX), Y0
VMOVDQU 32(AX), Y2
VMOVDQU 64(AX), Y4
VMOVDQU 96(AX), Y6
VMOVDQU 128(AX), Y8
VMOVDQU 160(AX), Y10
VMOVDQU 192(AX), Y12
VMOVDQU 224(AX), Y14
VMOVDQU 8(AX), Y1
VMOVDQU 40(AX), Y3
VMOVDQU 72(AX), Y5
VMOVDQU 104(AX), Y7
VMOVDQU 136(AX), Y9
VMOVDQU 168(AX), Y11
VMOVDQU 200(AX), Y13
VMOVDQU 232(AX), Y15
VPCMPEQQ Y0, Y1, Y1
VPCMPEQQ Y2, Y3, Y3
VPCMPEQQ Y4, Y5, Y5
VPCMPEQQ Y6, Y7, Y7
VPCMPEQQ Y8, Y9, Y9
VPCMPEQQ Y10, Y11, Y11
VPCMPEQQ Y12, Y13, Y13
VPCMPEQQ Y14, Y15, Y15
VPMOVMSKB Y1, BX
VPMOVMSKB Y3, SI
VPMOVMSKB Y5, DI
VPMOVMSKB Y7, R8
VPMOVMSKB Y9, R9
VPMOVMSKB Y11, R10
VPMOVMSKB Y13, R11
VPMOVMSKB Y15, R12
XORQ R13, R13
ORQ BX, R13
ORQ SI, R13
ORQ DI, R13
ORQ R8, R13
ORQ R9, R13
ORQ R10, R13
ORQ R11, R13
ORQ R12, R13
CMPQ R13, $0x00
JNE avx2_done
ADDQ $0x00000100, AX
SUBQ $0x00000100, CX
CMPQ CX, $0x00000108
JAE avx2_loop256
avx2_tail128:
CMPQ CX, $0x88
JB avx2_tail64
VMOVDQU (AX), Y0
VMOVDQU 32(AX), Y2
VMOVDQU 64(AX), Y4
VMOVDQU 96(AX), Y6
VMOVDQU 8(AX), Y1
VMOVDQU 40(AX), Y3
VMOVDQU 72(AX), Y5
VMOVDQU 104(AX), Y7
VPCMPEQQ Y0, Y1, Y1
VPCMPEQQ Y2, Y3, Y3
VPCMPEQQ Y4, Y5, Y5
VPCMPEQQ Y6, Y7, Y7
VPMOVMSKB Y1, BX
VPMOVMSKB Y3, SI
VPMOVMSKB Y5, DI
VPMOVMSKB Y7, R8
XORQ R13, R13
ORQ BX, R13
ORQ SI, R13
ORQ DI, R13
ORQ R8, R13
CMPQ R13, $0x00
JNE avx2_done
ADDQ $0x00000080, AX
SUBQ $0x00000080, CX
avx2_tail64:
CMPQ CX, $0x48
JB avx2_tail32
VMOVDQU (AX), Y0
VMOVDQU 32(AX), Y2
VMOVDQU 8(AX), Y1
VMOVDQU 40(AX), Y3
VPCMPEQQ Y0, Y1, Y1
VPCMPEQQ Y2, Y3, Y3
VPMOVMSKB Y1, BX
VPMOVMSKB Y3, SI
XORQ R13, R13
ORQ BX, R13
ORQ SI, R13
CMPQ R13, $0x00
JNE avx2_done
ADDQ $0x00000040, AX
SUBQ $0x00000040, CX
avx2_tail32:
CMPQ CX, $0x28
JB avx2_tail16
VMOVDQU (AX), Y0
VMOVDQU 8(AX), Y1
VPCMPEQQ Y0, Y1, Y1
VPMOVMSKB Y1, BX
CMPQ BX, $0x00
JNE avx2_done
ADDQ $0x00000020, AX
SUBQ $0x00000020, CX
avx2_tail16:
CMPQ CX, $0x18
JB avx2_tail
VMOVDQU (AX), X0
VMOVDQU 8(AX), X1
VPCMPEQQ X0, X1, X1
VPMOVMSKB X1, BX
CMPQ BX, $0x00
JNE avx2_done
ADDQ $0x10, AX
SUBQ $0x10, CX
avx2_tail:
VZEROUPPER
JMP tail
avx2_done:
VZEROUPPER
CMPQ BX, $0x00
JNE avx2_done0
CMPQ SI, $0x00
JNE avx2_done1
CMPQ DI, $0x00
JNE avx2_done2
CMPQ R8, $0x00
JNE avx2_done3
CMPQ R9, $0x00
JNE avx2_done4
CMPQ R10, $0x00
JNE avx2_done5
CMPQ R11, $0x00
JNE avx2_done6
CMPQ R12, $0x00
JNE avx2_done7
avx2_done0:
TZCNTQ BX, BX
ADDQ BX, AX
SUBQ BX, CX
JMP done
avx2_done1:
ADDQ $0x00000020, AX
SUBQ $0x00000020, CX
TZCNTQ SI, SI
ADDQ SI, AX
SUBQ SI, CX
JMP done
avx2_done2:
ADDQ $0x00000040, AX
SUBQ $0x00000040, CX
TZCNTQ DI, DI
ADDQ DI, AX
SUBQ DI, CX
JMP done
avx2_done3:
ADDQ $0x00000060, AX
SUBQ $0x00000060, CX
TZCNTQ R8, R8
ADDQ R8, AX
SUBQ R8, CX
JMP done
avx2_done4:
ADDQ $0x00000080, AX
SUBQ $0x00000080, CX
TZCNTQ R9, R9
ADDQ R9, AX
SUBQ R9, CX
JMP done
avx2_done5:
ADDQ $0x000000a0, AX
SUBQ $0x000000a0, CX
TZCNTQ R10, R10
ADDQ R10, AX
SUBQ R10, CX
JMP done
avx2_done6:
ADDQ $0x000000c0, AX
SUBQ $0x000000c0, CX
TZCNTQ R11, R11
ADDQ R11, AX
SUBQ R11, CX
JMP done
avx2_done7:
ADDQ $0x000000e0, AX
SUBQ $0x000000e0, CX
TZCNTQ R12, R12
ADDQ R12, AX
SUBQ R12, CX
JMP done
// func indexPair16(b []byte) int
// Requires: AVX, AVX2, BMI, SSE2, SSE4.1
TEXT ·indexPair16(SB), NOSPLIT, $0-32
MOVQ b_base+0(FP), AX
MOVQ b_len+8(FP), CX
MOVQ AX, DX
CMPQ CX, $0x00
JLE fail
SUBQ $0x10, CX
BTL $0x08, github·comâsegmentioâasmâcpu·X86+0(SB)
JCS avx2
tail:
CMPQ CX, $0x00
JE fail
generic:
MOVOU (AX), X0
MOVOU 16(AX), X1
PCMPEQQ X0, X1
PMOVMSKB X1, BX
CMPL BX, $0x0000ffff
JE done
ADDQ $0x10, AX
SUBQ $0x10, CX
CMPQ CX, $0x00
JA generic
fail:
MOVQ $0xffffffffffffffff, AX
MOVQ AX, ret+24(FP)
RET
done:
SUBQ DX, AX
MOVQ AX, ret+24(FP)
RET
avx2:
CMPQ CX, $0x00000110
JB avx2_tail128
XORQ BX, BX
XORQ SI, SI
XORQ DI, DI
XORQ R8, R8
XORQ R9, R9
XORQ R10, R10
XORQ R11, R11
XORQ R12, R12
avx2_loop256:
VMOVDQU (AX), Y0
VMOVDQU 32(AX), Y2
VMOVDQU 64(AX), Y4
VMOVDQU 96(AX), Y6
VMOVDQU 128(AX), Y8
VMOVDQU 160(AX), Y10
VMOVDQU 192(AX), Y12
VMOVDQU 224(AX), Y14
VPERM2I128 $0x21, Y2, Y0, Y1
VPERM2I128 $0x21, Y4, Y2, Y3
VPERM2I128 $0x21, Y6, Y4, Y5
VPERM2I128 $0x21, Y8, Y6, Y7
VPERM2I128 $0x21, Y10, Y8, Y9
VPERM2I128 $0x21, Y12, Y10, Y11
VPERM2I128 $0x21, Y14, Y12, Y13
VMOVDQU 240(AX), Y15
VPCMPEQQ Y0, Y1, Y1
VPCMPEQQ Y2, Y3, Y3
VPCMPEQQ Y4, Y5, Y5
VPCMPEQQ Y6, Y7, Y7
VPCMPEQQ Y8, Y9, Y9
VPCMPEQQ Y10, Y11, Y11
VPCMPEQQ Y12, Y13, Y13
VPCMPEQQ Y14, Y15, Y15
VPERMQ $0xb1, Y1, Y0
VPAND Y1, Y0, Y0
VPMOVMSKB Y0, BX
VPERMQ $0xb1, Y3, Y2
VPAND Y3, Y2, Y2
VPMOVMSKB Y2, SI
VPERMQ $0xb1, Y5, Y4
VPAND Y5, Y4, Y4
VPMOVMSKB Y4, DI
VPERMQ $0xb1, Y7, Y6
VPAND Y7, Y6, Y6
VPMOVMSKB Y6, R8
VPERMQ $0xb1, Y9, Y8
VPAND Y9, Y8, Y8
VPMOVMSKB Y8, R9
VPERMQ $0xb1, Y11, Y10
VPAND Y11, Y10, Y10
VPMOVMSKB Y10, R10
VPERMQ $0xb1, Y13, Y12
VPAND Y13, Y12, Y12
VPMOVMSKB Y12, R11
VPERMQ $0xb1, Y15, Y14
VPAND Y15, Y14, Y14
VPMOVMSKB Y14, R12
XORQ R13, R13
ORQ BX, R13
ORQ SI, R13
ORQ DI, R13
ORQ R8, R13
ORQ R9, R13
ORQ R10, R13
ORQ R11, R13
ORQ R12, R13
CMPQ R13, $0x00
JNE avx2_done
ADDQ $0x00000100, AX
SUBQ $0x00000100, CX
CMPQ CX, $0x00000110
JAE avx2_loop256
avx2_tail128:
CMPQ CX, $0x90
JB avx2_tail64
VMOVDQU (AX), Y0
VMOVDQU 32(AX), Y2
VMOVDQU 64(AX), Y4
VMOVDQU 96(AX), Y6
VPERM2I128 $0x21, Y2, Y0, Y1
VPERM2I128 $0x21, Y4, Y2, Y3
VPERM2I128 $0x21, Y6, Y4, Y5
VMOVDQU 112(AX), Y7
VPCMPEQQ Y0, Y1, Y1
VPCMPEQQ Y2, Y3, Y3
VPCMPEQQ Y4, Y5, Y5
VPCMPEQQ Y6, Y7, Y7
VPERMQ $0xb1, Y1, Y0
VPAND Y1, Y0, Y0
VPMOVMSKB Y0, BX
VPERMQ $0xb1, Y3, Y2
VPAND Y3, Y2, Y2
VPMOVMSKB Y2, SI
VPERMQ $0xb1, Y5, Y4
VPAND Y5, Y4, Y4
VPMOVMSKB Y4, DI
VPERMQ $0xb1, Y7, Y6
VPAND Y7, Y6, Y6
VPMOVMSKB Y6, R8
XORQ R13, R13
ORQ BX, R13
ORQ SI, R13
ORQ DI, R13
ORQ R8, R13
CMPQ R13, $0x00
JNE avx2_done
ADDQ $0x00000080, AX
SUBQ $0x00000080, CX
avx2_tail64:
CMPQ CX, $0x50
JB avx2_tail32
VMOVDQU (AX), Y0
VMOVDQU 32(AX), Y2
VPERM2I128 $0x21, Y2, Y0, Y1
VMOVDQU 48(AX), Y3
VPCMPEQQ Y0, Y1, Y1
VPCMPEQQ Y2, Y3, Y3
VPERMQ $0xb1, Y1, Y0
VPAND Y1, Y0, Y0
VPMOVMSKB Y0, BX
VPERMQ $0xb1, Y3, Y2
VPAND Y3, Y2, Y2
VPMOVMSKB Y2, SI
XORQ R13, R13
ORQ BX, R13
ORQ SI, R13
CMPQ R13, $0x00
JNE avx2_done
ADDQ $0x00000040, AX
SUBQ $0x00000040, CX
avx2_tail32:
CMPQ CX, $0x30
JB avx2_tail16
VMOVDQU (AX), Y0
VMOVDQU 16(AX), Y1
VPCMPEQQ Y0, Y1, Y1
VPERMQ $0xb1, Y1, Y0
VPAND Y1, Y0, Y0
VPMOVMSKB Y0, BX
CMPQ BX, $0x00
JNE avx2_done
ADDQ $0x00000020, AX
SUBQ $0x00000020, CX
avx2_tail16:
VZEROUPPER
JMP tail
avx2_done:
VZEROUPPER
CMPQ BX, $0x00
JNE avx2_done0
CMPQ SI, $0x00
JNE avx2_done1
CMPQ DI, $0x00
JNE avx2_done2
CMPQ R8, $0x00
JNE avx2_done3
CMPQ R9, $0x00
JNE avx2_done4
CMPQ R10, $0x00
JNE avx2_done5
CMPQ R11, $0x00
JNE avx2_done6
CMPQ R12, $0x00
JNE avx2_done7
avx2_done0:
TZCNTQ BX, BX
ADDQ BX, AX
SUBQ BX, CX
JMP done
avx2_done1:
ADDQ $0x00000020, AX
SUBQ $0x00000020, CX
TZCNTQ SI, SI
ADDQ SI, AX
SUBQ SI, CX
JMP done
avx2_done2:
ADDQ $0x00000040, AX
SUBQ $0x00000040, CX
TZCNTQ DI, DI
ADDQ DI, AX
SUBQ DI, CX
JMP done
avx2_done3:
ADDQ $0x00000060, AX
SUBQ $0x00000060, CX
TZCNTQ R8, R8
ADDQ R8, AX
SUBQ R8, CX
JMP done
avx2_done4:
ADDQ $0x00000080, AX
SUBQ $0x00000080, CX
TZCNTQ R9, R9
ADDQ R9, AX
SUBQ R9, CX
JMP done
avx2_done5:
ADDQ $0x000000a0, AX
SUBQ $0x000000a0, CX
TZCNTQ R10, R10
ADDQ R10, AX
SUBQ R10, CX
JMP done
avx2_done6:
ADDQ $0x000000c0, AX
SUBQ $0x000000c0, CX
TZCNTQ R11, R11
ADDQ R11, AX
SUBQ R11, CX
JMP done
avx2_done7:
ADDQ $0x000000e0, AX
SUBQ $0x000000e0, CX
TZCNTQ R12, R12
ADDQ R12, AX
SUBQ R12, CX
JMP done
// func indexPair32(b []byte) int
// Requires: AVX, AVX2, BMI, CMOV, SSE2, SSE4.1
TEXT ·indexPair32(SB), NOSPLIT, $0-32
MOVQ b_base+0(FP), AX
MOVQ b_len+8(FP), CX
MOVQ AX, DX
CMPQ CX, $0x00
JLE fail
SUBQ $0x20, CX
BTL $0x08, github·comâsegmentioâasmâcpu·X86+0(SB)
JCS avx2
tail:
CMPQ CX, $0x00
JE fail
generic:
MOVOU (AX), X0
MOVOU 16(AX), X1
MOVOU 32(AX), X2
MOVOU 48(AX), X3
PCMPEQQ X0, X2
PCMPEQQ X1, X3
PMOVMSKB X2, BX
PMOVMSKB X3, SI
ANDL SI, BX
CMPL BX, $0x0000ffff
JE done
ADDQ $0x20, AX
SUBQ $0x20, CX
CMPQ CX, $0x00
JA generic
fail:
MOVQ $0xffffffffffffffff, AX
MOVQ AX, ret+24(FP)
RET
done:
SUBQ DX, AX
MOVQ AX, ret+24(FP)
RET
avx2:
CMPQ CX, $0x00000120
JB avx2_tail128
XORQ BX, BX
XORQ SI, SI
XORQ DI, DI
XORQ R8, R8
XORQ R9, R9
XORQ R10, R10
XORQ R11, R11
XORQ R12, R12
avx2_loop256:
XORQ R13, R13
VMOVDQU (AX), Y0
VMOVDQU 32(AX), Y2
VMOVDQU 64(AX), Y4
VMOVDQU 96(AX), Y5
VMOVDQU 128(AX), Y7
VMOVDQU 160(AX), Y8
VMOVDQU 192(AX), Y9
VMOVDQU 224(AX), Y10
VMOVDQU 256(AX), Y11
VPCMPEQQ Y0, Y2, Y1
VPCMPEQQ Y2, Y4, Y3
VPCMPEQQ Y4, Y5, Y0
VPCMPEQQ Y5, Y7, Y6
VPCMPEQQ Y7, Y8, Y2
VPCMPEQQ Y8, Y9, Y4
VPCMPEQQ Y9, Y10, Y5
VPCMPEQQ Y10, Y11, Y11
VPMOVMSKB Y1, BX
CMPL BX, $0xffffffff
CMOVLNE R13, BX
VPMOVMSKB Y3, SI
CMPL SI, $0xffffffff
CMOVLNE R13, SI
VPMOVMSKB Y0, DI
CMPL DI, $0xffffffff
CMOVLNE R13, DI
VPMOVMSKB Y6, R8
CMPL R8, $0xffffffff
CMOVLNE R13, R8
VPMOVMSKB Y2, R9
CMPL R9, $0xffffffff
CMOVLNE R13, R9
VPMOVMSKB Y4, R10
CMPL R10, $0xffffffff
CMOVLNE R13, R10
VPMOVMSKB Y5, R11
CMPL R11, $0xffffffff
CMOVLNE R13, R11
VPMOVMSKB Y11, R12
CMPL R12, $0xffffffff
CMOVLNE R13, R12
XORQ R13, R13
ORQ BX, R13
ORQ SI, R13
ORQ DI, R13
ORQ R8, R13
ORQ R9, R13
ORQ R10, R13
ORQ R11, R13
ORQ R12, R13
CMPQ R13, $0x00
JNE avx2_done
ADDQ $0x00000100, AX
SUBQ $0x00000100, CX
CMPQ CX, $0x00000120
JAE avx2_loop256
avx2_tail128:
CMPQ CX, $0xa0
JB avx2_tail64
XORQ R13, R13
VMOVDQU (AX), Y0
VMOVDQU 32(AX), Y2
VMOVDQU 64(AX), Y4
VMOVDQU 96(AX), Y5
VMOVDQU 128(AX), Y6
VPCMPEQQ Y0, Y2, Y1
VPCMPEQQ Y2, Y4, Y3
VPCMPEQQ Y4, Y5, Y0
VPCMPEQQ Y5, Y6, Y6
VPMOVMSKB Y1, BX
CMPL BX, $0xffffffff
CMOVLNE R13, BX
VPMOVMSKB Y3, SI
CMPL SI, $0xffffffff
CMOVLNE R13, SI
VPMOVMSKB Y0, DI
CMPL DI, $0xffffffff
CMOVLNE R13, DI
VPMOVMSKB Y6, R8
CMPL R8, $0xffffffff
CMOVLNE R13, R8
XORQ R13, R13
ORQ BX, R13
ORQ SI, R13
ORQ DI, R13
ORQ R8, R13
CMPQ R13, $0x00
JNE avx2_done
ADDQ $0x00000080, AX
SUBQ $0x00000080, CX
avx2_tail64:
CMPQ CX, $0x60
JB avx2_tail32
XORQ R13, R13
VMOVDQU (AX), Y0
VMOVDQU 32(AX), Y2
VMOVDQU 64(AX), Y3
VPCMPEQQ Y0, Y2, Y1
VPCMPEQQ Y2, Y3, Y3
VPMOVMSKB Y1, BX
CMPL BX, $0xffffffff
CMOVLNE R13, BX
VPMOVMSKB Y3, SI
CMPL SI, $0xffffffff
CMOVLNE R13, SI
XORQ R13, R13
ORQ BX, R13
ORQ SI, R13
CMPQ R13, $0x00
JNE avx2_done
ADDQ $0x00000040, AX
SUBQ $0x00000040, CX
avx2_tail32:
CMPQ CX, $0x40
JB avx2_tail16
XORQ R13, R13
VMOVDQU (AX), Y0
VMOVDQU 32(AX), Y1
VPCMPEQQ Y0, Y1, Y1
VPMOVMSKB Y1, BX
CMPL BX, $0xffffffff
CMOVLNE R13, BX
CMPQ BX, $0x00
JNE avx2_done
ADDQ $0x00000020, AX
SUBQ $0x00000020, CX
avx2_tail16:
VZEROUPPER
JMP tail
avx2_done:
VZEROUPPER
CMPQ BX, $0x00
JNE avx2_done0
CMPQ SI, $0x00
JNE avx2_done1
CMPQ DI, $0x00
JNE avx2_done2
CMPQ R8, $0x00
JNE avx2_done3
CMPQ R9, $0x00
JNE avx2_done4
CMPQ R10, $0x00
JNE avx2_done5
CMPQ R11, $0x00
JNE avx2_done6
CMPQ R12, $0x00
JNE avx2_done7
avx2_done0:
TZCNTQ BX, BX
ADDQ BX, AX
SUBQ BX, CX
JMP done
avx2_done1:
ADDQ $0x00000020, AX
SUBQ $0x00000020, CX
TZCNTQ SI, SI
ADDQ SI, AX
SUBQ SI, CX
JMP done
avx2_done2:
ADDQ $0x00000040, AX
SUBQ $0x00000040, CX
TZCNTQ DI, DI
ADDQ DI, AX
SUBQ DI, CX
JMP done
avx2_done3:
ADDQ $0x00000060, AX
SUBQ $0x00000060, CX
TZCNTQ R8, R8
ADDQ R8, AX
SUBQ R8, CX
JMP done
avx2_done4:
ADDQ $0x00000080, AX
SUBQ $0x00000080, CX
TZCNTQ R9, R9
ADDQ R9, AX
SUBQ R9, CX
JMP done
avx2_done5:
ADDQ $0x000000a0, AX
SUBQ $0x000000a0, CX
TZCNTQ R10, R10
ADDQ R10, AX
SUBQ R10, CX
JMP done
avx2_done6:
ADDQ $0x000000c0, AX
SUBQ $0x000000c0, CX
TZCNTQ R11, R11
ADDQ R11, AX
SUBQ R11, CX
JMP done
avx2_done7:
ADDQ $0x000000e0, AX
SUBQ $0x000000e0, CX
TZCNTQ R12, R12
ADDQ R12, AX
SUBQ R12, CX
JMP done