llvm.org GIT mirror llvm / 4858c15
[AArch64] Assembler support for the ARMv8.2a dot product instructions Dot product is an optional ARMv8.2a extension, see also the public architecture specification here: https://developer.arm.com/products/architecture/a-profile/exploration-tools. This patch adds AArch64 assembler support for these dot product instructions. Differential Revision: https://reviews.llvm.org/D36515 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@310480 91177308-0d34-0410-b5e6-96231b3b80d8 Sjoerd Meijer 2 years ago
11 changed file(s) with 158 addition(s) and 27 deletion(s). Raw diff Collapse all Expand all
2727 AARCH64_ARCH("armv8.2-a", ARMV8_2A, "8.2-A", "v8.2a",
2828 ARMBuildAttrs::CPUArch::v8_A, FK_CRYPTO_NEON_FP_ARMV8,
2929 (AArch64::AEK_CRC | AArch64::AEK_CRYPTO | AArch64::AEK_FP |
30 AArch64::AEK_SIMD | AArch64::AEK_RAS | AArch64::AEK_LSE))
30 AArch64::AEK_SIMD | AArch64::AEK_RAS | AArch64::AEK_LSE |
31 AArch64::AEK_DOTPROD))
3132 #undef AARCH64_ARCH
3233
3334 #ifndef AARCH64_ARCH_EXT_NAME
3940 AARCH64_ARCH_EXT_NAME("crc", AArch64::AEK_CRC, "+crc", "-crc")
4041 AARCH64_ARCH_EXT_NAME("lse", AArch64::AEK_LSE, "+lse", "-lse")
4142 AARCH64_ARCH_EXT_NAME("crypto", AArch64::AEK_CRYPTO, "+crypto","-crypto")
43 AARCH64_ARCH_EXT_NAME("dotprod", AArch64::AEK_DOTPROD, "+dotprod","-dotprod")
4244 AARCH64_ARCH_EXT_NAME("fp", AArch64::AEK_FP, "+fp-armv8", "-fp-armv8")
4345 AARCH64_ARCH_EXT_NAME("simd", AArch64::AEK_SIMD, "+neon", "-neon")
4446 AARCH64_ARCH_EXT_NAME("fp16", AArch64::AEK_FP16, "+fullfp16", "-fullfp16")
165165 AEK_PROFILE = 0x40,
166166 AEK_RAS = 0x80,
167167 AEK_LSE = 0x100,
168 AEK_SVE = 0x200
168 AEK_SVE = 0x200,
169 AEK_DOTPROD = 0x400
169170 };
170171
171172 StringRef getCanonicalArchName(StringRef Arch);
120120 def FeatureUseRSqrt : SubtargetFeature<
121121 "use-reciprocal-square-root", "UseRSqrt", "true",
122122 "Use the reciprocal square root approximation">;
123
124 def FeatureDotProd : SubtargetFeature<
125 "dotprod", "HasDotProd", "true",
126 "Enable dot product support">;
123127
124128 def FeatureNoNegativeImmediates : SubtargetFeature<"no-neg-immediates",
125129 "NegativeImmediates", "false",
43714371 let Inst{10} = 1;
43724372 let Inst{9-5} = Rn;
43734373 let Inst{4-0} = Rd;
4374 }
4375
4376 class BaseSIMDThreeSameVectorDot
4377 string kind2> :
4378 BaseSIMDThreeSameVector {
4379 let AsmString = !strconcat(asm, "{\t$Rd" # kind1 # ", $Rn" # kind2 # ", $Rm" # kind2 # "}");
43744380 }
43754381
43764382 // All operand sizes distinguished in the encoding.
68006806 let Inst{4-0} = Rd;
68016807 }
68026808
6809 // ARMv8.2 Index Dot product instructions
6810 class BaseSIMDThreeSameVectorDotIndex
6811 string lhs_kind, string rhs_kind> :
6812 BaseSIMDIndexedTied
6813 asm, "", dst_kind, lhs_kind, rhs_kind, []> {
6814 bits<2> idx;
6815 let Inst{21} = idx{0}; // L
6816 let Inst{11} = idx{1}; // H
6817 }
6818
68036819 multiclass SIMDFPIndexed opc, string asm,
68046820 SDPatternOperator OpNode> {
68056821 let Predicates = [HasNEON, HasFullFP16] in {
95959611
95969612 //----------------------------------------------------------------------------
95979613 // Allow the size specifier tokens to be upper case, not just lower.
9614 def : TokenAlias<".4B", ".4b">; // Add dot product
95989615 def : TokenAlias<".8B", ".8b">;
95999616 def : TokenAlias<".4H", ".4h">;
96009617 def : TokenAlias<".2S", ".2s">;
2323 AssemblerPredicate<"FeatureNEON", "neon">;
2424 def HasCrypto : Predicate<"Subtarget->hasCrypto()">,
2525 AssemblerPredicate<"FeatureCrypto", "crypto">;
26 def HasDotProd : Predicate<"Subtarget->hasDotProd()">,
27 AssemblerPredicate<"FeatureDotProd", "dotprod">;
2628 def HasCRC : Predicate<"Subtarget->hasCRC()">,
2729 AssemblerPredicate<"FeatureCRC", "crc">;
2830 def HasLSE : Predicate<"Subtarget->hasLSE()">,
429431
430432 def ISB : CRmSystemI
431433 [(int_aarch64_isb (i32 imm32_0_15:$CRm))]>;
434 }
435
436 // ARMv8.2 Dot Product
437 let Predicates = [HasDotProd] in {
438 def UDOT2S : BaseSIMDThreeSameVectorDot<0, 1, "udot", ".2s", ".8b">;
439 def SDOT2S : BaseSIMDThreeSameVectorDot<0, 0, "sdot", ".2s", ".8b">;
440 def UDOT4S : BaseSIMDThreeSameVectorDot<1, 1, "udot", ".4s", ".16b">;
441 def SDOT4S : BaseSIMDThreeSameVectorDot<1, 0, "sdot", ".4s", ".16b">;
442 def UDOTIDX2S : BaseSIMDThreeSameVectorDotIndex<0, 1, "udot", ".2s", ".8b", ".4b">;
443 def SDOTIDX2S : BaseSIMDThreeSameVectorDotIndex<0, 0, "sdot", ".2s", ".8b", ".4b">;
444 def UDOTIDX4S : BaseSIMDThreeSameVectorDotIndex<1, 1, "udot", ".4s", ".16b", ".4b">;
445 def SDOTIDX4S : BaseSIMDThreeSameVectorDotIndex<1, 0, "sdot", ".4s", ".16b", ".4b">;
432446 }
433447
434448 def : InstAlias<"clrex", (CLREX 0xf)>;
6161 bool HasFPARMv8 = false;
6262 bool HasNEON = false;
6363 bool HasCrypto = false;
64 bool HasDotProd = false;
6465 bool HasCRC = false;
6566 bool HasLSE = false;
6667 bool HasRAS = false;
200201 bool hasFPARMv8() const { return HasFPARMv8; }
201202 bool hasNEON() const { return HasNEON; }
202203 bool hasCrypto() const { return HasCrypto; }
204 bool hasDotProd() const { return HasDotProd; }
203205 bool hasCRC() const { return HasCRC; }
204206 bool hasLSE() const { return HasLSE; }
205207 bool hasRAS() const { return HasRAS; }
18091809 .Case(".d", true)
18101810 // Needed for fp16 scalar pairwise reductions
18111811 .Case(".2h", true)
1812 // another special case for the ARMv8.2a dot product operand
1813 .Case(".4b", true)
18121814 .Default(false);
18131815 }
18141816
0 // RUN: not llvm-mc -triple aarch64 -mattr=+dotprod -show-encoding < %s 2> %t
1 // RUN: FileCheck --check-prefix=CHECK-ERROR < %t %s
2
3 udot v0.2s, v1.8b, v2.4b[4]
4 sdot v0.2s, v1.8b, v2.4b[4]
5 udot v0.4s, v1.16b, v2.4b[4]
6 sdot v0.4s, v1.16b, v2.4b[4]
7
8 // CHECK-ERROR: vector lane must be an integer in range [0, 3]
9 // CHECK-ERROR: vector lane must be an integer in range [0, 3]
10 // CHECK-ERROR: vector lane must be an integer in range [0, 3]
11 // CHECK-ERROR: vector lane must be an integer in range [0, 3]
0 // RUN: llvm-mc -triple aarch64 -mattr=+dotprod -show-encoding < %s | FileCheck %s --check-prefix=CHECK-DOTPROD
1 // RUN: not llvm-mc -triple aarch64 -mattr=+v8.2a -show-encoding < %s 2> %t
2 // RUN: FileCheck --check-prefix=CHECK-NO-DOTPROD < %t %s
3
4 udot v0.2s, v1.8b, v2.8b
5 sdot v0.2s, v1.8b, v2.8b
6 udot v0.4s, v1.16b, v2.16b
7 sdot v0.4s, v1.16b, v2.16b
8 udot v0.2s, v1.8b, v2.4b[0]
9 sdot v0.2s, v1.8b, v2.4b[1]
10 udot v0.4s, v1.16b, v2.4b[2]
11 sdot v0.4s, v1.16b, v2.4b[3]
12
13 // Check that the upper case types are aliases
14 udot v0.2S, v1.8B, v2.4B[0]
15 udot v0.4S, v1.16B, v2.4B[2]
16
17 // CHECK-DOTPROD: udot v0.2s, v1.8b, v2.8b // encoding: [0x20,0x94,0x82,0x2e]
18 // CHECK-DOTPROD: sdot v0.2s, v1.8b, v2.8b // encoding: [0x20,0x94,0x82,0x0e]
19 // CHECK-DOTPROD: udot v0.4s, v1.16b, v2.16b // encoding: [0x20,0x94,0x82,0x6e]
20 // CHECK-DOTPROD: sdot v0.4s, v1.16b, v2.16b // encoding: [0x20,0x94,0x82,0x4e]
21 // CHECK-DOTPROD: udot v0.2s, v1.8b, v2.4b[0] // encoding: [0x20,0xe0,0x82,0x2f]
22 // CHECK-DOTPROD: sdot v0.2s, v1.8b, v2.4b[1] // encoding: [0x20,0xe0,0xa2,0x0f]
23 // CHECK-DOTPROD: udot v0.4s, v1.16b, v2.4b[2] // encoding: [0x20,0xe8,0x82,0x6f]
24 // CHECK-DOTPROD: sdot v0.4s, v1.16b, v2.4b[3] // encoding: [0x20,0xe8,0xa2,0x4f]
25
26 // CHECK-DOTPROD: udot v0.2s, v1.8b, v2.4b[0] // encoding: [0x20,0xe0,0x82,0x2f]
27 // CHECK-DOTPROD: udot v0.4s, v1.16b, v2.4b[2] // encoding: [0x20,0xe8,0x82,0x6f]
28
29 // CHECK-NO-DOTPROD: error: instruction requires: dotprod
30 // CHECK-NO-DOTPROD: udot v0.2s, v1.8b, v2.8b
31 // CHECK-NO-DOTPROD: ^
32 // CHECK-NO-DOTPROD: error: instruction requires: dotprod
33 // CHECK-NO-DOTPROD: sdot v0.2s, v1.8b, v2.8b
34 // CHECK-NO-DOTPROD: ^
35 // CHECK-NO-DOTPROD: error: instruction requires: dotprod
36 // CHECK-NO-DOTPROD: udot v0.4s, v1.16b, v2.16b
37 // CHECK-NO-DOTPROD: ^
38 // CHECK-NO-DOTPROD: error: instruction requires: dotprod
39 // CHECK-NO-DOTPROD: sdot v0.4s, v1.16b, v2.16b
40 // CHECK-NO-DOTPROD: ^
41 // CHECK-NO-DOTPROD: error: instruction requires: dotprod
42 // CHECK-NO-DOTPROD: udot v0.2s, v1.8b, v2.4b[0]
43 // CHECK-NO-DOTPROD: ^
44 // CHECK-NO-DOTPROD: error: instruction requires: dotprod
45 // CHECK-NO-DOTPROD: sdot v0.2s, v1.8b, v2.4b[1]
46 // CHECK-NO-DOTPROD: ^
47 // CHECK-NO-DOTPROD: error: instruction requires: dotprod
48 // CHECK-NO-DOTPROD: udot v0.4s, v1.16b, v2.4b[2]
49 // CHECK-NO-DOTPROD: ^
50 // CHECK-NO-DOTPROD: error: instruction requires: dotprod
51 // CHECK-NO-DOTPROD: sdot v0.4s, v1.16b, v2.4b[3]
52 // CHECK-NO-DOTPROD: ^
53
54 // CHECK-NO-DOTPROD: error: instruction requires: dotprod
55 // CHECK-NO-DOTPROD: udot v0.2S, v1.8B, v2.4B[0]
56 // CHECK-NO-DOTPROD: ^
57 // CHECK-NO-DOTPROD: error: instruction requires: dotprod
58 // CHECK-NO-DOTPROD: udot v0.4S, v1.16B, v2.4B[2]
59 // CHECK-NO-DOTPROD: ^
63946394 uzp1 v0.16b, v1.8b, v2.8b
63956395 // CHECK-ERROR: [[@LINE-1]]:22: error: invalid operand for instruction
63966396 uzp1 v0.8b, v1.4b, v2.4b
6397 // CHECK-ERROR: [[@LINE-1]]:21: error: invalid vector kind qualifier
6398 // CHECK-ERROR: [[@LINE-2]]:28: error: invalid vector kind qualifier
6397 // CHECK-ERROR: [[@LINE-1]]:21: error: invalid operand for instruction
63996398 uzp1 v0.8h, v1.4h, v2.4h
64006399 // CHECK-ERROR: [[@LINE-1]]:21: error: invalid operand for instruction
64016400 uzp1 v0.4h, v1.2h, v2.2h
64156414 uzp2 v0.16b, v1.8b, v2.8b
64166415 // CHECK-ERROR: [[@LINE-1]]:22: error: invalid operand for instruction
64176416 uzp2 v0.8b, v1.4b, v2.4b
6418 // CHECK-ERROR: [[@LINE-1]]:21: error: invalid vector kind qualifier
6419 // CHECK-ERROR: [[@LINE-2]]:28: error: invalid vector kind qualifier
6417 // CHECK-ERROR: [[@LINE-1]]:21: error: invalid operand for instruction
64206418 uzp2 v0.8h, v1.4h, v2.4h
64216419 // CHECK-ERROR: [[@LINE-1]]:21: error: invalid operand for instruction
64226420 uzp2 v0.4h, v1.2h, v2.2h
64366434 zip1 v0.16b, v1.8b, v2.8b
64376435 // CHECK-ERROR: [[@LINE-1]]:22: error: invalid operand for instruction
64386436 zip1 v0.8b, v1.4b, v2.4b
6439 // CHECK-ERROR: [[@LINE-1]]:21: error: invalid vector kind qualifier
6440 // CHECK-ERROR: [[@LINE-2]]:28: error: invalid vector kind qualifier
6437 // CHECK-ERROR: [[@LINE-1]]:21: error: invalid operand for instruction
64416438 zip1 v0.8h, v1.4h, v2.4h
64426439 // CHECK-ERROR: [[@LINE-1]]:21: error: invalid operand for instruction
64436440 zip1 v0.4h, v1.2h, v2.2h
64536450 // CHECK-ERROR: [[@LINE-1]]:14: error: invalid operand for instruction
64546451
64556452
6456 \
6453
64576454 zip2 v0.16b, v1.8b, v2.8b
64586455 // CHECK-ERROR: [[@LINE-1]]:22: error: invalid operand for instruction
64596456 zip2 v0.8b, v1.4b, v2.4b
6460 // CHECK-ERROR: [[@LINE-1]]:21: error: invalid vector kind qualifier
6461 // CHECK-ERROR: [[@LINE-2]]:28: error: invalid vector kind qualifier
6457 // CHECK-ERROR: [[@LINE-1]]:21: error: invalid operand for instruction
64626458 zip2 v0.8h, v1.4h, v2.4h
64636459 // CHECK-ERROR: [[@LINE-1]]:21: error: invalid operand for instruction
64646460 zip2 v0.4h, v1.2h, v2.2h
64786474 trn1 v0.16b, v1.8b, v2.8b
64796475 // CHECK-ERROR: [[@LINE-1]]:22: error: invalid operand for instruction
64806476 trn1 v0.8b, v1.4b, v2.4b
6481 // CHECK-ERROR: [[@LINE-1]]:21: error: invalid vector kind qualifier
6482 // CHECK-ERROR: [[@LINE-2]]:28: error: invalid vector kind qualifier
6477 // CHECK-ERROR: [[@LINE-1]]:21: error: invalid operand for instruction
64836478 trn1 v0.8h, v1.4h, v2.4h
64846479 // CHECK-ERROR: [[@LINE-1]]:21: error: invalid operand for instruction
64856480 trn1 v0.4h, v1.2h, v2.2h
64996494 trn2 v0.16b, v1.8b, v2.8b
65006495 // CHECK-ERROR: [[@LINE-1]]:22: error: invalid operand for instruction
65016496 trn2 v0.8b, v1.4b, v2.4b
6502 // CHECK-ERROR: [[@LINE-1]]:21: error: invalid vector kind qualifier
6503 // CHECK-ERROR: [[@LINE-2]]:28: error: invalid vector kind qualifier
6497 // CHECK-ERROR: [[@LINE-1]]:21: error: invalid operand for instruction
65046498 trn2 v0.8h, v1.4h, v2.4h
65056499 // CHECK-ERROR: [[@LINE-1]]:21: error: invalid operand for instruction
65066500 trn2 v0.4h, v1.2h, v2.2h
65226516 uzp1 v0.16b, v1.8b, v2.8b
65236517 // CHECK-ERROR: [[@LINE-1]]:22: error: invalid operand for instruction
65246518 uzp1 v0.8b, v1.4b, v2.4b
6525 // CHECK-ERROR: [[@LINE-1]]:21: error: invalid vector kind qualifier
6526 // CHECK-ERROR: [[@LINE-2]]:28: error: invalid vector kind qualifier
6519 // CHECK-ERROR: [[@LINE-1]]:21: error: invalid operand for instruction
65276520 uzp1 v0.8h, v1.4h, v2.4h
65286521 // CHECK-ERROR: [[@LINE-1]]:21: error: invalid operand for instruction
65296522 uzp1 v0.4h, v1.2h, v2.2h
65416534 uzp2 v0.16b, v1.8b, v2.8b
65426535 // CHECK-ERROR: [[@LINE-1]]:22: error: invalid operand for instruction
65436536 uzp2 v0.8b, v1.4b, v2.4b
6544 // CHECK-ERROR: [[@LINE-1]]:21: error: invalid vector kind qualifier
6545 // CHECK-ERROR: [[@LINE-2]]:28: error: invalid vector kind qualifier
6537 // CHECK-ERROR: [[@LINE-1]]:21: error: invalid operand for instruction
65466538 uzp2 v0.8h, v1.4h, v2.4h
65476539 // CHECK-ERROR: [[@LINE-1]]:21: error: invalid operand for instruction
65486540 uzp2 v0.4h, v1.2h, v2.2h
65606552 zip1 v0.16b, v1.8b, v2.8b
65616553 // CHECK-ERROR: [[@LINE-1]]:22: error: invalid operand for instruction
65626554 zip1 v0.8b, v1.4b, v2.4b
6563 // CHECK-ERROR: [[@LINE-1]]:21: error: invalid vector kind qualifier
6564 // CHECK-ERROR: [[@LINE-2]]:28: error: invalid vector kind qualifier
6555 // CHECK-ERROR: [[@LINE-1]]:21: error: invalid operand for instruction
65656556 zip1 v0.8h, v1.4h, v2.4h
65666557 // CHECK-ERROR: [[@LINE-1]]:21: error: invalid operand for instruction
65676558 zip1 v0.4h, v1.2h, v2.2h
65836574 zip2 v0.16b, v1.8b, v2.8b
65846575 // CHECK-ERROR: [[@LINE-1]]:22: error: invalid operand for instruction
65856576 zip2 v0.8b, v1.4b, v2.4b
6586 // CHECK-ERROR: [[@LINE-1]]:21: error: invalid vector kind qualifier
6587 // CHECK-ERROR: [[@LINE-2]]:28: error: invalid vector kind qualifier
6577 // CHECK-ERROR: [[@LINE-1]]:21: error: invalid operand for instruction
65886578 zip2 v0.8h, v1.4h, v2.4h
65896579 // CHECK-ERROR: [[@LINE-1]]:21: error: invalid operand for instruction
65906580 zip2 v0.4h, v1.2h, v2.2h
66056595 trn1 v0.16b, v1.8b, v2.8b
66066596 // CHECK-ERROR: [[@LINE-1]]:22: error: invalid operand for instruction
66076597 trn1 v0.8b, v1.4b, v2.4b
6608 // CHECK-ERROR: [[@LINE-1]]:21: error: invalid vector kind qualifier
6609 // CHECK-ERROR: [[@LINE-2]]:28: error: invalid vector kind qualifier
6598 // CHECK-ERROR: [[@LINE-1]]:21: error: invalid operand for instruction
66106599 trn1 v0.8h, v1.4h, v2.4h
66116600 // CHECK-ERROR: [[@LINE-1]]:21: error: invalid operand for instruction
66126601 trn1 v0.4h, v1.2h, v2.2h
66266615 trn2 v0.16b, v1.8b, v2.8b
66276616 // CHECK-ERROR: [[@LINE-1]]:22: error: invalid operand for instruction
66286617 trn2 v0.8b, v1.4b, v2.4b
6629 // CHECK-ERROR: [[@LINE-1]]:21: error: invalid vector kind qualifier
6630 // CHECK-ERROR: [[@LINE-2]]:28: error: invalid vector kind qualifier
6618 // CHECK-ERROR: [[@LINE-1]]:21: error: invalid operand for instruction
66316619 trn2 v0.8h, v1.4h, v2.4h
66326620 // CHECK-ERROR: [[@LINE-1]]:21: error: invalid operand for instruction
66336621 trn2 v0.4h, v1.2h, v2.2h
0 # RUN: llvm-mc -triple aarch64-none-linux-gnu -mattr=+dotprod --disassemble < %s | FileCheck %s
1 # RUN: llvm-mc -triple aarch64-none-linux-gnu -mattr=-dotprod --disassemble < %s 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
2
3 0x20,0x94,0x82,0x2e
4 0x20,0x94,0x82,0x0e
5 0x20,0x94,0x82,0x6e
6 0x20,0x94,0x82,0x4e
7 0x20,0xe0,0x82,0x2f
8 0x20,0xe0,0xa2,0x0f
9 0x20,0xe8,0x82,0x6f
10 0x20,0xe8,0xa2,0x4f
11
12 #CHECK: udot v0.2s, v1.8b, v2.8b
13 #CHECK: sdot v0.2s, v1.8b, v2.8b
14 #CHECK: udot v0.4s, v1.16b, v2.16b
15 #CHECK: sdot v0.4s, v1.16b, v2.16b
16 #CHECK: udot v0.2s, v1.8b, v2.4b[0]
17 #CHECK: sdot v0.2s, v1.8b, v2.4b[1]
18 #CHECK: udot v0.4s, v1.16b, v2.4b[2]
19 #CHECK: sdot v0.4s, v1.16b, v2.4b[3]
20
21 # CHECK-ERROR: invalid instruction encoding
22 # CHECK-ERROR: invalid instruction encoding
23 # CHECK-ERROR: invalid instruction encoding
24 # CHECK-ERROR: invalid instruction encoding
25 # CHECK-ERROR: invalid instruction encoding
26 # CHECK-ERROR: invalid instruction encoding
27 # CHECK-ERROR: invalid instruction encoding
28 # CHECK-ERROR: invalid instruction encoding