llvm.org GIT mirror llvm / 8127683
[AMDGPU] Scalar Memory instructions TD refactoring Differential revision: https://reviews.llvm.org/D23996 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@280349 91177308-0d34-0410-b5e6-96231b3b80d8 Valery Pykhtin 4 years ago
7 changed file(s) with 431 addition(s) and 398 deletion(s). Raw diff Collapse all Expand all
6969 VOP_I64_I32_I32_I64
7070 >;
7171 } // End isCommutable = 1
72
73
74 //===----------------------------------------------------------------------===//
75 // SMRD Instructions
76 //===----------------------------------------------------------------------===//
77
78 defm S_DCACHE_INV_VOL : SMRD_Inval ,
79 "s_dcache_inv_vol", int_amdgcn_s_dcache_inv_vol>;
8072
8173 //===----------------------------------------------------------------------===//
8274 // MUBUF Instructions
197197 }
198198
199199 } // End Uses = [EXEC]
200
201 //===----------------------------------------------------------------------===//
202 // Scalar operations
203 //===----------------------------------------------------------------------===//
204
205 class SMRDe op, bits<1> imm> : Enc32 {
206 bits<7> sdst;
207 bits<7> sbase;
208
209 let Inst{8} = imm;
210 let Inst{14-9} = sbase{6-1};
211 let Inst{21-15} = sdst;
212 let Inst{26-22} = op;
213 let Inst{31-27} = 0x18; //encoding
214 }
215
216 class SMRD_IMMe op> : SMRDe {
217 bits<8> offset;
218 let Inst{7-0} = offset;
219 }
220
221 class SMRD_SOFFe op> : SMRDe {
222 bits<8> soff;
223 let Inst{7-0} = soff;
224 }
225
226
227
228 class SMRD_IMMe_ci op> : Enc64 {
229 bits<7> sdst;
230 bits<7> sbase;
231 bits<32> offset;
232
233 let Inst{7-0} = 0xff;
234 let Inst{8} = 0;
235 let Inst{14-9} = sbase{6-1};
236 let Inst{21-15} = sdst;
237 let Inst{26-22} = op;
238 let Inst{31-27} = 0x18; //encoding
239 let Inst{63-32} = offset;
240 }
241
242
243 class SMRD pattern> :
244 InstSI {
245
246 let LGKM_CNT = 1;
247 let SMRD = 1;
248 let mayStore = 0;
249 let mayLoad = 1;
250 let hasSideEffects = 0;
251 let UseNamedOperandTable = 1;
252 let SchedRW = [WriteSMEM];
253 }
254200
255201 //===----------------------------------------------------------------------===//
256202 // Vector ALU operations
5353 let VI3 = vi;
5454 }
5555
56 // Specify an SMRD opcode for SI and SMEM opcode for VI
57
58 // FIXME: This should really be bits<5> si, Tablegen crashes if
59 // parameter default value is other parameter with different bit size
60 class smrd si, bits<8> vi = si> {
61 field bits<5> SI = si{4-0};
62 field bits<8> VI = vi;
63 }
6456
6557 // Execpt for the NONE field, this must be kept in sync with the
6658 // SIEncodingFamily enum in AMDGPUInstrInfo.cpp
172164
173165 def mubuf_load_atomic : MubufLoad ;
174166
175 def smrd_load : PatFrag <(ops node:$ptr), (load node:$ptr), [{
176 auto Ld = cast(N);
177 return Ld->getAlignment() >= 4 &&
178 Ld->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS &&
179 static_cast(getTargetLowering())->isMemOpUniform(N);
180 }]>;
181
182167 //===----------------------------------------------------------------------===//
183168 // PatFrags for global memory operations
184169 //===----------------------------------------------------------------------===//
476461 def omod : NamedOperandU32<"OModSI", NamedMatchClass<"OModSI">>;
477462 def clampmod : NamedOperandBit<"ClampSI", NamedMatchClass<"ClampSI">>;
478463
479 def smrd_offset : NamedOperandU32<"SMRDOffset", NamedMatchClass<"SMRDOffset">>;
480 def smrd_literal_offset : NamedOperandU32<"SMRDLiteralOffset", NamedMatchClass<"SMRDLiteralOffset">>;
481464
482465 def glc : NamedOperandBit<"GLC", NamedMatchClass<"GLC">>;
483466 def slc : NamedOperandBit<"SLC", NamedMatchClass<"SLC">>;
545528 def MUBUFOffsetAtomic : ComplexPattern;
546529 def MUBUFIntrinsicOffset : ComplexPattern;
547530 def MUBUFIntrinsicVOffset : ComplexPattern;
548
549 def SMRDImm : ComplexPattern;
550 def SMRDImm32 : ComplexPattern;
551 def SMRDSgpr : ComplexPattern;
552 def SMRDBufferImm : ComplexPattern;
553 def SMRDBufferImm32 : ComplexPattern;
554 def SMRDBufferSgpr : ComplexPattern;
555531
556532 def MOVRELOffset : ComplexPattern;
557533
637613 let DecoderNamespace="VI";
638614 let DisableDecoder = DisableVIDecoder;
639615 }
640 }
641
642 //===----------------------------------------------------------------------===//
643 // Scalar classes
644 //===----------------------------------------------------------------------===//
645
646
647 //===----------------------------------------------------------------------===//
648 // SMRD classes
649 //===----------------------------------------------------------------------===//
650
651 class SMRD_Pseudo pattern> :
652 SMRD ,
653 SIMCInstr {
654 let isPseudo = 1;
655 let isCodeGenOnly = 1;
656 }
657
658 class SMRD_IMM_Real_si op, string opName, dag outs, dag ins,
659 string asm> :
660 SMRD ,
661 SMRD_IMMe ,
662 SIMCInstr {
663 let AssemblerPredicates = [isSICI];
664 let DecoderNamespace = "SICI";
665 let DisableDecoder = DisableSIDecoder;
666 }
667
668 class SMRD_SOFF_Real_si op, string opName, dag outs, dag ins,
669 string asm> :
670 SMRD ,
671 SMRD_SOFFe ,
672 SIMCInstr {
673 let AssemblerPredicates = [isSICI];
674 let DecoderNamespace = "SICI";
675 let DisableDecoder = DisableSIDecoder;
676 }
677
678
679 class SMRD_IMM_Real_vi op, string opName, dag outs, dag ins,
680 string asm, list pattern = []> :
681 SMRD ,
682 SMEM_IMMe_vi ,
683 SIMCInstr {
684 let AssemblerPredicates = [isVI];
685 let DecoderNamespace = "VI";
686 let DisableDecoder = DisableVIDecoder;
687 }
688
689 class SMRD_SOFF_Real_vi op, string opName, dag outs, dag ins,
690 string asm, list pattern = []> :
691 SMRD ,
692 SMEM_SOFFe_vi ,
693 SIMCInstr {
694 let AssemblerPredicates = [isVI];
695 let DecoderNamespace = "VI";
696 let DisableDecoder = DisableVIDecoder;
697 }
698
699
700 multiclass SMRD_IMM_m
701 string asm, list pattern> {
702
703 def "" : SMRD_Pseudo ;
704
705 def _si : SMRD_IMM_Real_si ;
706
707 // glc is only applicable to scalar stores, which are not yet
708 // implemented.
709 let glc = 0 in {
710 def _vi : SMRD_IMM_Real_vi ;
711 }
712 }
713
714 multiclass SMRD_SOFF_m
715 string asm, list pattern> {
716
717 def "" : SMRD_Pseudo ;
718
719 def _si : SMRD_SOFF_Real_si ;
720
721 // glc is only applicable to scalar stores, which are not yet
722 // implemented.
723 let glc = 0 in {
724 def _vi : SMRD_SOFF_Real_vi ;
725 }
726 }
727
728 multiclass SMRD_Special
729 int sdst_ = ?,
730 string opStr = "",
731 list pattern = []> {
732 let hasSideEffects = 1 in {
733 def "" : SMRD_Pseudo ;
734
735 let sbase = 0, soff = 0, sdst = sdst_ in {
736 def _si : SMRD_SOFF_Real_si ;
737
738 let glc = 0 in {
739 def _vi : SMRD_SOFF_Real_vi ;
740 }
741 }
742 }
743 }
744
745 multiclass SMRD_Inval
746 SDPatternOperator node> {
747 let mayStore = 1 in {
748 defm : SMRD_Special;
749 }
750 }
751
752 class SMEM_Inval op, string opName, SDPatternOperator node> :
753 SMRD_SOFF_Real_vi {
754 let hasSideEffects = 1;
755 let mayStore = 1;
756 let sbase = 0;
757 let sdst = 0;
758 let glc = 0;
759 let soff = 0;
760 }
761
762 class SMEM_Ret op, string opName, SDPatternOperator node> :
763 SMRD_SOFF_Real_vi
764 opName#" $sdst", [(set i64:$sdst, (node))]> {
765 let hasSideEffects = 1;
766 let mayStore = ?;
767 let mayLoad = ?;
768 let sbase = 0;
769 let glc = 0;
770 let soff = 0;
771 }
772
773 multiclass SMRD_Helper
774 RegisterClass dstClass> {
775 defm _IMM : SMRD_IMM_m <
776 op, opName#"_IMM", (outs dstClass:$sdst),
777 (ins baseClass:$sbase, smrd_offset:$offset),
778 opName#" $sdst, $sbase, $offset", []
779 >;
780
781 def _IMM_ci : SMRD <
782 (outs dstClass:$sdst), (ins baseClass:$sbase, smrd_literal_offset:$offset),
783 opName#" $sdst, $sbase, $offset", []>, SMRD_IMMe_ci {
784 let AssemblerPredicates = [isCIOnly];
785 let DecoderNamespace = "CI";
786 }
787
788 defm _SGPR : SMRD_SOFF_m <
789 op, opName#"_SGPR", (outs dstClass:$sdst),
790 (ins baseClass:$sbase, SReg_32:$soff),
791 opName#" $sdst, $sbase, $soff", []
792 >;
793616 }
794617
795618 //===----------------------------------------------------------------------===//
2121 def has32BankLDS : Predicate<"Subtarget->getLDSBankCount() == 32">;
2222
2323 include "SOPInstructions.td"
24 include "SMInstructions.td"
2425
2526 let SubtargetPredicate = isGCN in {
2627
3132 defm EXP : EXP_m;
3233
3334 //===----------------------------------------------------------------------===//
34 // SMRD Instructions
35 //===----------------------------------------------------------------------===//
36
37 // We are using the SReg_32_XM0 and not the SReg_32 register class for 32-bit
38 // SMRD instructions, because the SReg_32_XM0 register class does not include M0
39 // and writing to M0 from an SMRD instruction will hang the GPU.
40 defm S_LOAD_DWORD : SMRD_Helper , "s_load_dword", SReg_64, SReg_32_XM0>;
41 defm S_LOAD_DWORDX2 : SMRD_Helper , "s_load_dwordx2", SReg_64, SReg_64>;
42 defm S_LOAD_DWORDX4 : SMRD_Helper , "s_load_dwordx4", SReg_64, SReg_128>;
43 defm S_LOAD_DWORDX8 : SMRD_Helper , "s_load_dwordx8", SReg_64, SReg_256>;
44 defm S_LOAD_DWORDX16 : SMRD_Helper , "s_load_dwordx16", SReg_64, SReg_512>;
45
46 defm S_BUFFER_LOAD_DWORD : SMRD_Helper <
47 smrd<0x08>, "s_buffer_load_dword", SReg_128, SReg_32_XM0
48 >;
49
50 defm S_BUFFER_LOAD_DWORDX2 : SMRD_Helper <
51 smrd<0x09>, "s_buffer_load_dwordx2", SReg_128, SReg_64
52 >;
53
54 defm S_BUFFER_LOAD_DWORDX4 : SMRD_Helper <
55 smrd<0x0a>, "s_buffer_load_dwordx4", SReg_128, SReg_128
56 >;
57
58 defm S_BUFFER_LOAD_DWORDX8 : SMRD_Helper <
59 smrd<0x0b>, "s_buffer_load_dwordx8", SReg_128, SReg_256
60 >;
61
62 defm S_BUFFER_LOAD_DWORDX16 : SMRD_Helper <
63 smrd<0x0c>, "s_buffer_load_dwordx16", SReg_128, SReg_512
64 >;
65
66 let mayStore = ? in {
67 // FIXME: mayStore = ? is a workaround for tablegen bug for different
68 // inferred mayStore flags for the instruction pattern vs. standalone
69 // Pat. Each considers the other contradictory.
70
71 defm S_MEMTIME : SMRD_Special , "s_memtime",
72 (outs SReg_64:$sdst), ?, " $sdst", [(set i64:$sdst, (int_amdgcn_s_memtime))]
73 >;
74 }
75
76 defm S_DCACHE_INV : SMRD_Inval , "s_dcache_inv",
77 int_amdgcn_s_dcache_inv>;
78
79 //===----------------------------------------------------------------------===//
80
8135 // VOPC Instructions
8236 //===----------------------------------------------------------------------===//
8337
18251779 def : FCMP_Pattern ;
18261780 def : FCMP_Pattern ;
18271781 def : FCMP_Pattern ;
1828 // SMRD Patterns
1829 //===----------------------------------------------------------------------===//
1830
1831 multiclass SMRD_Pattern {
1832
1833 // 1. IMM offset
1834 def : Pat <
1835 (smrd_load (SMRDImm i64:$sbase, i32:$offset)),
1836 (vt (!cast(Instr#"_IMM") $sbase, $offset))
1837 >;
1838
1839 // 2. SGPR offset
1840 def : Pat <
1841 (smrd_load (SMRDSgpr i64:$sbase, i32:$offset)),
1842 (vt (!cast(Instr#"_SGPR") $sbase, $offset))
1843 >;
1844
1845 def : Pat <
1846 (smrd_load (SMRDImm32 i64:$sbase, i32:$offset)),
1847 (vt (!cast(Instr#"_IMM_ci") $sbase, $offset))
1848 > {
1849 let Predicates = [isCIOnly];
1850 }
1851 }
1852
1853 // Global and constant loads can be selected to either MUBUF or SMRD
1854 // instructions, but SMRD instructions are faster so we want the instruction
1855 // selector to prefer those.
1856 let AddedComplexity = 100 in {
1857
1858 defm : SMRD_Pattern <"S_LOAD_DWORD", i32>;
1859 defm : SMRD_Pattern <"S_LOAD_DWORDX2", v2i32>;
1860 defm : SMRD_Pattern <"S_LOAD_DWORDX4", v4i32>;
1861 defm : SMRD_Pattern <"S_LOAD_DWORDX8", v8i32>;
1862 defm : SMRD_Pattern <"S_LOAD_DWORDX16", v16i32>;
1863
1864 // 1. Offset as an immediate
1865 def : Pat <
1866 (SIload_constant v4i32:$sbase, (SMRDBufferImm i32:$offset)),
1867 (S_BUFFER_LOAD_DWORD_IMM $sbase, $offset)
1868 >;
1869
1870 // 2. Offset loaded in an 32bit SGPR
1871 def : Pat <
1872 (SIload_constant v4i32:$sbase, (SMRDBufferSgpr i32:$offset)),
1873 (S_BUFFER_LOAD_DWORD_SGPR $sbase, $offset)
1874 >;
1875
1876 let Predicates = [isCI] in {
1877
1878 def : Pat <
1879 (SIload_constant v4i32:$sbase, (SMRDBufferImm32 i32:$offset)),
1880 (S_BUFFER_LOAD_DWORD_IMM_ci $sbase, $offset)
1881 >;
1882
1883 } // End Predicates = [isCI]
1884
1885 } // End let AddedComplexity = 10000
18861782
18871783 //===----------------------------------------------------------------------===//
18881784 // VOP1 Patterns
29822878
29832879 def : BFEPattern ;
29842880
2985 let Predicates = [isSICI] in {
2986 def : Pat <
2987 (i64 (readcyclecounter)),
2988 (S_MEMTIME)
2989 >;
2990 }
2991
29922881 def : Pat<
29932882 (fcanonicalize f32:$src),
29942883 (V_MUL_F32_e64 0, CONST.FP32_ONE, 0, $src, 0, 0)
0 //===---- SMInstructions.td - Scalar Memory Instruction Defintions --------===//
1 //
2 // The LLVM Compiler Infrastructure
3 //
4 // This file is distributed under the University of Illinois Open Source
5 // License. See LICENSE.TXT for details.
6 //
7 //===----------------------------------------------------------------------===//
8
9 def smrd_offset : NamedOperandU32<"SMRDOffset",
10 NamedMatchClass<"SMRDOffset">> {
11 let OperandType = "OPERAND_IMMEDIATE";
12 }
13
14
15 //===----------------------------------------------------------------------===//
16 // Scalar Memory classes
17 //===----------------------------------------------------------------------===//
18
19 class SM_Pseudo pattern=[]> :
20 InstSI ,
21 SIMCInstr {
22 let isPseudo = 1;
23 let isCodeGenOnly = 1;
24
25 let LGKM_CNT = 1;
26 let SMRD = 1;
27 let mayStore = 0;
28 let mayLoad = 1;
29 let hasSideEffects = 0;
30 let UseNamedOperandTable = 1;
31 let SchedRW = [WriteSMEM];
32 let SubtargetPredicate = isGCN;
33
34 string Mnemonic = opName;
35 string AsmOperands = asmOps;
36
37 bits<1> has_sbase = 1;
38 bits<1> has_sdst = 1;
39 bits<1> has_offset = 1;
40 bits<1> offset_is_imm = 0;
41 }
42
43 class SM_Real
44 : InstSI {
45
46 let isPseudo = 0;
47 let isCodeGenOnly = 0;
48
49 // copy relevant pseudo op flags
50 let SubtargetPredicate = ps.SubtargetPredicate;
51 let AsmMatchConverter = ps.AsmMatchConverter;
52
53 // encoding
54 bits<7> sbase;
55 bits<7> sdst;
56 bits<32> offset;
57 bits<1> imm = !if(ps.has_offset, ps.offset_is_imm, 0);
58 }
59
60 class SM_Load_Pseudo pattern=[]>
61 : SM_Pseudo {
62 RegisterClass BaseClass;
63 }
64
65 multiclass SM_Pseudo_Loads
66 RegisterClass baseClass,
67 RegisterClass dstClass> {
68 def _IMM : SM_Load_Pseudo
69 (outs dstClass:$sdst),
70 (ins baseClass:$sbase, i32imm:$offset),
71 " $sdst, $sbase, $offset", []> {
72 let offset_is_imm = 1;
73 let BaseClass = baseClass;
74 let PseudoInstr = opName # "_IMM";
75 }
76 def _SGPR : SM_Load_Pseudo
77 (outs dstClass:$sdst),
78 (ins baseClass:$sbase, SReg_32:$soff),
79 " $sdst, $sbase, $offset", []> {
80 let BaseClass = baseClass;
81 let PseudoInstr = opName # "_SGPR";
82 }
83 }
84
85 class SM_Time_Pseudo : SM_Pseudo<
86 opName, (outs SReg_64:$sdst), (ins),
87 " $sdst", [(set i64:$sdst, (node))]> {
88 let hasSideEffects = 1;
89 // FIXME: mayStore = ? is a workaround for tablegen bug for different
90 // inferred mayStore flags for the instruction pattern vs. standalone
91 // Pat. Each considers the other contradictory.
92 let mayStore = ?;
93 let mayLoad = ?;
94 let has_sbase = 0;
95 let has_offset = 0;
96 }
97
98 class SM_Inval_Pseudo : SM_Pseudo<
99 opName, (outs), (ins), "", [(node)]> {
100 let hasSideEffects = 1;
101 let mayStore = 1;
102 let has_sdst = 0;
103 let has_sbase = 0;
104 let has_offset = 0;
105 }
106
107
108 //===----------------------------------------------------------------------===//
109 // Scalar Memory Instructions
110 //===----------------------------------------------------------------------===//
111
112 // We are using the SReg_32_XM0 and not the SReg_32 register class for 32-bit
113 // SMRD instructions, because the SReg_32_XM0 register class does not include M0
114 // and writing to M0 from an SMRD instruction will hang the GPU.
115 defm S_LOAD_DWORD : SM_Pseudo_Loads <"s_load_dword", SReg_64, SReg_32_XM0>;
116 defm S_LOAD_DWORDX2 : SM_Pseudo_Loads <"s_load_dwordx2", SReg_64, SReg_64>;
117 defm S_LOAD_DWORDX4 : SM_Pseudo_Loads <"s_load_dwordx4", SReg_64, SReg_128>;
118 defm S_LOAD_DWORDX8 : SM_Pseudo_Loads <"s_load_dwordx8", SReg_64, SReg_256>;
119 defm S_LOAD_DWORDX16 : SM_Pseudo_Loads <"s_load_dwordx16", SReg_64, SReg_512>;
120
121 defm S_BUFFER_LOAD_DWORD : SM_Pseudo_Loads <
122 "s_buffer_load_dword", SReg_128, SReg_32_XM0
123 >;
124
125 defm S_BUFFER_LOAD_DWORDX2 : SM_Pseudo_Loads <
126 "s_buffer_load_dwordx2", SReg_128, SReg_64
127 >;
128
129 defm S_BUFFER_LOAD_DWORDX4 : SM_Pseudo_Loads <
130 "s_buffer_load_dwordx4", SReg_128, SReg_128
131 >;
132
133 defm S_BUFFER_LOAD_DWORDX8 : SM_Pseudo_Loads <
134 "s_buffer_load_dwordx8", SReg_128, SReg_256
135 >;
136
137 defm S_BUFFER_LOAD_DWORDX16 : SM_Pseudo_Loads <
138 "s_buffer_load_dwordx16", SReg_128, SReg_512
139 >;
140
141 def S_MEMTIME : SM_Time_Pseudo <"s_memtime", int_amdgcn_s_memtime>;
142 def S_DCACHE_INV : SM_Inval_Pseudo <"s_dcache_inv", int_amdgcn_s_dcache_inv>;
143
144 let SubtargetPredicate = isCIVI in {
145 def S_DCACHE_INV_VOL : SM_Inval_Pseudo <"s_dcache_inv_vol", int_amdgcn_s_dcache_inv_vol>;
146 } // let SubtargetPredicate = isCIVI
147
148 let SubtargetPredicate = isVI in {
149 def S_DCACHE_WB : SM_Inval_Pseudo <"s_dcache_wb", int_amdgcn_s_dcache_wb>;
150 def S_DCACHE_WB_VOL : SM_Inval_Pseudo <"s_dcache_wb_vol", int_amdgcn_s_dcache_wb_vol>;
151 def S_MEMREALTIME : SM_Time_Pseudo <"s_memrealtime", int_amdgcn_s_memrealtime>;
152 } // SubtargetPredicate = isVI
153
154
155
156 //===----------------------------------------------------------------------===//
157 // Scalar Memory Patterns
158 //===----------------------------------------------------------------------===//
159
160 def smrd_load : PatFrag <(ops node:$ptr), (load node:$ptr), [{
161 auto Ld = cast(N);
162 return Ld->getAlignment() >= 4 &&
163 Ld->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS &&
164 static_cast(getTargetLowering())->isMemOpUniform(N);
165 }]>;
166
167 def SMRDImm : ComplexPattern;
168 def SMRDImm32 : ComplexPattern;
169 def SMRDSgpr : ComplexPattern;
170 def SMRDBufferImm : ComplexPattern;
171 def SMRDBufferImm32 : ComplexPattern;
172 def SMRDBufferSgpr : ComplexPattern;
173
174 let Predicates = [isGCN] in {
175
176 multiclass SMRD_Pattern {
177
178 // 1. IMM offset
179 def : Pat <
180 (smrd_load (SMRDImm i64:$sbase, i32:$offset)),
181 (vt (!cast(Instr#"_IMM") $sbase, $offset))
182 >;
183
184 // 2. SGPR offset
185 def : Pat <
186 (smrd_load (SMRDSgpr i64:$sbase, i32:$offset)),
187 (vt (!cast(Instr#"_SGPR") $sbase, $offset))
188 >;
189 }
190
191 let Predicates = [isSICI] in {
192 def : Pat <
193 (i64 (readcyclecounter)),
194 (S_MEMTIME)
195 >;
196 }
197
198 // Global and constant loads can be selected to either MUBUF or SMRD
199 // instructions, but SMRD instructions are faster so we want the instruction
200 // selector to prefer those.
201 let AddedComplexity = 100 in {
202
203 defm : SMRD_Pattern <"S_LOAD_DWORD", i32>;
204 defm : SMRD_Pattern <"S_LOAD_DWORDX2", v2i32>;
205 defm : SMRD_Pattern <"S_LOAD_DWORDX4", v4i32>;
206 defm : SMRD_Pattern <"S_LOAD_DWORDX8", v8i32>;
207 defm : SMRD_Pattern <"S_LOAD_DWORDX16", v16i32>;
208
209 // 1. Offset as an immediate
210 def SM_LOAD_PATTERN : Pat < // name this pattern to reuse AddedComplexity on CI
211 (SIload_constant v4i32:$sbase, (SMRDBufferImm i32:$offset)),
212 (S_BUFFER_LOAD_DWORD_IMM $sbase, $offset)
213 >;
214
215 // 2. Offset loaded in an 32bit SGPR
216 def : Pat <
217 (SIload_constant v4i32:$sbase, (SMRDBufferSgpr i32:$offset)),
218 (S_BUFFER_LOAD_DWORD_SGPR $sbase, $offset)
219 >;
220
221 } // End let AddedComplexity = 100
222
223 } // let Predicates = [isGCN]
224
225 let Predicates = [isVI] in {
226
227 // 1. Offset as 20bit DWORD immediate
228 def : Pat <
229 (SIload_constant v4i32:$sbase, IMM20bit:$offset),
230 (S_BUFFER_LOAD_DWORD_IMM $sbase, (as_i32imm $offset))
231 >;
232
233 def : Pat <
234 (i64 (readcyclecounter)),
235 (S_MEMREALTIME)
236 >;
237
238 } // let Predicates = [isVI]
239
240
241 //===----------------------------------------------------------------------===//
242 // Targets
243 //===----------------------------------------------------------------------===//
244
245 //===----------------------------------------------------------------------===//
246 // SI
247 //===----------------------------------------------------------------------===//
248
249 class SMRD_Real_si op, SM_Pseudo ps>
250 : SM_Real
251 , SIMCInstr
252 , Enc32 {
253
254 let AssemblerPredicates = [isSICI];
255 let DecoderNamespace = "SICI";
256
257 let Inst{7-0} = !if(ps.has_offset, offset{7-0}, ?);
258 let Inst{8} = imm;
259 let Inst{14-9} = !if(ps.has_sbase, sbase{6-1}, ?);
260 let Inst{21-15} = !if(ps.has_sdst, sdst{6-0}, ?);
261 let Inst{26-22} = op;
262 let Inst{31-27} = 0x18; //encoding
263 }
264
265 multiclass SM_Real_Loads_si op, string ps,
266 SM_Load_Pseudo immPs = !cast(ps#_IMM),
267 SM_Load_Pseudo sgprPs = !cast(ps#_SGPR)> {
268 def _IMM_si : SMRD_Real_si {
269 let InOperandList = (ins immPs.BaseClass:$sbase, smrd_offset:$offset);
270 }
271 def _SGPR_si : SMRD_Real_si {
272 let InOperandList = (ins sgprPs.BaseClass:$sbase, SReg_32:$offset);
273 }
274 }
275
276 defm S_LOAD_DWORD : SM_Real_Loads_si <0x00, "S_LOAD_DWORD">;
277 defm S_LOAD_DWORDX2 : SM_Real_Loads_si <0x01, "S_LOAD_DWORDX2">;
278 defm S_LOAD_DWORDX4 : SM_Real_Loads_si <0x02, "S_LOAD_DWORDX4">;
279 defm S_LOAD_DWORDX8 : SM_Real_Loads_si <0x03, "S_LOAD_DWORDX8">;
280 defm S_LOAD_DWORDX16 : SM_Real_Loads_si <0x04, "S_LOAD_DWORDX16">;
281 defm S_BUFFER_LOAD_DWORD : SM_Real_Loads_si <0x08, "S_BUFFER_LOAD_DWORD">;
282 defm S_BUFFER_LOAD_DWORDX2 : SM_Real_Loads_si <0x09, "S_BUFFER_LOAD_DWORDX2">;
283 defm S_BUFFER_LOAD_DWORDX4 : SM_Real_Loads_si <0x0a, "S_BUFFER_LOAD_DWORDX4">;
284 defm S_BUFFER_LOAD_DWORDX8 : SM_Real_Loads_si <0x0b, "S_BUFFER_LOAD_DWORDX8">;
285 defm S_BUFFER_LOAD_DWORDX16 : SM_Real_Loads_si <0x0c, "S_BUFFER_LOAD_DWORDX16">;
286
287 def S_MEMTIME_si : SMRD_Real_si <0x1e, S_MEMTIME>;
288 def S_DCACHE_INV_si : SMRD_Real_si <0x1f, S_DCACHE_INV>;
289
290
291 //===----------------------------------------------------------------------===//
292 // VI
293 //===----------------------------------------------------------------------===//
294
295 class SMEM_Real_vi op, SM_Pseudo ps>
296 : SM_Real
297 , SIMCInstr
298 , Enc64 {
299
300 let AssemblerPredicates = [isVI];
301 let DecoderNamespace = "VI";
302
303 let Inst{5-0} = !if(ps.has_sbase, sbase{6-1}, ?);
304 let Inst{12-6} = !if(ps.has_sdst, sdst{6-0}, ?);
305
306 // glc is only applicable to scalar stores, which are not yet
307 // implemented.
308 let Inst{16} = 0; // glc bit
309 let Inst{17} = imm;
310 let Inst{25-18} = op;
311 let Inst{31-26} = 0x30; //encoding
312 let Inst{51-32} = !if(ps.has_offset, offset{19-0}, ?);
313 }
314
315 multiclass SM_Real_Loads_vi op, string ps,
316 SM_Load_Pseudo immPs = !cast(ps#_IMM),
317 SM_Load_Pseudo sgprPs = !cast(ps#_SGPR)> {
318 def _IMM_vi : SMEM_Real_vi {
319 let InOperandList = (ins immPs.BaseClass:$sbase, smrd_offset:$offset);
320 }
321 def _SGPR_vi : SMEM_Real_vi {
322 let InOperandList = (ins sgprPs.BaseClass:$sbase, SReg_32:$offset);
323 }
324 }
325
326 defm S_LOAD_DWORD : SM_Real_Loads_vi <0x00, "S_LOAD_DWORD">;
327 defm S_LOAD_DWORDX2 : SM_Real_Loads_vi <0x01, "S_LOAD_DWORDX2">;
328 defm S_LOAD_DWORDX4 : SM_Real_Loads_vi <0x02, "S_LOAD_DWORDX4">;
329 defm S_LOAD_DWORDX8 : SM_Real_Loads_vi <0x03, "S_LOAD_DWORDX8">;
330 defm S_LOAD_DWORDX16 : SM_Real_Loads_vi <0x04, "S_LOAD_DWORDX16">;
331 defm S_BUFFER_LOAD_DWORD : SM_Real_Loads_vi <0x08, "S_BUFFER_LOAD_DWORD">;
332 defm S_BUFFER_LOAD_DWORDX2 : SM_Real_Loads_vi <0x09, "S_BUFFER_LOAD_DWORDX2">;
333 defm S_BUFFER_LOAD_DWORDX4 : SM_Real_Loads_vi <0x0a, "S_BUFFER_LOAD_DWORDX4">;
334 defm S_BUFFER_LOAD_DWORDX8 : SM_Real_Loads_vi <0x0b, "S_BUFFER_LOAD_DWORDX8">;
335 defm S_BUFFER_LOAD_DWORDX16 : SM_Real_Loads_vi <0x0c, "S_BUFFER_LOAD_DWORDX16">;
336
337 def S_DCACHE_INV_vi : SMEM_Real_vi <0x20, S_DCACHE_INV>;
338 def S_DCACHE_WB_vi : SMEM_Real_vi <0x21, S_DCACHE_WB>;
339 def S_DCACHE_INV_VOL_vi : SMEM_Real_vi <0x22, S_DCACHE_INV_VOL>;
340 def S_DCACHE_WB_VOL_vi : SMEM_Real_vi <0x23, S_DCACHE_WB_VOL>;
341 def S_MEMTIME_vi : SMEM_Real_vi <0x24, S_MEMTIME>;
342 def S_MEMREALTIME_vi : SMEM_Real_vi <0x25, S_MEMREALTIME>;
343
344
345 //===----------------------------------------------------------------------===//
346 // CI
347 //===----------------------------------------------------------------------===//
348
349 def smrd_literal_offset : NamedOperandU32<"SMRDLiteralOffset",
350 NamedMatchClass<"SMRDLiteralOffset">> {
351 let OperandType = "OPERAND_IMMEDIATE";
352 }
353
354 class SMRD_Real_Load_IMM_ci op, SM_Load_Pseudo ps> :
355 SM_Real,
356 Enc64 {
357
358 let AssemblerPredicates = [isCIOnly];
359 let DecoderNamespace = "CI";
360 let InOperandList = (ins ps.BaseClass:$sbase, smrd_literal_offset:$offset);
361
362 let LGKM_CNT = ps.LGKM_CNT;
363 let SMRD = ps.SMRD;
364 let mayLoad = ps.mayLoad;
365 let mayStore = ps.mayStore;
366 let hasSideEffects = ps.hasSideEffects;
367 let SchedRW = ps.SchedRW;
368 let UseNamedOperandTable = ps.UseNamedOperandTable;
369
370 let Inst{7-0} = 0xff;
371 let Inst{8} = 0;
372 let Inst{14-9} = sbase{6-1};
373 let Inst{21-15} = sdst{6-0};
374 let Inst{26-22} = op;
375 let Inst{31-27} = 0x18; //encoding
376 let Inst{63-32} = offset{31-0};
377 }
378
379 def S_LOAD_DWORD_IMM_ci : SMRD_Real_Load_IMM_ci <0x00, S_LOAD_DWORD_IMM>;
380 def S_LOAD_DWORDX2_IMM_ci : SMRD_Real_Load_IMM_ci <0x01, S_LOAD_DWORDX2_IMM>;
381 def S_LOAD_DWORDX4_IMM_ci : SMRD_Real_Load_IMM_ci <0x02, S_LOAD_DWORDX4_IMM>;
382 def S_LOAD_DWORDX8_IMM_ci : SMRD_Real_Load_IMM_ci <0x03, S_LOAD_DWORDX8_IMM>;
383 def S_LOAD_DWORDX16_IMM_ci : SMRD_Real_Load_IMM_ci <0x04, S_LOAD_DWORDX16_IMM>;
384 def S_BUFFER_LOAD_DWORD_IMM_ci : SMRD_Real_Load_IMM_ci <0x08, S_BUFFER_LOAD_DWORD_IMM>;
385 def S_BUFFER_LOAD_DWORDX2_IMM_ci : SMRD_Real_Load_IMM_ci <0x09, S_BUFFER_LOAD_DWORDX2_IMM>;
386 def S_BUFFER_LOAD_DWORDX4_IMM_ci : SMRD_Real_Load_IMM_ci <0x0a, S_BUFFER_LOAD_DWORDX4_IMM>;
387 def S_BUFFER_LOAD_DWORDX8_IMM_ci : SMRD_Real_Load_IMM_ci <0x0b, S_BUFFER_LOAD_DWORDX8_IMM>;
388 def S_BUFFER_LOAD_DWORDX16_IMM_ci : SMRD_Real_Load_IMM_ci <0x0c, S_BUFFER_LOAD_DWORDX16_IMM>;
389
390 class SMRD_Real_ci op, SM_Pseudo ps>
391 : SM_Real
392 , SIMCInstr
393 , Enc32 {
394
395 let AssemblerPredicates = [isCIOnly];
396 let DecoderNamespace = "CI";
397
398 let Inst{7-0} = !if(ps.has_offset, offset{7-0}, ?);
399 let Inst{8} = imm;
400 let Inst{14-9} = !if(ps.has_sbase, sbase{6-1}, ?);
401 let Inst{21-15} = !if(ps.has_sdst, sdst{6-0}, ?);
402 let Inst{26-22} = op;
403 let Inst{31-27} = 0x18; //encoding
404 }
405
406 def S_DCACHE_INV_VOL_ci : SMRD_Real_ci <0x1d, S_DCACHE_INV_VOL>;
407
408 let AddedComplexity = SM_LOAD_PATTERN.AddedComplexity in {
409
410 class SMRD_Pattern_ci : Pat <
411 (smrd_load (SMRDImm32 i64:$sbase, i32:$offset)),
412 (vt (!cast(Instr#"_IMM_ci") $sbase, $offset))> {
413 let Predicates = [isCIOnly];
414 }
415
416 def : SMRD_Pattern_ci <"S_LOAD_DWORD", i32>;
417 def : SMRD_Pattern_ci <"S_LOAD_DWORDX2", v2i32>;
418 def : SMRD_Pattern_ci <"S_LOAD_DWORDX4", v4i32>;
419 def : SMRD_Pattern_ci <"S_LOAD_DWORDX8", v8i32>;
420 def : SMRD_Pattern_ci <"S_LOAD_DWORDX16", v16i32>;
421
422 def : Pat <
423 (SIload_constant v4i32:$sbase, (SMRDBufferImm32 i32:$offset)),
424 (S_BUFFER_LOAD_DWORD_IMM_ci $sbase, $offset)> {
425 let Predicates = [isCI]; // should this be isCIOnly?
426 }
427
428 } // End let AddedComplexity = SM_LOAD_PATTERN.AddedComplexity
429
6868 let Inst{63-56} = soffset;
6969 }
7070
71 class SMEMe_vi op, bit imm> : Enc64 {
72 bits<7> sbase;
73 bits<7> sdst;
74 bits<1> glc;
75
76 let Inst{5-0} = sbase{6-1};
77 let Inst{12-6} = sdst;
78 let Inst{16} = glc;
79 let Inst{17} = imm;
80 let Inst{25-18} = op;
81 let Inst{31-26} = 0x30; //encoding
82 }
83
84 class SMEM_IMMe_vi op> : SMEMe_vi {
85 bits<20> offset;
86 let Inst{51-32} = offset;
87 }
88
89 class SMEM_SOFFe_vi op> : SMEMe_vi {
90 bits<20> soff;
91 let Inst{51-32} = soff;
92 }
93
9471 class VOP3a_vi op> : Enc64 {
9572 bits<2> src0_modifiers;
9673 bits<9> src0;
100100 def : SI2_VI3Alias <"v_cvt_pknorm_u16_f32", V_CVT_PKNORM_U16_F32_e64_vi>;
101101 def : SI2_VI3Alias <"v_cvt_pkrtz_f16_f32", V_CVT_PKRTZ_F16_F32_e64_vi>;
102102
103 //===----------------------------------------------------------------------===//
104 // SMEM Instructions
105 //===----------------------------------------------------------------------===//
106
107 def S_DCACHE_WB : SMEM_Inval <0x21,
108 "s_dcache_wb", int_amdgcn_s_dcache_wb>;
109
110 def S_DCACHE_WB_VOL : SMEM_Inval <0x23,
111 "s_dcache_wb_vol", int_amdgcn_s_dcache_wb_vol>;
112
113 def S_MEMREALTIME : SMEM_Ret<0x25,
114 "s_memrealtime", int_amdgcn_s_memrealtime>;
115
116103 } // End SIAssemblerPredicate = DisableInst, SubtargetPredicate = isVI
117104
118105 let Predicates = [isVI] in {
119
120 // 1. Offset as 20bit DWORD immediate
121 def : Pat <
122 (SIload_constant v4i32:$sbase, IMM20bit:$offset),
123 (S_BUFFER_LOAD_DWORD_IMM $sbase, (as_i32imm $offset))
124 >;
125106
126107 //===----------------------------------------------------------------------===//
127108 // DPP Patterns
138119 // Misc Patterns
139120 //===----------------------------------------------------------------------===//
140121
141 def : Pat <
142 (i64 (readcyclecounter)),
143 (S_MEMREALTIME)
144 >;
145
146122 } // End Predicates = [isVI]