30#include "llvm/IR/IntrinsicsAArch64.h"
31#include "llvm/IR/IntrinsicsARM.h"
32#include "llvm/IR/IntrinsicsNVPTX.h"
33#include "llvm/IR/IntrinsicsRISCV.h"
34#include "llvm/IR/IntrinsicsWebAssembly.h"
35#include "llvm/IR/IntrinsicsX86.h"
50 cl::desc(
"Disable autoupgrade of debug info"));
60 Type *Arg0Type =
F->getFunctionType()->getParamType(0);
75 Type *LastArgType =
F->getFunctionType()->getParamType(
76 F->getFunctionType()->getNumParams() - 1);
91 if (
F->getReturnType()->isVectorTy())
101 if (
F->getReturnType()->getScalarType()->isBFloatTy())
111 if (
F->getFunctionType()->getParamType(1)->getScalarType()->isBFloatTy())
125 if (
Name.consume_front(
"avx."))
126 return (
Name.starts_with(
"blend.p") ||
127 Name ==
"cvt.ps2.pd.256" ||
128 Name ==
"cvtdq2.pd.256" ||
129 Name ==
"cvtdq2.ps.256" ||
130 Name.starts_with(
"movnt.") ||
131 Name.starts_with(
"sqrt.p") ||
132 Name.starts_with(
"storeu.") ||
133 Name.starts_with(
"vbroadcast.s") ||
134 Name.starts_with(
"vbroadcastf128") ||
135 Name.starts_with(
"vextractf128.") ||
136 Name.starts_with(
"vinsertf128.") ||
137 Name.starts_with(
"vperm2f128.") ||
138 Name.starts_with(
"vpermil."));
140 if (
Name.consume_front(
"avx2."))
141 return (
Name ==
"movntdqa" ||
142 Name.starts_with(
"pabs.") ||
143 Name.starts_with(
"padds.") ||
144 Name.starts_with(
"paddus.") ||
145 Name.starts_with(
"pblendd.") ||
147 Name.starts_with(
"pbroadcast") ||
148 Name.starts_with(
"pcmpeq.") ||
149 Name.starts_with(
"pcmpgt.") ||
150 Name.starts_with(
"pmax") ||
151 Name.starts_with(
"pmin") ||
152 Name.starts_with(
"pmovsx") ||
153 Name.starts_with(
"pmovzx") ||
155 Name ==
"pmulu.dq" ||
156 Name.starts_with(
"psll.dq") ||
157 Name.starts_with(
"psrl.dq") ||
158 Name.starts_with(
"psubs.") ||
159 Name.starts_with(
"psubus.") ||
160 Name.starts_with(
"vbroadcast") ||
161 Name ==
"vbroadcasti128" ||
162 Name ==
"vextracti128" ||
163 Name ==
"vinserti128" ||
164 Name ==
"vperm2i128");
166 if (
Name.consume_front(
"avx512.")) {
167 if (
Name.consume_front(
"mask."))
169 return (
Name.starts_with(
"add.p") ||
170 Name.starts_with(
"and.") ||
171 Name.starts_with(
"andn.") ||
172 Name.starts_with(
"broadcast.s") ||
173 Name.starts_with(
"broadcastf32x4.") ||
174 Name.starts_with(
"broadcastf32x8.") ||
175 Name.starts_with(
"broadcastf64x2.") ||
176 Name.starts_with(
"broadcastf64x4.") ||
177 Name.starts_with(
"broadcasti32x4.") ||
178 Name.starts_with(
"broadcasti32x8.") ||
179 Name.starts_with(
"broadcasti64x2.") ||
180 Name.starts_with(
"broadcasti64x4.") ||
181 Name.starts_with(
"cmp.b") ||
182 Name.starts_with(
"cmp.d") ||
183 Name.starts_with(
"cmp.q") ||
184 Name.starts_with(
"cmp.w") ||
185 Name.starts_with(
"compress.b") ||
186 Name.starts_with(
"compress.d") ||
187 Name.starts_with(
"compress.p") ||
188 Name.starts_with(
"compress.q") ||
189 Name.starts_with(
"compress.store.") ||
190 Name.starts_with(
"compress.w") ||
191 Name.starts_with(
"conflict.") ||
192 Name.starts_with(
"cvtdq2pd.") ||
193 Name.starts_with(
"cvtdq2ps.") ||
194 Name ==
"cvtpd2dq.256" ||
195 Name ==
"cvtpd2ps.256" ||
196 Name ==
"cvtps2pd.128" ||
197 Name ==
"cvtps2pd.256" ||
198 Name.starts_with(
"cvtqq2pd.") ||
199 Name ==
"cvtqq2ps.256" ||
200 Name ==
"cvtqq2ps.512" ||
201 Name ==
"cvttpd2dq.256" ||
202 Name ==
"cvttps2dq.128" ||
203 Name ==
"cvttps2dq.256" ||
204 Name.starts_with(
"cvtudq2pd.") ||
205 Name.starts_with(
"cvtudq2ps.") ||
206 Name.starts_with(
"cvtuqq2pd.") ||
207 Name ==
"cvtuqq2ps.256" ||
208 Name ==
"cvtuqq2ps.512" ||
209 Name.starts_with(
"dbpsadbw.") ||
210 Name.starts_with(
"div.p") ||
211 Name.starts_with(
"expand.b") ||
212 Name.starts_with(
"expand.d") ||
213 Name.starts_with(
"expand.load.") ||
214 Name.starts_with(
"expand.p") ||
215 Name.starts_with(
"expand.q") ||
216 Name.starts_with(
"expand.w") ||
217 Name.starts_with(
"fpclass.p") ||
218 Name.starts_with(
"insert") ||
219 Name.starts_with(
"load.") ||
220 Name.starts_with(
"loadu.") ||
221 Name.starts_with(
"lzcnt.") ||
222 Name.starts_with(
"max.p") ||
223 Name.starts_with(
"min.p") ||
224 Name.starts_with(
"movddup") ||
225 Name.starts_with(
"move.s") ||
226 Name.starts_with(
"movshdup") ||
227 Name.starts_with(
"movsldup") ||
228 Name.starts_with(
"mul.p") ||
229 Name.starts_with(
"or.") ||
230 Name.starts_with(
"pabs.") ||
231 Name.starts_with(
"packssdw.") ||
232 Name.starts_with(
"packsswb.") ||
233 Name.starts_with(
"packusdw.") ||
234 Name.starts_with(
"packuswb.") ||
235 Name.starts_with(
"padd.") ||
236 Name.starts_with(
"padds.") ||
237 Name.starts_with(
"paddus.") ||
238 Name.starts_with(
"palignr.") ||
239 Name.starts_with(
"pand.") ||
240 Name.starts_with(
"pandn.") ||
241 Name.starts_with(
"pavg") ||
242 Name.starts_with(
"pbroadcast") ||
243 Name.starts_with(
"pcmpeq.") ||
244 Name.starts_with(
"pcmpgt.") ||
245 Name.starts_with(
"perm.df.") ||
246 Name.starts_with(
"perm.di.") ||
247 Name.starts_with(
"permvar.") ||
248 Name.starts_with(
"pmaddubs.w.") ||
249 Name.starts_with(
"pmaddw.d.") ||
250 Name.starts_with(
"pmax") ||
251 Name.starts_with(
"pmin") ||
252 Name ==
"pmov.qd.256" ||
253 Name ==
"pmov.qd.512" ||
254 Name ==
"pmov.wb.256" ||
255 Name ==
"pmov.wb.512" ||
256 Name.starts_with(
"pmovsx") ||
257 Name.starts_with(
"pmovzx") ||
258 Name.starts_with(
"pmul.dq.") ||
259 Name.starts_with(
"pmul.hr.sw.") ||
260 Name.starts_with(
"pmulh.w.") ||
261 Name.starts_with(
"pmulhu.w.") ||
262 Name.starts_with(
"pmull.") ||
263 Name.starts_with(
"pmultishift.qb.") ||
264 Name.starts_with(
"pmulu.dq.") ||
265 Name.starts_with(
"por.") ||
266 Name.starts_with(
"prol.") ||
267 Name.starts_with(
"prolv.") ||
268 Name.starts_with(
"pror.") ||
269 Name.starts_with(
"prorv.") ||
270 Name.starts_with(
"pshuf.b.") ||
271 Name.starts_with(
"pshuf.d.") ||
272 Name.starts_with(
"pshufh.w.") ||
273 Name.starts_with(
"pshufl.w.") ||
274 Name.starts_with(
"psll.d") ||
275 Name.starts_with(
"psll.q") ||
276 Name.starts_with(
"psll.w") ||
277 Name.starts_with(
"pslli") ||
278 Name.starts_with(
"psllv") ||
279 Name.starts_with(
"psra.d") ||
280 Name.starts_with(
"psra.q") ||
281 Name.starts_with(
"psra.w") ||
282 Name.starts_with(
"psrai") ||
283 Name.starts_with(
"psrav") ||
284 Name.starts_with(
"psrl.d") ||
285 Name.starts_with(
"psrl.q") ||
286 Name.starts_with(
"psrl.w") ||
287 Name.starts_with(
"psrli") ||
288 Name.starts_with(
"psrlv") ||
289 Name.starts_with(
"psub.") ||
290 Name.starts_with(
"psubs.") ||
291 Name.starts_with(
"psubus.") ||
292 Name.starts_with(
"pternlog.") ||
293 Name.starts_with(
"punpckh") ||
294 Name.starts_with(
"punpckl") ||
295 Name.starts_with(
"pxor.") ||
296 Name.starts_with(
"shuf.f") ||
297 Name.starts_with(
"shuf.i") ||
298 Name.starts_with(
"shuf.p") ||
299 Name.starts_with(
"sqrt.p") ||
300 Name.starts_with(
"store.b.") ||
301 Name.starts_with(
"store.d.") ||
302 Name.starts_with(
"store.p") ||
303 Name.starts_with(
"store.q.") ||
304 Name.starts_with(
"store.w.") ||
305 Name ==
"store.ss" ||
306 Name.starts_with(
"storeu.") ||
307 Name.starts_with(
"sub.p") ||
308 Name.starts_with(
"ucmp.") ||
309 Name.starts_with(
"unpckh.") ||
310 Name.starts_with(
"unpckl.") ||
311 Name.starts_with(
"valign.") ||
312 Name ==
"vcvtph2ps.128" ||
313 Name ==
"vcvtph2ps.256" ||
314 Name.starts_with(
"vextract") ||
315 Name.starts_with(
"vfmadd.") ||
316 Name.starts_with(
"vfmaddsub.") ||
317 Name.starts_with(
"vfnmadd.") ||
318 Name.starts_with(
"vfnmsub.") ||
319 Name.starts_with(
"vpdpbusd.") ||
320 Name.starts_with(
"vpdpbusds.") ||
321 Name.starts_with(
"vpdpwssd.") ||
322 Name.starts_with(
"vpdpwssds.") ||
323 Name.starts_with(
"vpermi2var.") ||
324 Name.starts_with(
"vpermil.p") ||
325 Name.starts_with(
"vpermilvar.") ||
326 Name.starts_with(
"vpermt2var.") ||
327 Name.starts_with(
"vpmadd52") ||
328 Name.starts_with(
"vpshld.") ||
329 Name.starts_with(
"vpshldv.") ||
330 Name.starts_with(
"vpshrd.") ||
331 Name.starts_with(
"vpshrdv.") ||
332 Name.starts_with(
"vpshufbitqmb.") ||
333 Name.starts_with(
"xor."));
335 if (
Name.consume_front(
"mask3."))
337 return (
Name.starts_with(
"vfmadd.") ||
338 Name.starts_with(
"vfmaddsub.") ||
339 Name.starts_with(
"vfmsub.") ||
340 Name.starts_with(
"vfmsubadd.") ||
341 Name.starts_with(
"vfnmsub."));
343 if (
Name.consume_front(
"maskz."))
345 return (
Name.starts_with(
"pternlog.") ||
346 Name.starts_with(
"vfmadd.") ||
347 Name.starts_with(
"vfmaddsub.") ||
348 Name.starts_with(
"vpdpbusd.") ||
349 Name.starts_with(
"vpdpbusds.") ||
350 Name.starts_with(
"vpdpwssd.") ||
351 Name.starts_with(
"vpdpwssds.") ||
352 Name.starts_with(
"vpermt2var.") ||
353 Name.starts_with(
"vpmadd52") ||
354 Name.starts_with(
"vpshldv.") ||
355 Name.starts_with(
"vpshrdv."));
358 return (
Name ==
"movntdqa" ||
359 Name ==
"pmul.dq.512" ||
360 Name ==
"pmulu.dq.512" ||
361 Name.starts_with(
"broadcastm") ||
362 Name.starts_with(
"cmp.p") ||
363 Name.starts_with(
"cvtb2mask.") ||
364 Name.starts_with(
"cvtd2mask.") ||
365 Name.starts_with(
"cvtmask2") ||
366 Name.starts_with(
"cvtq2mask.") ||
367 Name ==
"cvtusi2sd" ||
368 Name.starts_with(
"cvtw2mask.") ||
373 Name ==
"kortestc.w" ||
374 Name ==
"kortestz.w" ||
375 Name.starts_with(
"kunpck") ||
378 Name.starts_with(
"padds.") ||
379 Name.starts_with(
"pbroadcast") ||
380 Name.starts_with(
"prol") ||
381 Name.starts_with(
"pror") ||
382 Name.starts_with(
"psll.dq") ||
383 Name.starts_with(
"psrl.dq") ||
384 Name.starts_with(
"psubs.") ||
385 Name.starts_with(
"ptestm") ||
386 Name.starts_with(
"ptestnm") ||
387 Name.starts_with(
"storent.") ||
388 Name.starts_with(
"vbroadcast.s") ||
389 Name.starts_with(
"vpshld.") ||
390 Name.starts_with(
"vpshrd."));
393 if (
Name.consume_front(
"fma."))
394 return (
Name.starts_with(
"vfmadd.") ||
395 Name.starts_with(
"vfmsub.") ||
396 Name.starts_with(
"vfmsubadd.") ||
397 Name.starts_with(
"vfnmadd.") ||
398 Name.starts_with(
"vfnmsub."));
400 if (
Name.consume_front(
"fma4."))
401 return Name.starts_with(
"vfmadd.s");
403 if (
Name.consume_front(
"sse."))
404 return (
Name ==
"add.ss" ||
405 Name ==
"cvtsi2ss" ||
406 Name ==
"cvtsi642ss" ||
409 Name.starts_with(
"sqrt.p") ||
411 Name.starts_with(
"storeu.") ||
414 if (
Name.consume_front(
"sse2."))
415 return (
Name ==
"add.sd" ||
416 Name ==
"cvtdq2pd" ||
417 Name ==
"cvtdq2ps" ||
418 Name ==
"cvtps2pd" ||
419 Name ==
"cvtsi2sd" ||
420 Name ==
"cvtsi642sd" ||
421 Name ==
"cvtss2sd" ||
424 Name.starts_with(
"padds.") ||
425 Name.starts_with(
"paddus.") ||
426 Name.starts_with(
"pcmpeq.") ||
427 Name.starts_with(
"pcmpgt.") ||
432 Name ==
"pmulu.dq" ||
433 Name.starts_with(
"pshuf") ||
434 Name.starts_with(
"psll.dq") ||
435 Name.starts_with(
"psrl.dq") ||
436 Name.starts_with(
"psubs.") ||
437 Name.starts_with(
"psubus.") ||
438 Name.starts_with(
"sqrt.p") ||
440 Name ==
"storel.dq" ||
441 Name.starts_with(
"storeu.") ||
444 if (
Name.consume_front(
"sse41."))
445 return (
Name.starts_with(
"blendp") ||
446 Name ==
"movntdqa" ||
456 Name.starts_with(
"pmovsx") ||
457 Name.starts_with(
"pmovzx") ||
460 if (
Name.consume_front(
"sse42."))
461 return Name ==
"crc32.64.8";
463 if (
Name.consume_front(
"sse4a."))
464 return Name.starts_with(
"movnt.");
466 if (
Name.consume_front(
"ssse3."))
467 return (
Name ==
"pabs.b.128" ||
468 Name ==
"pabs.d.128" ||
469 Name ==
"pabs.w.128");
471 if (
Name.consume_front(
"xop."))
472 return (
Name ==
"vpcmov" ||
473 Name ==
"vpcmov.256" ||
474 Name.starts_with(
"vpcom") ||
475 Name.starts_with(
"vprot"));
477 return (
Name ==
"addcarry.u32" ||
478 Name ==
"addcarry.u64" ||
479 Name ==
"addcarryx.u32" ||
480 Name ==
"addcarryx.u64" ||
481 Name ==
"subborrow.u32" ||
482 Name ==
"subborrow.u64" ||
483 Name.starts_with(
"vcvtph2ps."));
489 if (!
Name.consume_front(
"x86."))
497 if (
Name ==
"rdtscp") {
499 if (
F->getFunctionType()->getNumParams() == 0)
504 Intrinsic::x86_rdtscp);
511 if (
Name.consume_front(
"sse41.ptest")) {
513 .
Case(
"c", Intrinsic::x86_sse41_ptestc)
514 .
Case(
"z", Intrinsic::x86_sse41_ptestz)
515 .
Case(
"nzc", Intrinsic::x86_sse41_ptestnzc)
528 .
Case(
"sse41.insertps", Intrinsic::x86_sse41_insertps)
529 .
Case(
"sse41.dppd", Intrinsic::x86_sse41_dppd)
530 .
Case(
"sse41.dpps", Intrinsic::x86_sse41_dpps)
531 .
Case(
"sse41.mpsadbw", Intrinsic::x86_sse41_mpsadbw)
532 .
Case(
"avx.dp.ps.256", Intrinsic::x86_avx_dp_ps_256)
533 .
Case(
"avx2.mpsadbw", Intrinsic::x86_avx2_mpsadbw)
538 if (
Name.consume_front(
"avx512.mask.cmp.")) {
541 .
Case(
"pd.128", Intrinsic::x86_avx512_mask_cmp_pd_128)
542 .
Case(
"pd.256", Intrinsic::x86_avx512_mask_cmp_pd_256)
543 .
Case(
"pd.512", Intrinsic::x86_avx512_mask_cmp_pd_512)
544 .
Case(
"ps.128", Intrinsic::x86_avx512_mask_cmp_ps_128)
545 .
Case(
"ps.256", Intrinsic::x86_avx512_mask_cmp_ps_256)
546 .
Case(
"ps.512", Intrinsic::x86_avx512_mask_cmp_ps_512)
553 if (
Name.consume_front(
"avx512bf16.")) {
556 .
Case(
"cvtne2ps2bf16.128",
557 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128)
558 .
Case(
"cvtne2ps2bf16.256",
559 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256)
560 .
Case(
"cvtne2ps2bf16.512",
561 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512)
562 .
Case(
"mask.cvtneps2bf16.128",
563 Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
564 .
Case(
"cvtneps2bf16.256",
565 Intrinsic::x86_avx512bf16_cvtneps2bf16_256)
566 .
Case(
"cvtneps2bf16.512",
567 Intrinsic::x86_avx512bf16_cvtneps2bf16_512)
574 .
Case(
"dpbf16ps.128", Intrinsic::x86_avx512bf16_dpbf16ps_128)
575 .
Case(
"dpbf16ps.256", Intrinsic::x86_avx512bf16_dpbf16ps_256)
576 .
Case(
"dpbf16ps.512", Intrinsic::x86_avx512bf16_dpbf16ps_512)
583 if (
Name.consume_front(
"xop.")) {
585 if (
Name.starts_with(
"vpermil2")) {
588 auto Idx =
F->getFunctionType()->getParamType(2);
589 if (
Idx->isFPOrFPVectorTy()) {
590 unsigned IdxSize =
Idx->getPrimitiveSizeInBits();
591 unsigned EltSize =
Idx->getScalarSizeInBits();
592 if (EltSize == 64 && IdxSize == 128)
593 ID = Intrinsic::x86_xop_vpermil2pd;
594 else if (EltSize == 32 && IdxSize == 128)
595 ID = Intrinsic::x86_xop_vpermil2ps;
596 else if (EltSize == 64 && IdxSize == 256)
597 ID = Intrinsic::x86_xop_vpermil2pd_256;
599 ID = Intrinsic::x86_xop_vpermil2ps_256;
601 }
else if (
F->arg_size() == 2)
604 .
Case(
"vfrcz.ss", Intrinsic::x86_xop_vfrcz_ss)
605 .
Case(
"vfrcz.sd", Intrinsic::x86_xop_vfrcz_sd)
616 if (
Name ==
"seh.recoverfp") {
629 if (
Name.starts_with(
"rbit")) {
632 F->arg_begin()->getType());
636 if (
Name ==
"thread.pointer") {
643 bool Neon =
Name.consume_front(
"neon.");
648 if (
Name.consume_front(
"bfdot.")) {
652 .
Cases(
"v2f32.v8i8",
"v4f32.v16i8",
657 size_t OperandWidth =
F->getReturnType()->getPrimitiveSizeInBits();
658 assert((OperandWidth == 64 || OperandWidth == 128) &&
659 "Unexpected operand width");
661 std::array<Type *, 2> Tys{
672 if (
Name.consume_front(
"bfm")) {
674 if (
Name.consume_back(
".v4f32.v16i8")) {
714 F->arg_begin()->getType());
718 if (
Name.consume_front(
"vst")) {
720 static const Regex vstRegex(
"^([1234]|[234]lane)\\.v[a-z0-9]*$");
724 Intrinsic::arm_neon_vst1, Intrinsic::arm_neon_vst2,
725 Intrinsic::arm_neon_vst3, Intrinsic::arm_neon_vst4};
728 Intrinsic::arm_neon_vst2lane, Intrinsic::arm_neon_vst3lane,
729 Intrinsic::arm_neon_vst4lane};
731 auto fArgs =
F->getFunctionType()->params();
732 Type *Tys[] = {fArgs[0], fArgs[1]};
735 StoreInts[fArgs.size() - 3], Tys);
738 F->getParent(), StoreLaneInts[fArgs.size() - 5], Tys);
747 if (
Name.consume_front(
"mve.")) {
749 if (
Name ==
"vctp64") {
750 if (cast<FixedVectorType>(
F->getReturnType())->getNumElements() == 4) {
760 if (
Name.consume_back(
".v4i1")) {
762 if (
Name.consume_back(
".predicated.v2i64.v4i32"))
764 return Name ==
"mull.int" ||
Name ==
"vqdmull";
766 if (
Name.consume_back(
".v2i64")) {
768 bool IsGather =
Name.consume_front(
"vldr.gather.");
769 if (IsGather ||
Name.consume_front(
"vstr.scatter.")) {
770 if (
Name.consume_front(
"base.")) {
772 Name.consume_front(
"wb.");
775 return Name ==
"predicated.v2i64";
778 if (
Name.consume_front(
"offset.predicated."))
779 return Name == (IsGather ?
"v2i64.p0i64" :
"p0i64.v2i64") ||
780 Name == (IsGather ?
"v2i64.p0" :
"p0.v2i64");
793 if (
Name.consume_front(
"cde.vcx")) {
795 if (
Name.consume_back(
".predicated.v2i64.v4i1"))
812 F->arg_begin()->getType());
816 if (
Name.starts_with(
"addp")) {
818 if (
F->arg_size() != 2)
820 VectorType *Ty = dyn_cast<VectorType>(
F->getReturnType());
821 if (Ty && Ty->getElementType()->isFloatingPointTy()) {
823 Intrinsic::aarch64_neon_faddp, Ty);
829 if (
Name.consume_front(
"sve.")) {
831 if (
Name.consume_front(
"bf")) {
832 if (
Name.consume_back(
".lane")) {
836 .
Case(
"dot", Intrinsic::aarch64_sve_bfdot_lane_v2)
837 .
Case(
"mlalb", Intrinsic::aarch64_sve_bfmlalb_lane_v2)
838 .
Case(
"mlalt", Intrinsic::aarch64_sve_bfmlalt_lane_v2)
849 if (
Name.consume_front(
"addqv")) {
851 if (!
F->getReturnType()->isFPOrFPVectorTy())
854 auto Args =
F->getFunctionType()->params();
855 Type *Tys[] = {
F->getReturnType(), Args[1]};
857 Intrinsic::aarch64_sve_faddqv, Tys);
861 if (
Name.consume_front(
"ld")) {
863 static const Regex LdRegex(
"^[234](.nxv[a-z0-9]+|$)");
866 dyn_cast<VectorType>(
F->getReturnType())->getElementType();
867 ElementCount EC = dyn_cast<VectorType>(
F->arg_begin()->getType())
869 Type *Ty = VectorType::get(ScalarTy, EC);
871 Intrinsic::aarch64_sve_ld2_sret,
872 Intrinsic::aarch64_sve_ld3_sret,
873 Intrinsic::aarch64_sve_ld4_sret,
876 LoadIDs[
Name[0] -
'2'], Ty);
882 if (
Name.consume_front(
"tuple.")) {
884 if (
Name.starts_with(
"get")) {
886 Type *Tys[] = {
F->getReturnType(),
F->arg_begin()->getType()};
888 Intrinsic::vector_extract, Tys);
892 if (
Name.starts_with(
"set")) {
894 auto Args =
F->getFunctionType()->params();
895 Type *Tys[] = {Args[0], Args[2], Args[1]};
897 Intrinsic::vector_insert, Tys);
901 static const Regex CreateTupleRegex(
"^create[234](.nxv[a-z0-9]+|$)");
904 auto Args =
F->getFunctionType()->params();
905 Type *Tys[] = {
F->getReturnType(), Args[1]};
907 Intrinsic::vector_insert, Tys);
919 if (
Name.consume_front(
"abs."))
921 .
Case(
"bf16", Intrinsic::nvvm_abs_bf16)
922 .
Case(
"bf16x2", Intrinsic::nvvm_abs_bf16x2)
925 if (
Name.consume_front(
"fma.rn."))
927 .
Case(
"bf16", Intrinsic::nvvm_fma_rn_bf16)
928 .
Case(
"bf16x2", Intrinsic::nvvm_fma_rn_bf16x2)
929 .
Case(
"ftz.bf16", Intrinsic::nvvm_fma_rn_ftz_bf16)
930 .
Case(
"ftz.bf16x2", Intrinsic::nvvm_fma_rn_ftz_bf16x2)
931 .
Case(
"ftz.relu.bf16", Intrinsic::nvvm_fma_rn_ftz_relu_bf16)
932 .
Case(
"ftz.relu.bf16x2", Intrinsic::nvvm_fma_rn_ftz_relu_bf16x2)
933 .
Case(
"ftz.sat.bf16", Intrinsic::nvvm_fma_rn_ftz_sat_bf16)
934 .
Case(
"ftz.sat.bf16x2", Intrinsic::nvvm_fma_rn_ftz_sat_bf16x2)
935 .
Case(
"relu.bf16", Intrinsic::nvvm_fma_rn_relu_bf16)
936 .
Case(
"relu.bf16x2", Intrinsic::nvvm_fma_rn_relu_bf16x2)
937 .
Case(
"sat.bf16", Intrinsic::nvvm_fma_rn_sat_bf16)
938 .
Case(
"sat.bf16x2", Intrinsic::nvvm_fma_rn_sat_bf16x2)
941 if (
Name.consume_front(
"fmax."))
943 .
Case(
"bf16", Intrinsic::nvvm_fmax_bf16)
944 .
Case(
"bf16x2", Intrinsic::nvvm_fmax_bf16x2)
945 .
Case(
"ftz.bf16", Intrinsic::nvvm_fmax_ftz_bf16)
946 .
Case(
"ftz.bf16x2", Intrinsic::nvvm_fmax_ftz_bf16x2)
947 .
Case(
"ftz.nan.bf16", Intrinsic::nvvm_fmax_ftz_nan_bf16)
948 .
Case(
"ftz.nan.bf16x2", Intrinsic::nvvm_fmax_ftz_nan_bf16x2)
949 .
Case(
"ftz.nan.xorsign.abs.bf16",
950 Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16)
951 .
Case(
"ftz.nan.xorsign.abs.bf16x2",
952 Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16x2)
953 .
Case(
"ftz.xorsign.abs.bf16", Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16)
954 .
Case(
"ftz.xorsign.abs.bf16x2",
955 Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16x2)
956 .
Case(
"nan.bf16", Intrinsic::nvvm_fmax_nan_bf16)
957 .
Case(
"nan.bf16x2", Intrinsic::nvvm_fmax_nan_bf16x2)
958 .
Case(
"nan.xorsign.abs.bf16", Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16)
959 .
Case(
"nan.xorsign.abs.bf16x2",
960 Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16x2)
961 .
Case(
"xorsign.abs.bf16", Intrinsic::nvvm_fmax_xorsign_abs_bf16)
962 .
Case(
"xorsign.abs.bf16x2", Intrinsic::nvvm_fmax_xorsign_abs_bf16x2)
965 if (
Name.consume_front(
"fmin."))
967 .
Case(
"bf16", Intrinsic::nvvm_fmin_bf16)
968 .
Case(
"bf16x2", Intrinsic::nvvm_fmin_bf16x2)
969 .
Case(
"ftz.bf16", Intrinsic::nvvm_fmin_ftz_bf16)
970 .
Case(
"ftz.bf16x2", Intrinsic::nvvm_fmin_ftz_bf16x2)
971 .
Case(
"ftz.nan.bf16", Intrinsic::nvvm_fmin_ftz_nan_bf16)
972 .
Case(
"ftz.nan.bf16x2", Intrinsic::nvvm_fmin_ftz_nan_bf16x2)
973 .
Case(
"ftz.nan.xorsign.abs.bf16",
974 Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16)
975 .
Case(
"ftz.nan.xorsign.abs.bf16x2",
976 Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16x2)
977 .
Case(
"ftz.xorsign.abs.bf16", Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16)
978 .
Case(
"ftz.xorsign.abs.bf16x2",
979 Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16x2)
980 .
Case(
"nan.bf16", Intrinsic::nvvm_fmin_nan_bf16)
981 .
Case(
"nan.bf16x2", Intrinsic::nvvm_fmin_nan_bf16x2)
982 .
Case(
"nan.xorsign.abs.bf16", Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16)
983 .
Case(
"nan.xorsign.abs.bf16x2",
984 Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16x2)
985 .
Case(
"xorsign.abs.bf16", Intrinsic::nvvm_fmin_xorsign_abs_bf16)
986 .
Case(
"xorsign.abs.bf16x2", Intrinsic::nvvm_fmin_xorsign_abs_bf16x2)
989 if (
Name.consume_front(
"neg."))
991 .
Case(
"bf16", Intrinsic::nvvm_neg_bf16)
992 .
Case(
"bf16x2", Intrinsic::nvvm_neg_bf16x2)
999 bool CanUpgradeDebugIntrinsicsToRecords) {
1000 assert(
F &&
"Illegal to upgrade a non-existent Function.");
1005 if (!
Name.consume_front(
"llvm.") ||
Name.empty())
1011 bool IsArm =
Name.consume_front(
"arm.");
1012 if (IsArm ||
Name.consume_front(
"aarch64.")) {
1018 if (
Name.consume_front(
"amdgcn.")) {
1019 if (
Name ==
"alignbit") {
1022 {F->getReturnType()});
1026 if (
Name.consume_front(
"atomic.")) {
1027 if (
Name.starts_with(
"inc") ||
Name.starts_with(
"dec")) {
1036 if (
Name.starts_with(
"ldexp.")) {
1039 F->getParent(), Intrinsic::ldexp,
1040 {F->getReturnType(), F->getArg(1)->getType()});
1049 if (
F->arg_size() == 1) {
1057 F->arg_begin()->getType());
1062 if (
F->arg_size() == 2 &&
Name ==
"coro.end") {
1071 if (
Name.consume_front(
"dbg.")) {
1073 if (CanUpgradeDebugIntrinsicsToRecords &&
1074 F->getParent()->IsNewDbgInfoFormat) {
1075 if (
Name ==
"addr" ||
Name ==
"value" ||
Name ==
"assign" ||
1076 Name ==
"declare" ||
Name ==
"label") {
1085 if (
Name ==
"addr" || (
Name ==
"value" &&
F->arg_size() == 4)) {
1094 if (
Name.consume_front(
"experimental.vector.")) {
1097 .
StartsWith(
"extract.", Intrinsic::vector_extract)
1098 .
StartsWith(
"insert.", Intrinsic::vector_insert)
1099 .
StartsWith(
"splice.", Intrinsic::vector_splice)
1100 .
StartsWith(
"reverse.", Intrinsic::vector_reverse)
1101 .
StartsWith(
"interleave2.", Intrinsic::vector_interleave2)
1102 .
StartsWith(
"deinterleave2.", Intrinsic::vector_deinterleave2)
1105 const auto *FT =
F->getFunctionType();
1107 if (
ID == Intrinsic::vector_extract ||
1108 ID == Intrinsic::vector_interleave2)
1111 if (
ID != Intrinsic::vector_interleave2)
1113 if (
ID == Intrinsic::vector_insert)
1121 if (
Name.consume_front(
"reduce.")) {
1123 static const Regex R(
"^([a-z]+)\\.[a-z][0-9]+");
1126 .
Case(
"add", Intrinsic::vector_reduce_add)
1127 .
Case(
"mul", Intrinsic::vector_reduce_mul)
1128 .
Case(
"and", Intrinsic::vector_reduce_and)
1129 .
Case(
"or", Intrinsic::vector_reduce_or)
1130 .
Case(
"xor", Intrinsic::vector_reduce_xor)
1131 .
Case(
"smax", Intrinsic::vector_reduce_smax)
1132 .
Case(
"smin", Intrinsic::vector_reduce_smin)
1133 .
Case(
"umax", Intrinsic::vector_reduce_umax)
1134 .
Case(
"umin", Intrinsic::vector_reduce_umin)
1135 .
Case(
"fmax", Intrinsic::vector_reduce_fmax)
1136 .
Case(
"fmin", Intrinsic::vector_reduce_fmin)
1141 static const Regex R2(
"^v2\\.([a-z]+)\\.[fi][0-9]+");
1146 .
Case(
"fadd", Intrinsic::vector_reduce_fadd)
1147 .
Case(
"fmul", Intrinsic::vector_reduce_fmul)
1152 auto Args =
F->getFunctionType()->params();
1163 if (
Name.starts_with(
"flt.rounds")) {
1170 if (
Name.starts_with(
"invariant.group.barrier")) {
1172 auto Args =
F->getFunctionType()->params();
1173 Type* ObjectPtr[1] = {Args[0]};
1176 Intrinsic::launder_invariant_group, ObjectPtr);
1185 .StartsWith(
"memcpy.", Intrinsic::memcpy)
1186 .StartsWith(
"memmove.", Intrinsic::memmove)
1188 if (
F->arg_size() == 5) {
1192 F->getFunctionType()->params().slice(0, 3);
1197 if (
Name.starts_with(
"memset.") &&
F->arg_size() == 5) {
1200 const auto *FT =
F->getFunctionType();
1201 Type *ParamTypes[2] = {
1202 FT->getParamType(0),
1212 if (
Name.consume_front(
"nvvm.")) {
1214 if (
F->arg_size() == 1) {
1217 .
Cases(
"brev32",
"brev64", Intrinsic::bitreverse)
1218 .
Case(
"clz.i", Intrinsic::ctlz)
1219 .
Case(
"popc.i", Intrinsic::ctpop)
1223 {F->getReturnType()});
1229 if (!
F->getReturnType()->getScalarType()->isBFloatTy()) {
1241 bool Expand =
false;
1242 if (
Name.consume_front(
"abs."))
1244 Expand =
Name ==
"i" ||
Name ==
"ll";
1245 else if (
Name ==
"clz.ll" ||
Name ==
"popc.ll" ||
Name ==
"h2f")
1247 else if (
Name.consume_front(
"max.") ||
Name.consume_front(
"min."))
1251 else if (
Name.consume_front(
"atomic.load.add."))
1253 Expand =
Name.starts_with(
"f32.p") ||
Name.starts_with(
"f64.p");
1268 if (
Name.starts_with(
"objectsize.")) {
1269 Type *Tys[2] = {
F->getReturnType(),
F->arg_begin()->getType() };
1270 if (
F->arg_size() == 2 ||
F->arg_size() == 3 ||
1282 if (
Name.starts_with(
"ptr.annotation.") &&
F->arg_size() == 4) {
1285 F->getParent(), Intrinsic::ptr_annotation,
1286 {F->arg_begin()->getType(), F->getArg(1)->getType()});
1292 if (
Name.consume_front(
"riscv.")) {
1295 .
Case(
"aes32dsi", Intrinsic::riscv_aes32dsi)
1296 .
Case(
"aes32dsmi", Intrinsic::riscv_aes32dsmi)
1297 .
Case(
"aes32esi", Intrinsic::riscv_aes32esi)
1298 .
Case(
"aes32esmi", Intrinsic::riscv_aes32esmi)
1301 if (!
F->getFunctionType()->getParamType(2)->isIntegerTy(32)) {
1314 if (!
F->getFunctionType()->getParamType(2)->isIntegerTy(32) ||
1315 F->getFunctionType()->getReturnType()->isIntegerTy(64)) {
1324 .
StartsWith(
"sha256sig0", Intrinsic::riscv_sha256sig0)
1325 .
StartsWith(
"sha256sig1", Intrinsic::riscv_sha256sig1)
1326 .
StartsWith(
"sha256sum0", Intrinsic::riscv_sha256sum0)
1327 .
StartsWith(
"sha256sum1", Intrinsic::riscv_sha256sum1)
1332 if (
F->getFunctionType()->getReturnType()->isIntegerTy(64)) {
1344 if (
Name ==
"stackprotectorcheck") {
1351 if (
Name ==
"var.annotation" &&
F->arg_size() == 4) {
1354 F->getParent(), Intrinsic::var_annotation,
1355 {{F->arg_begin()->getType(), F->getArg(1)->getType()}});
1362 if (
Name.consume_front(
"wasm.")) {
1365 .
StartsWith(
"fma.", Intrinsic::wasm_relaxed_madd)
1366 .
StartsWith(
"fms.", Intrinsic::wasm_relaxed_nmadd)
1367 .
StartsWith(
"laneselect.", Intrinsic::wasm_relaxed_laneselect)
1376 if (
Name.consume_front(
"dot.i8x16.i7x16.")) {
1378 .
Case(
"signed", Intrinsic::wasm_relaxed_dot_i8x16_i7x16_signed)
1380 Intrinsic::wasm_relaxed_dot_i8x16_i7x16_add_signed)
1398 auto *
ST = dyn_cast<StructType>(
F->getReturnType());
1399 if (ST && (!
ST->isLiteral() ||
ST->isPacked()) &&
1408 auto *FT =
F->getFunctionType();
1410 auto *NewFT = FunctionType::get(NewST, FT->params(), FT->isVarArg());
1411 std::string
Name =
F->getName().str();
1414 Name,
F->getParent());
1425 if (Result != std::nullopt) {
1438 bool CanUpgradeDebugIntrinsicsToRecords) {
1442 assert(
F != NewFn &&
"Intrinsic function upgraded to the same function");
1454 GV->
getName() ==
"llvm.global_dtors")) ||
1469 unsigned N =
Init->getNumOperands();
1470 std::vector<Constant *> NewCtors(
N);
1471 for (
unsigned i = 0; i !=
N; ++i) {
1472 auto Ctor = cast<Constant>(
Init->getOperand(i));
1474 Ctor->getAggregateElement(1),
1487 auto *ResultTy = cast<FixedVectorType>(
Op->getType());
1488 unsigned NumElts = ResultTy->getNumElements() * 8;
1502 for (
unsigned l = 0; l != NumElts; l += 16)
1503 for (
unsigned i = 0; i != 16; ++i) {
1504 unsigned Idx = NumElts + i - Shift;
1506 Idx -= NumElts - 16;
1507 Idxs[l + i] =
Idx + l;
1521 auto *ResultTy = cast<FixedVectorType>(
Op->getType());
1522 unsigned NumElts = ResultTy->getNumElements() * 8;
1536 for (
unsigned l = 0; l != NumElts; l += 16)
1537 for (
unsigned i = 0; i != 16; ++i) {
1538 unsigned Idx = i + Shift;
1540 Idx += NumElts - 16;
1541 Idxs[l + i] =
Idx + l;
1562 for (
unsigned i = 0; i != NumElts; ++i)
1574 if (
const auto *
C = dyn_cast<Constant>(Mask))
1575 if (
C->isAllOnesValue())
1579 cast<FixedVectorType>(Op0->
getType())->getNumElements());
1586 if (
const auto *
C = dyn_cast<Constant>(Mask))
1587 if (
C->isAllOnesValue())
1591 Mask->getType()->getIntegerBitWidth());
1604 unsigned ShiftVal = cast<llvm::ConstantInt>(Shift)->getZExtValue();
1606 unsigned NumElts = cast<FixedVectorType>(Op0->
getType())->getNumElements();
1607 assert((IsVALIGN || NumElts % 16 == 0) &&
"Illegal NumElts for PALIGNR!");
1608 assert((!IsVALIGN || NumElts <= 16) &&
"NumElts too large for VALIGN!");
1613 ShiftVal &= (NumElts - 1);
1622 if (ShiftVal > 16) {
1630 for (
unsigned l = 0; l < NumElts; l += 16) {
1631 for (
unsigned i = 0; i != 16; ++i) {
1632 unsigned Idx = ShiftVal + i;
1633 if (!IsVALIGN &&
Idx >= 16)
1634 Idx += NumElts - 16;
1635 Indices[l + i] =
Idx + l;
1640 Op1, Op0,
ArrayRef(Indices, NumElts),
"palignr");
1646 bool ZeroMask,
bool IndexForm) {
1652 if (VecWidth == 128 && EltWidth == 32 && IsFloat)
1653 IID = Intrinsic::x86_avx512_vpermi2var_ps_128;
1654 else if (VecWidth == 128 && EltWidth == 32 && !IsFloat)
1655 IID = Intrinsic::x86_avx512_vpermi2var_d_128;
1656 else if (VecWidth == 128 && EltWidth == 64 && IsFloat)
1657 IID = Intrinsic::x86_avx512_vpermi2var_pd_128;
1658 else if (VecWidth == 128 && EltWidth == 64 && !IsFloat)
1659 IID = Intrinsic::x86_avx512_vpermi2var_q_128;
1660 else if (VecWidth == 256 && EltWidth == 32 && IsFloat)
1661 IID = Intrinsic::x86_avx512_vpermi2var_ps_256;
1662 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
1663 IID = Intrinsic::x86_avx512_vpermi2var_d_256;
1664 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
1665 IID = Intrinsic::x86_avx512_vpermi2var_pd_256;
1666 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
1667 IID = Intrinsic::x86_avx512_vpermi2var_q_256;
1668 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
1669 IID = Intrinsic::x86_avx512_vpermi2var_ps_512;
1670 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
1671 IID = Intrinsic::x86_avx512_vpermi2var_d_512;
1672 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
1673 IID = Intrinsic::x86_avx512_vpermi2var_pd_512;
1674 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
1675 IID = Intrinsic::x86_avx512_vpermi2var_q_512;
1676 else if (VecWidth == 128 && EltWidth == 16)
1677 IID = Intrinsic::x86_avx512_vpermi2var_hi_128;
1678 else if (VecWidth == 256 && EltWidth == 16)
1679 IID = Intrinsic::x86_avx512_vpermi2var_hi_256;
1680 else if (VecWidth == 512 && EltWidth == 16)
1681 IID = Intrinsic::x86_avx512_vpermi2var_hi_512;
1682 else if (VecWidth == 128 && EltWidth == 8)
1683 IID = Intrinsic::x86_avx512_vpermi2var_qi_128;
1684 else if (VecWidth == 256 && EltWidth == 8)
1685 IID = Intrinsic::x86_avx512_vpermi2var_qi_256;
1686 else if (VecWidth == 512 && EltWidth == 8)
1687 IID = Intrinsic::x86_avx512_vpermi2var_qi_512;
1723 bool IsRotateRight) {
1732 unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
1737 Intrinsic::ID IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
1758 Pred = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
1761 Pred = IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
1764 Pred = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
1767 Pred = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
1770 Pred = ICmpInst::ICMP_EQ;
1773 Pred = ICmpInst::ICMP_NE;
1789 bool IsShiftRight,
bool ZeroMask) {
1802 unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
1807 Intrinsic::ID IID = IsShiftRight ? Intrinsic::fshr : Intrinsic::fshl;
1827 const Align Alignment =
1829 ?
Align(Data->getType()->getPrimitiveSizeInBits().getFixedValue() / 8)
1833 if (
const auto *
C = dyn_cast<Constant>(Mask))
1834 if (
C->isAllOnesValue())
1838 unsigned NumElts = cast<FixedVectorType>(Data->getType())->getNumElements();
1848 const Align Alignment =
1856 if (
const auto *
C = dyn_cast<Constant>(Mask))
1857 if (
C->isAllOnesValue())
1861 unsigned NumElts = cast<FixedVectorType>(ValTy)->getNumElements();
1885 Constant *ShiftAmt = ConstantInt::get(Ty, 32);
1892 Constant *Mask = ConstantInt::get(Ty, 0xffffffff);
1908 unsigned NumElts = cast<FixedVectorType>(Vec->
getType())->getNumElements();
1910 const auto *
C = dyn_cast<Constant>(Mask);
1911 if (!
C || !
C->isAllOnesValue())
1917 for (
unsigned i = 0; i != NumElts; ++i)
1919 for (
unsigned i = NumElts; i != 8; ++i)
1920 Indices[i] = NumElts + i % NumElts;
1931 unsigned NumElts = cast<FixedVectorType>(Op0->
getType())->getNumElements();
1937 }
else if (
CC == 7) {
1944 case 0: Pred = ICmpInst::ICMP_EQ;
break;
1945 case 1: Pred =
Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
break;
1946 case 2: Pred =
Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
break;
1947 case 4: Pred = ICmpInst::ICMP_NE;
break;
1948 case 5: Pred =
Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
break;
1949 case 6: Pred =
Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
break;
1985 unsigned NumElts = cast<FixedVectorType>(CI.
getType())->getNumElements();
1987 return Builder.
CreateSExt(Mask, ReturnOp,
"vpmovm2");
1998 if (
Name.starts_with(
"max.p")) {
1999 if (VecWidth == 128 && EltWidth == 32)
2000 IID = Intrinsic::x86_sse_max_ps;
2001 else if (VecWidth == 128 && EltWidth == 64)
2002 IID = Intrinsic::x86_sse2_max_pd;
2003 else if (VecWidth == 256 && EltWidth == 32)
2004 IID = Intrinsic::x86_avx_max_ps_256;
2005 else if (VecWidth == 256 && EltWidth == 64)
2006 IID = Intrinsic::x86_avx_max_pd_256;
2009 }
else if (
Name.starts_with(
"min.p")) {
2010 if (VecWidth == 128 && EltWidth == 32)
2011 IID = Intrinsic::x86_sse_min_ps;
2012 else if (VecWidth == 128 && EltWidth == 64)
2013 IID = Intrinsic::x86_sse2_min_pd;
2014 else if (VecWidth == 256 && EltWidth == 32)
2015 IID = Intrinsic::x86_avx_min_ps_256;
2016 else if (VecWidth == 256 && EltWidth == 64)
2017 IID = Intrinsic::x86_avx_min_pd_256;
2020 }
else if (
Name.starts_with(
"pshuf.b.")) {
2021 if (VecWidth == 128)
2022 IID = Intrinsic::x86_ssse3_pshuf_b_128;
2023 else if (VecWidth == 256)
2024 IID = Intrinsic::x86_avx2_pshuf_b;
2025 else if (VecWidth == 512)
2026 IID = Intrinsic::x86_avx512_pshuf_b_512;
2029 }
else if (
Name.starts_with(
"pmul.hr.sw.")) {
2030 if (VecWidth == 128)
2031 IID = Intrinsic::x86_ssse3_pmul_hr_sw_128;
2032 else if (VecWidth == 256)
2033 IID = Intrinsic::x86_avx2_pmul_hr_sw;
2034 else if (VecWidth == 512)
2035 IID = Intrinsic::x86_avx512_pmul_hr_sw_512;
2038 }
else if (
Name.starts_with(
"pmulh.w.")) {
2039 if (VecWidth == 128)
2040 IID = Intrinsic::x86_sse2_pmulh_w;
2041 else if (VecWidth == 256)
2042 IID = Intrinsic::x86_avx2_pmulh_w;
2043 else if (VecWidth == 512)
2044 IID = Intrinsic::x86_avx512_pmulh_w_512;
2047 }
else if (
Name.starts_with(
"pmulhu.w.")) {
2048 if (VecWidth == 128)
2049 IID = Intrinsic::x86_sse2_pmulhu_w;
2050 else if (VecWidth == 256)
2051 IID = Intrinsic::x86_avx2_pmulhu_w;
2052 else if (VecWidth == 512)
2053 IID = Intrinsic::x86_avx512_pmulhu_w_512;
2056 }
else if (
Name.starts_with(
"pmaddw.d.")) {
2057 if (VecWidth == 128)
2058 IID = Intrinsic::x86_sse2_pmadd_wd;
2059 else if (VecWidth == 256)
2060 IID = Intrinsic::x86_avx2_pmadd_wd;
2061 else if (VecWidth == 512)
2062 IID = Intrinsic::x86_avx512_pmaddw_d_512;
2065 }
else if (
Name.starts_with(
"pmaddubs.w.")) {
2066 if (VecWidth == 128)
2067 IID = Intrinsic::x86_ssse3_pmadd_ub_sw_128;
2068 else if (VecWidth == 256)
2069 IID = Intrinsic::x86_avx2_pmadd_ub_sw;
2070 else if (VecWidth == 512)
2071 IID = Intrinsic::x86_avx512_pmaddubs_w_512;
2074 }
else if (
Name.starts_with(
"packsswb.")) {
2075 if (VecWidth == 128)
2076 IID = Intrinsic::x86_sse2_packsswb_128;
2077 else if (VecWidth == 256)
2078 IID = Intrinsic::x86_avx2_packsswb;
2079 else if (VecWidth == 512)
2080 IID = Intrinsic::x86_avx512_packsswb_512;
2083 }
else if (
Name.starts_with(
"packssdw.")) {
2084 if (VecWidth == 128)
2085 IID = Intrinsic::x86_sse2_packssdw_128;
2086 else if (VecWidth == 256)
2087 IID = Intrinsic::x86_avx2_packssdw;
2088 else if (VecWidth == 512)
2089 IID = Intrinsic::x86_avx512_packssdw_512;
2092 }
else if (
Name.starts_with(
"packuswb.")) {
2093 if (VecWidth == 128)
2094 IID = Intrinsic::x86_sse2_packuswb_128;
2095 else if (VecWidth == 256)
2096 IID = Intrinsic::x86_avx2_packuswb;
2097 else if (VecWidth == 512)
2098 IID = Intrinsic::x86_avx512_packuswb_512;
2101 }
else if (
Name.starts_with(
"packusdw.")) {
2102 if (VecWidth == 128)
2103 IID = Intrinsic::x86_sse41_packusdw;
2104 else if (VecWidth == 256)
2105 IID = Intrinsic::x86_avx2_packusdw;
2106 else if (VecWidth == 512)
2107 IID = Intrinsic::x86_avx512_packusdw_512;
2110 }
else if (
Name.starts_with(
"vpermilvar.")) {
2111 if (VecWidth == 128 && EltWidth == 32)
2112 IID = Intrinsic::x86_avx_vpermilvar_ps;
2113 else if (VecWidth == 128 && EltWidth == 64)
2114 IID = Intrinsic::x86_avx_vpermilvar_pd;
2115 else if (VecWidth == 256 && EltWidth == 32)
2116 IID = Intrinsic::x86_avx_vpermilvar_ps_256;
2117 else if (VecWidth == 256 && EltWidth == 64)
2118 IID = Intrinsic::x86_avx_vpermilvar_pd_256;
2119 else if (VecWidth == 512 && EltWidth == 32)
2120 IID = Intrinsic::x86_avx512_vpermilvar_ps_512;
2121 else if (VecWidth == 512 && EltWidth == 64)
2122 IID = Intrinsic::x86_avx512_vpermilvar_pd_512;
2125 }
else if (
Name ==
"cvtpd2dq.256") {
2126 IID = Intrinsic::x86_avx_cvt_pd2dq_256;
2127 }
else if (
Name ==
"cvtpd2ps.256") {
2128 IID = Intrinsic::x86_avx_cvt_pd2_ps_256;
2129 }
else if (
Name ==
"cvttpd2dq.256") {
2130 IID = Intrinsic::x86_avx_cvtt_pd2dq_256;
2131 }
else if (
Name ==
"cvttps2dq.128") {
2132 IID = Intrinsic::x86_sse2_cvttps2dq;
2133 }
else if (
Name ==
"cvttps2dq.256") {
2134 IID = Intrinsic::x86_avx_cvtt_ps2dq_256;
2135 }
else if (
Name.starts_with(
"permvar.")) {
2137 if (VecWidth == 256 && EltWidth == 32 && IsFloat)
2138 IID = Intrinsic::x86_avx2_permps;
2139 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
2140 IID = Intrinsic::x86_avx2_permd;
2141 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
2142 IID = Intrinsic::x86_avx512_permvar_df_256;
2143 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
2144 IID = Intrinsic::x86_avx512_permvar_di_256;
2145 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
2146 IID = Intrinsic::x86_avx512_permvar_sf_512;
2147 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
2148 IID = Intrinsic::x86_avx512_permvar_si_512;
2149 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
2150 IID = Intrinsic::x86_avx512_permvar_df_512;
2151 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
2152 IID = Intrinsic::x86_avx512_permvar_di_512;
2153 else if (VecWidth == 128 && EltWidth == 16)
2154 IID = Intrinsic::x86_avx512_permvar_hi_128;
2155 else if (VecWidth == 256 && EltWidth == 16)
2156 IID = Intrinsic::x86_avx512_permvar_hi_256;
2157 else if (VecWidth == 512 && EltWidth == 16)
2158 IID = Intrinsic::x86_avx512_permvar_hi_512;
2159 else if (VecWidth == 128 && EltWidth == 8)
2160 IID = Intrinsic::x86_avx512_permvar_qi_128;
2161 else if (VecWidth == 256 && EltWidth == 8)
2162 IID = Intrinsic::x86_avx512_permvar_qi_256;
2163 else if (VecWidth == 512 && EltWidth == 8)
2164 IID = Intrinsic::x86_avx512_permvar_qi_512;
2167 }
else if (
Name.starts_with(
"dbpsadbw.")) {
2168 if (VecWidth == 128)
2169 IID = Intrinsic::x86_avx512_dbpsadbw_128;
2170 else if (VecWidth == 256)
2171 IID = Intrinsic::x86_avx512_dbpsadbw_256;
2172 else if (VecWidth == 512)
2173 IID = Intrinsic::x86_avx512_dbpsadbw_512;
2176 }
else if (
Name.starts_with(
"pmultishift.qb.")) {
2177 if (VecWidth == 128)
2178 IID = Intrinsic::x86_avx512_pmultishift_qb_128;
2179 else if (VecWidth == 256)
2180 IID = Intrinsic::x86_avx512_pmultishift_qb_256;
2181 else if (VecWidth == 512)
2182 IID = Intrinsic::x86_avx512_pmultishift_qb_512;
2185 }
else if (
Name.starts_with(
"conflict.")) {
2186 if (
Name[9] ==
'd' && VecWidth == 128)
2187 IID = Intrinsic::x86_avx512_conflict_d_128;
2188 else if (
Name[9] ==
'd' && VecWidth == 256)
2189 IID = Intrinsic::x86_avx512_conflict_d_256;
2190 else if (
Name[9] ==
'd' && VecWidth == 512)
2191 IID = Intrinsic::x86_avx512_conflict_d_512;
2192 else if (
Name[9] ==
'q' && VecWidth == 128)
2193 IID = Intrinsic::x86_avx512_conflict_q_128;
2194 else if (
Name[9] ==
'q' && VecWidth == 256)
2195 IID = Intrinsic::x86_avx512_conflict_q_256;
2196 else if (
Name[9] ==
'q' && VecWidth == 512)
2197 IID = Intrinsic::x86_avx512_conflict_q_512;
2200 }
else if (
Name.starts_with(
"pavg.")) {
2201 if (
Name[5] ==
'b' && VecWidth == 128)
2202 IID = Intrinsic::x86_sse2_pavg_b;
2203 else if (
Name[5] ==
'b' && VecWidth == 256)
2204 IID = Intrinsic::x86_avx2_pavg_b;
2205 else if (
Name[5] ==
'b' && VecWidth == 512)
2206 IID = Intrinsic::x86_avx512_pavg_b_512;
2207 else if (
Name[5] ==
'w' && VecWidth == 128)
2208 IID = Intrinsic::x86_sse2_pavg_w;
2209 else if (
Name[5] ==
'w' && VecWidth == 256)
2210 IID = Intrinsic::x86_avx2_pavg_w;
2211 else if (
Name[5] ==
'w' && VecWidth == 512)
2212 IID = Intrinsic::x86_avx512_pavg_w_512;
2233 if (AsmStr->find(
"mov\tfp") == 0 &&
2234 AsmStr->find(
"objc_retainAutoreleaseReturnValue") != std::string::npos &&
2235 (Pos = AsmStr->find(
"# marker")) != std::string::npos) {
2236 AsmStr->replace(Pos, 1,
";");
2242 if (
Name ==
"mve.vctp64.old") {
2250 F->getParent(), Intrinsic::arm_mve_pred_v2i,
2251 {VectorType::get(Builder.getInt1Ty(), 2, false)}),
2255 F->getParent(), Intrinsic::arm_mve_pred_i2v,
2256 {VectorType::get(Builder.getInt1Ty(), 4, false)}),
2258 }
else if (
Name ==
"mve.mull.int.predicated.v2i64.v4i32.v4i1" ||
2259 Name ==
"mve.vqdmull.predicated.v2i64.v4i32.v4i1" ||
2260 Name ==
"mve.vldr.gather.base.predicated.v2i64.v2i64.v4i1" ||
2261 Name ==
"mve.vldr.gather.base.wb.predicated.v2i64.v2i64.v4i1" ||
2263 "mve.vldr.gather.offset.predicated.v2i64.p0i64.v2i64.v4i1" ||
2264 Name ==
"mve.vldr.gather.offset.predicated.v2i64.p0.v2i64.v4i1" ||
2265 Name ==
"mve.vstr.scatter.base.predicated.v2i64.v2i64.v4i1" ||
2266 Name ==
"mve.vstr.scatter.base.wb.predicated.v2i64.v2i64.v4i1" ||
2268 "mve.vstr.scatter.offset.predicated.p0i64.v2i64.v2i64.v4i1" ||
2269 Name ==
"mve.vstr.scatter.offset.predicated.p0.v2i64.v2i64.v4i1" ||
2270 Name ==
"cde.vcx1q.predicated.v2i64.v4i1" ||
2271 Name ==
"cde.vcx1qa.predicated.v2i64.v4i1" ||
2272 Name ==
"cde.vcx2q.predicated.v2i64.v4i1" ||
2273 Name ==
"cde.vcx2qa.predicated.v2i64.v4i1" ||
2274 Name ==
"cde.vcx3q.predicated.v2i64.v4i1" ||
2275 Name ==
"cde.vcx3qa.predicated.v2i64.v4i1") {
2276 std::vector<Type *> Tys;
2280 case Intrinsic::arm_mve_mull_int_predicated:
2281 case Intrinsic::arm_mve_vqdmull_predicated:
2282 case Intrinsic::arm_mve_vldr_gather_base_predicated:
2285 case Intrinsic::arm_mve_vldr_gather_base_wb_predicated:
2286 case Intrinsic::arm_mve_vstr_scatter_base_predicated:
2287 case Intrinsic::arm_mve_vstr_scatter_base_wb_predicated:
2291 case Intrinsic::arm_mve_vldr_gather_offset_predicated:
2295 case Intrinsic::arm_mve_vstr_scatter_offset_predicated:
2299 case Intrinsic::arm_cde_vcx1q_predicated:
2300 case Intrinsic::arm_cde_vcx1qa_predicated:
2301 case Intrinsic::arm_cde_vcx2q_predicated:
2302 case Intrinsic::arm_cde_vcx2qa_predicated:
2303 case Intrinsic::arm_cde_vcx3q_predicated:
2304 case Intrinsic::arm_cde_vcx3qa_predicated:
2311 std::vector<Value *> Ops;
2313 Type *Ty =
Op->getType();
2317 F->getParent(), Intrinsic::arm_mve_pred_v2i,
2318 {VectorType::get(Builder.getInt1Ty(), 4, false)}),
2322 Intrinsic::arm_mve_pred_i2v, {V2I1Ty}),
2336 const bool IsInc =
Name.starts_with(
"atomic.inc.");
2337 if (IsInc ||
Name.starts_with(
"atomic.dec.")) {
2352 if (Order == AtomicOrdering::NotAtomic ||
2353 Order == AtomicOrdering::Unordered)
2354 Order = AtomicOrdering::SequentiallyConsistent;
2358 SyncScope::ID SSID =
F->getContext().getOrInsertSyncScopeID(
"agent");
2362 if (!VolatileArg || !VolatileArg->
isZero())
2371template <
typename MDType>
2374 return dyn_cast<MDType>(MAV->getMetadata());
2383 if (
Name ==
"label") {
2385 }
else if (
Name ==
"assign") {
2387 unwrapMAVOp<Metadata>(CI, 0), unwrapMAVOp<DILocalVariable>(CI, 1),
2388 unwrapMAVOp<DIExpression>(CI, 2), unwrapMAVOp<DIAssignID>(CI, 3),
2389 unwrapMAVOp<Metadata>(CI, 4), unwrapMAVOp<DIExpression>(CI, 5),
2391 }
else if (
Name ==
"declare") {
2393 unwrapMAVOp<Metadata>(CI, 0), unwrapMAVOp<DILocalVariable>(CI, 1),
2394 unwrapMAVOp<DIExpression>(CI, 2), CI->
getDebugLoc(),
2395 DbgVariableRecord::LocationType::Declare);
2396 }
else if (
Name ==
"addr") {
2401 unwrapMAVOp<DILocalVariable>(CI, 1), Expr,
2403 }
else if (
Name ==
"value") {
2406 unsigned ExprOp = 2;
2416 unwrapMAVOp<Metadata>(CI, 0), unwrapMAVOp<DILocalVariable>(CI, VarOp),
2417 unwrapMAVOp<DIExpression>(CI, ExprOp), CI->
getDebugLoc());
2419 assert(DR &&
"Unhandled intrinsic kind in upgrade to DbgRecord");
2438 bool FallthroughToDefaultUpgrade =
false;
2442 assert(
Name.starts_with(
"llvm.") &&
"Intrinsic doesn't start with 'llvm.'");
2445 bool IsX86 =
Name.consume_front(
"x86.");
2446 bool IsNVVM =
Name.consume_front(
"nvvm.");
2447 bool IsARM =
Name.consume_front(
"arm.");
2448 bool IsAMDGCN =
Name.consume_front(
"amdgcn.");
2449 bool IsDbg =
Name.consume_front(
"dbg.");
2451 if (IsX86 &&
Name.starts_with(
"sse4a.movnt.")) {
2462 Type *SrcEltTy = cast<VectorType>(Arg1->
getType())->getElementType();
2463 PointerType *EltPtrTy = PointerType::getUnqual(SrcEltTy);
2469 SI->setMetadata(LLVMContext::MD_nontemporal, Node);
2476 if (IsX86 && (
Name.starts_with(
"avx.movnt.") ||
2477 Name.starts_with(
"avx512.storent."))) {
2488 PointerType::getUnqual(Arg1->
getType()),
2493 SI->setMetadata(LLVMContext::MD_nontemporal, Node);
2500 if (IsX86 &&
Name ==
"sse2.storel.dq") {
2508 PointerType::getUnqual(Elt->
getType()),
2517 if (IsX86 && (
Name.starts_with(
"sse.storeu.") ||
2518 Name.starts_with(
"sse2.storeu.") ||
2519 Name.starts_with(
"avx.storeu."))) {
2524 PointerType::getUnqual(Arg1->
getType()),
2533 if (IsX86 &&
Name ==
"avx512.mask.store.ss") {
2543 if (IsX86 &&
Name.starts_with(
"avx512.mask.store")) {
2554 Value *Rep =
nullptr;
2556 if (IsX86 && (
Name.starts_with(
"sse2.pcmp") ||
2557 Name.starts_with(
"avx2.pcmp"))) {
2559 bool CmpEq =
Name[9] ==
'e';
2560 Rep = Builder.
CreateICmp(CmpEq ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_SGT,
2563 }
else if (IsX86 && (
Name.starts_with(
"avx512.broadcastm"))) {
2571 }
else if (IsX86 && (
Name ==
"sse.sqrt.ss" ||
2572 Name ==
"sse2.sqrt.sd")) {
2576 Intrinsic::sqrt, Elt0->
getType());
2579 }
else if (IsX86 && (
Name.starts_with(
"avx.sqrt.p") ||
2580 Name.starts_with(
"sse2.sqrt.p") ||
2581 Name.starts_with(
"sse.sqrt.p"))) {
2585 {CI->getArgOperand(0)});
2586 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.sqrt.p"))) {
2589 cast<ConstantInt>(CI->
getArgOperand(3))->getZExtValue() != 4)) {
2591 : Intrinsic::x86_avx512_sqrt_pd_512;
2600 {CI->getArgOperand(0)});
2604 }
else if (IsX86 && (
Name.starts_with(
"avx512.ptestm") ||
2605 Name.starts_with(
"avx512.ptestnm"))) {
2613 Name.starts_with(
"avx512.ptestm") ? ICmpInst::ICMP_NE : ICmpInst::ICMP_EQ;
2616 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.pbroadcast"))){
2622 }
else if (IsX86 && (
Name.starts_with(
"avx512.kunpck"))) {
2627 for (
unsigned i = 0; i != NumElts; ++i)
2640 }
else if (IsX86 &&
Name ==
"avx512.kand.w") {
2645 }
else if (IsX86 &&
Name ==
"avx512.kandn.w") {
2651 }
else if (IsX86 &&
Name ==
"avx512.kor.w") {
2656 }
else if (IsX86 &&
Name ==
"avx512.kxor.w") {
2661 }
else if (IsX86 &&
Name ==
"avx512.kxnor.w") {
2667 }
else if (IsX86 &&
Name ==
"avx512.knot.w") {
2672 (
Name ==
"avx512.kortestz.w" ||
Name ==
"avx512.kortestc.w")) {
2678 if (
Name[14] ==
'c')
2679 C = ConstantInt::getAllOnesValue(Builder.
getInt16Ty());
2681 C = ConstantInt::getNullValue(Builder.
getInt16Ty());
2684 }
else if (IsX86 && (
Name ==
"sse.add.ss" ||
Name ==
"sse2.add.sd" ||
2685 Name ==
"sse.sub.ss" ||
Name ==
"sse2.sub.sd" ||
2686 Name ==
"sse.mul.ss" ||
Name ==
"sse2.mul.sd" ||
2687 Name ==
"sse.div.ss" ||
Name ==
"sse2.div.sd")) {
2690 ConstantInt::get(I32Ty, 0));
2692 ConstantInt::get(I32Ty, 0));
2694 if (
Name.contains(
".add."))
2696 else if (
Name.contains(
".sub."))
2698 else if (
Name.contains(
".mul."))
2703 ConstantInt::get(I32Ty, 0));
2704 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.pcmp")) {
2706 bool CmpEq =
Name[16] ==
'e';
2708 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.vpshufbitqmb.")) {
2714 case 128: IID = Intrinsic::x86_avx512_vpshufbitqmb_128;
break;
2715 case 256: IID = Intrinsic::x86_avx512_vpshufbitqmb_256;
break;
2716 case 512: IID = Intrinsic::x86_avx512_vpshufbitqmb_512;
break;
2720 { CI->getOperand(0), CI->getArgOperand(1) });
2722 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.fpclass.p")) {
2727 if (VecWidth == 128 && EltWidth == 32)
2728 IID = Intrinsic::x86_avx512_fpclass_ps_128;
2729 else if (VecWidth == 256 && EltWidth == 32)
2730 IID = Intrinsic::x86_avx512_fpclass_ps_256;
2731 else if (VecWidth == 512 && EltWidth == 32)
2732 IID = Intrinsic::x86_avx512_fpclass_ps_512;
2733 else if (VecWidth == 128 && EltWidth == 64)
2734 IID = Intrinsic::x86_avx512_fpclass_pd_128;
2735 else if (VecWidth == 256 && EltWidth == 64)
2736 IID = Intrinsic::x86_avx512_fpclass_pd_256;
2737 else if (VecWidth == 512 && EltWidth == 64)
2738 IID = Intrinsic::x86_avx512_fpclass_pd_512;
2743 { CI->getOperand(0), CI->getArgOperand(1) });
2745 }
else if (IsX86 &&
Name.starts_with(
"avx512.cmp.p")) {
2747 Type *OpTy = Args[0]->getType();
2751 if (VecWidth == 128 && EltWidth == 32)
2752 IID = Intrinsic::x86_avx512_mask_cmp_ps_128;
2753 else if (VecWidth == 256 && EltWidth == 32)
2754 IID = Intrinsic::x86_avx512_mask_cmp_ps_256;
2755 else if (VecWidth == 512 && EltWidth == 32)
2756 IID = Intrinsic::x86_avx512_mask_cmp_ps_512;
2757 else if (VecWidth == 128 && EltWidth == 64)
2758 IID = Intrinsic::x86_avx512_mask_cmp_pd_128;
2759 else if (VecWidth == 256 && EltWidth == 64)
2760 IID = Intrinsic::x86_avx512_mask_cmp_pd_256;
2761 else if (VecWidth == 512 && EltWidth == 64)
2762 IID = Intrinsic::x86_avx512_mask_cmp_pd_512;
2767 if (VecWidth == 512)
2769 Args.push_back(Mask);
2773 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.cmp.")) {
2775 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(2))->getZExtValue();
2777 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.ucmp.")) {
2778 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(2))->getZExtValue();
2780 }
else if (IsX86 && (
Name.starts_with(
"avx512.cvtb2mask.") ||
2781 Name.starts_with(
"avx512.cvtw2mask.") ||
2782 Name.starts_with(
"avx512.cvtd2mask.") ||
2783 Name.starts_with(
"avx512.cvtq2mask."))) {
2786 Rep = Builder.
CreateICmp(ICmpInst::ICMP_SLT,
Op, Zero);
2788 }
else if(IsX86 && (
Name ==
"ssse3.pabs.b.128" ||
2789 Name ==
"ssse3.pabs.w.128" ||
2790 Name ==
"ssse3.pabs.d.128" ||
2791 Name.starts_with(
"avx2.pabs") ||
2792 Name.starts_with(
"avx512.mask.pabs"))) {
2794 }
else if (IsX86 && (
Name ==
"sse41.pmaxsb" ||
2795 Name ==
"sse2.pmaxs.w" ||
2796 Name ==
"sse41.pmaxsd" ||
2797 Name.starts_with(
"avx2.pmaxs") ||
2798 Name.starts_with(
"avx512.mask.pmaxs"))) {
2800 }
else if (IsX86 && (
Name ==
"sse2.pmaxu.b" ||
2801 Name ==
"sse41.pmaxuw" ||
2802 Name ==
"sse41.pmaxud" ||
2803 Name.starts_with(
"avx2.pmaxu") ||
2804 Name.starts_with(
"avx512.mask.pmaxu"))) {
2806 }
else if (IsX86 && (
Name ==
"sse41.pminsb" ||
2807 Name ==
"sse2.pmins.w" ||
2808 Name ==
"sse41.pminsd" ||
2809 Name.starts_with(
"avx2.pmins") ||
2810 Name.starts_with(
"avx512.mask.pmins"))) {
2812 }
else if (IsX86 && (
Name ==
"sse2.pminu.b" ||
2813 Name ==
"sse41.pminuw" ||
2814 Name ==
"sse41.pminud" ||
2815 Name.starts_with(
"avx2.pminu") ||
2816 Name.starts_with(
"avx512.mask.pminu"))) {
2818 }
else if (IsX86 && (
Name ==
"sse2.pmulu.dq" ||
2819 Name ==
"avx2.pmulu.dq" ||
2820 Name ==
"avx512.pmulu.dq.512" ||
2821 Name.starts_with(
"avx512.mask.pmulu.dq."))) {
2823 }
else if (IsX86 && (
Name ==
"sse41.pmuldq" ||
2824 Name ==
"avx2.pmul.dq" ||
2825 Name ==
"avx512.pmul.dq.512" ||
2826 Name.starts_with(
"avx512.mask.pmul.dq."))) {
2828 }
else if (IsX86 && (
Name ==
"sse.cvtsi2ss" ||
2829 Name ==
"sse2.cvtsi2sd" ||
2830 Name ==
"sse.cvtsi642ss" ||
2831 Name ==
"sse2.cvtsi642sd")) {
2834 cast<VectorType>(CI->
getType())->getElementType());
2836 }
else if (IsX86 &&
Name ==
"avx512.cvtusi2sd") {
2839 cast<VectorType>(CI->
getType())->getElementType());
2841 }
else if (IsX86 &&
Name ==
"sse2.cvtss2sd") {
2844 Rep, cast<VectorType>(CI->
getType())->getElementType());
2846 }
else if (IsX86 && (
Name ==
"sse2.cvtdq2pd" ||
2847 Name ==
"sse2.cvtdq2ps" ||
2848 Name ==
"avx.cvtdq2.pd.256" ||
2849 Name ==
"avx.cvtdq2.ps.256" ||
2850 Name.starts_with(
"avx512.mask.cvtdq2pd.") ||
2851 Name.starts_with(
"avx512.mask.cvtudq2pd.") ||
2852 Name.starts_with(
"avx512.mask.cvtdq2ps.") ||
2853 Name.starts_with(
"avx512.mask.cvtudq2ps.") ||
2854 Name.starts_with(
"avx512.mask.cvtqq2pd.") ||
2855 Name.starts_with(
"avx512.mask.cvtuqq2pd.") ||
2856 Name ==
"avx512.mask.cvtqq2ps.256" ||
2857 Name ==
"avx512.mask.cvtqq2ps.512" ||
2858 Name ==
"avx512.mask.cvtuqq2ps.256" ||
2859 Name ==
"avx512.mask.cvtuqq2ps.512" ||
2860 Name ==
"sse2.cvtps2pd" ||
2861 Name ==
"avx.cvt.ps2.pd.256" ||
2862 Name ==
"avx512.mask.cvtps2pd.128" ||
2863 Name ==
"avx512.mask.cvtps2pd.256")) {
2864 auto *DstTy = cast<FixedVectorType>(CI->
getType());
2866 auto *SrcTy = cast<FixedVectorType>(Rep->
getType());
2868 unsigned NumDstElts = DstTy->getNumElements();
2869 if (NumDstElts < SrcTy->getNumElements()) {
2870 assert(NumDstElts == 2 &&
"Unexpected vector size");
2874 bool IsPS2PD = SrcTy->getElementType()->isFloatTy();
2875 bool IsUnsigned =
Name.contains(
"cvtu");
2877 Rep = Builder.
CreateFPExt(Rep, DstTy,
"cvtps2pd");
2880 cast<ConstantInt>(CI->
getArgOperand(3))->getZExtValue() != 4)) {
2881 Intrinsic::ID IID = IsUnsigned ? Intrinsic::x86_avx512_uitofp_round
2882 : Intrinsic::x86_avx512_sitofp_round;
2887 Rep = IsUnsigned ? Builder.
CreateUIToFP(Rep, DstTy,
"cvt")
2894 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.vcvtph2ps.") ||
2895 Name.starts_with(
"vcvtph2ps."))) {
2896 auto *DstTy = cast<FixedVectorType>(CI->
getType());
2898 auto *SrcTy = cast<FixedVectorType>(Rep->
getType());
2899 unsigned NumDstElts = DstTy->getNumElements();
2900 if (NumDstElts != SrcTy->getNumElements()) {
2901 assert(NumDstElts == 4 &&
"Unexpected vector size");
2906 Rep = Builder.
CreateFPExt(Rep, DstTy,
"cvtph2ps");
2910 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.load")) {
2916 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.expand.load.")) {
2917 auto *ResultTy = cast<FixedVectorType>(CI->
getType());
2918 Type *PtrTy = ResultTy->getElementType();
2925 ResultTy->getNumElements());
2928 Intrinsic::masked_expandload,
2931 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.compress.store.")) {
2933 Type *PtrTy = ResultTy->getElementType();
2941 cast<FixedVectorType>(ResultTy)->getNumElements());
2944 Intrinsic::masked_compressstore,
2947 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.compress.") ||
2948 Name.starts_with(
"avx512.mask.expand."))) {
2949 auto *ResultTy = cast<FixedVectorType>(CI->
getType());
2952 ResultTy->getNumElements());
2954 bool IsCompress =
Name[12] ==
'c';
2955 Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress
2956 : Intrinsic::x86_avx512_mask_expand;
2960 }
else if (IsX86 &&
Name.starts_with(
"xop.vpcom")) {
2962 if (
Name.ends_with(
"ub") ||
Name.ends_with(
"uw") ||
Name.ends_with(
"ud") ||
2963 Name.ends_with(
"uq"))
2965 else if (
Name.ends_with(
"b") ||
Name.ends_with(
"w") ||
Name.ends_with(
"d") ||
2966 Name.ends_with(
"q"))
2973 Imm = cast<ConstantInt>(CI->
getArgOperand(2))->getZExtValue();
2976 if (
Name.starts_with(
"lt"))
2978 else if (
Name.starts_with(
"le"))
2980 else if (
Name.starts_with(
"gt"))
2982 else if (
Name.starts_with(
"ge"))
2984 else if (
Name.starts_with(
"eq"))
2986 else if (
Name.starts_with(
"ne"))
2988 else if (
Name.starts_with(
"false"))
2990 else if (
Name.starts_with(
"true"))
2997 }
else if (IsX86 &&
Name.starts_with(
"xop.vpcmov")) {
3002 Rep = Builder.
CreateOr(Sel0, Sel1);
3003 }
else if (IsX86 && (
Name.starts_with(
"xop.vprot") ||
3004 Name.starts_with(
"avx512.prol") ||
3005 Name.starts_with(
"avx512.mask.prol"))) {
3007 }
else if (IsX86 && (
Name.starts_with(
"avx512.pror") ||
3008 Name.starts_with(
"avx512.mask.pror"))) {
3010 }
else if (IsX86 && (
Name.starts_with(
"avx512.vpshld.") ||
3011 Name.starts_with(
"avx512.mask.vpshld") ||
3012 Name.starts_with(
"avx512.maskz.vpshld"))) {
3013 bool ZeroMask =
Name[11] ==
'z';
3015 }
else if (IsX86 && (
Name.starts_with(
"avx512.vpshrd.") ||
3016 Name.starts_with(
"avx512.mask.vpshrd") ||
3017 Name.starts_with(
"avx512.maskz.vpshrd"))) {
3018 bool ZeroMask =
Name[11] ==
'z';
3020 }
else if (IsX86 &&
Name ==
"sse42.crc32.64.8") {
3022 Intrinsic::x86_sse42_crc32_32_8);
3026 }
else if (IsX86 && (
Name.starts_with(
"avx.vbroadcast.s") ||
3027 Name.starts_with(
"avx512.vbroadcast.s"))) {
3029 auto *VecTy = cast<FixedVectorType>(CI->
getType());
3030 Type *EltTy = VecTy->getElementType();
3031 unsigned EltNum = VecTy->getNumElements();
3035 for (
unsigned I = 0;
I < EltNum; ++
I)
3037 ConstantInt::get(I32Ty,
I));
3038 }
else if (IsX86 && (
Name.starts_with(
"sse41.pmovsx") ||
3039 Name.starts_with(
"sse41.pmovzx") ||
3040 Name.starts_with(
"avx2.pmovsx") ||
3041 Name.starts_with(
"avx2.pmovzx") ||
3042 Name.starts_with(
"avx512.mask.pmovsx") ||
3043 Name.starts_with(
"avx512.mask.pmovzx"))) {
3044 auto *DstTy = cast<FixedVectorType>(CI->
getType());
3045 unsigned NumDstElts = DstTy->getNumElements();
3049 for (
unsigned i = 0; i != NumDstElts; ++i)
3055 bool DoSext =
Name.contains(
"pmovsx");
3062 }
else if (
Name ==
"avx512.mask.pmov.qd.256" ||
3063 Name ==
"avx512.mask.pmov.qd.512" ||
3064 Name ==
"avx512.mask.pmov.wb.256" ||
3065 Name ==
"avx512.mask.pmov.wb.512") {
3070 }
else if (IsX86 && (
Name.starts_with(
"avx.vbroadcastf128") ||
3071 Name ==
"avx2.vbroadcasti128")) {
3073 Type *EltTy = cast<VectorType>(CI->
getType())->getElementType();
3077 PointerType::getUnqual(VT));
3079 if (NumSrcElts == 2)
3084 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.shuf.i") ||
3085 Name.starts_with(
"avx512.mask.shuf.f"))) {
3086 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(2))->getZExtValue();
3090 unsigned ControlBitsMask = NumLanes - 1;
3091 unsigned NumControlBits = NumLanes / 2;
3094 for (
unsigned l = 0; l != NumLanes; ++l) {
3095 unsigned LaneMask = (Imm >> (l * NumControlBits)) & ControlBitsMask;
3097 if (l >= NumLanes / 2)
3098 LaneMask += NumLanes;
3099 for (
unsigned i = 0; i != NumElementsInLane; ++i)
3100 ShuffleMask.
push_back(LaneMask * NumElementsInLane + i);
3106 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.broadcastf") ||
3107 Name.starts_with(
"avx512.mask.broadcasti"))) {
3108 unsigned NumSrcElts =
3111 unsigned NumDstElts =
3112 cast<FixedVectorType>(CI->
getType())->getNumElements();
3115 for (
unsigned i = 0; i != NumDstElts; ++i)
3116 ShuffleMask[i] = i % NumSrcElts;
3123 }
else if (IsX86 && (
Name.starts_with(
"avx2.pbroadcast") ||
3124 Name.starts_with(
"avx2.vbroadcast") ||
3125 Name.starts_with(
"avx512.pbroadcast") ||
3126 Name.starts_with(
"avx512.mask.broadcast.s"))) {
3138 }
else if (IsX86 && (
Name.starts_with(
"sse2.padds.") ||
3139 Name.starts_with(
"avx2.padds.") ||
3140 Name.starts_with(
"avx512.padds.") ||
3141 Name.starts_with(
"avx512.mask.padds."))) {
3143 }
else if (IsX86 && (
Name.starts_with(
"sse2.psubs.") ||
3144 Name.starts_with(
"avx2.psubs.") ||
3145 Name.starts_with(
"avx512.psubs.") ||
3146 Name.starts_with(
"avx512.mask.psubs."))) {
3148 }
else if (IsX86 && (
Name.starts_with(
"sse2.paddus.") ||
3149 Name.starts_with(
"avx2.paddus.") ||
3150 Name.starts_with(
"avx512.mask.paddus."))) {
3152 }
else if (IsX86 && (
Name.starts_with(
"sse2.psubus.") ||
3153 Name.starts_with(
"avx2.psubus.") ||
3154 Name.starts_with(
"avx512.mask.psubus."))) {
3156 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.palignr.")) {
3161 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.valign.")) {
3166 }
else if (IsX86 && (
Name ==
"sse2.psll.dq" ||
3167 Name ==
"avx2.psll.dq")) {
3169 unsigned Shift = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
3172 }
else if (IsX86 && (
Name ==
"sse2.psrl.dq" ||
3173 Name ==
"avx2.psrl.dq")) {
3175 unsigned Shift = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
3178 }
else if (IsX86 && (
Name ==
"sse2.psll.dq.bs" ||
3179 Name ==
"avx2.psll.dq.bs" ||
3180 Name ==
"avx512.psll.dq.512")) {
3182 unsigned Shift = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
3184 }
else if (IsX86 && (
Name ==
"sse2.psrl.dq.bs" ||
3185 Name ==
"avx2.psrl.dq.bs" ||
3186 Name ==
"avx512.psrl.dq.512")) {
3188 unsigned Shift = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
3190 }
else if (IsX86 && (
Name ==
"sse41.pblendw" ||
3191 Name.starts_with(
"sse41.blendp") ||
3192 Name.starts_with(
"avx.blend.p") ||
3193 Name ==
"avx2.pblendw" ||
3194 Name.starts_with(
"avx2.pblendd."))) {
3197 unsigned Imm = cast <ConstantInt>(CI->
getArgOperand(2))->getZExtValue();
3198 auto *VecTy = cast<FixedVectorType>(CI->
getType());
3199 unsigned NumElts = VecTy->getNumElements();
3202 for (
unsigned i = 0; i != NumElts; ++i)
3203 Idxs[i] = ((Imm >> (i%8)) & 1) ? i + NumElts : i;
3206 }
else if (IsX86 && (
Name.starts_with(
"avx.vinsertf128.") ||
3207 Name ==
"avx2.vinserti128" ||
3208 Name.starts_with(
"avx512.mask.insert"))) {
3211 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(2))->getZExtValue();
3212 unsigned DstNumElts =
3213 cast<FixedVectorType>(CI->
getType())->getNumElements();
3214 unsigned SrcNumElts =
3215 cast<FixedVectorType>(Op1->
getType())->getNumElements();
3216 unsigned Scale = DstNumElts / SrcNumElts;
3223 for (
unsigned i = 0; i != SrcNumElts; ++i)
3225 for (
unsigned i = SrcNumElts; i != DstNumElts; ++i)
3226 Idxs[i] = SrcNumElts;
3241 for (
unsigned i = 0; i != DstNumElts; ++i)
3244 for (
unsigned i = 0; i != SrcNumElts; ++i)
3245 Idxs[i + Imm * SrcNumElts] = i + DstNumElts;
3252 }
else if (IsX86 && (
Name.starts_with(
"avx.vextractf128.") ||
3253 Name ==
"avx2.vextracti128" ||
3254 Name.starts_with(
"avx512.mask.vextract"))) {
3256 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
3257 unsigned DstNumElts =
3258 cast<FixedVectorType>(CI->
getType())->getNumElements();
3259 unsigned SrcNumElts =
3260 cast<FixedVectorType>(Op0->
getType())->getNumElements();
3261 unsigned Scale = SrcNumElts / DstNumElts;
3268 for (
unsigned i = 0; i != DstNumElts; ++i) {
3269 Idxs[i] = i + (Imm * DstNumElts);
3277 }
else if (!IsX86 &&
Name ==
"stackprotectorcheck") {
3279 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.perm.df.") ||
3280 Name.starts_with(
"avx512.mask.perm.di."))) {
3282 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
3283 auto *VecTy = cast<FixedVectorType>(CI->
getType());
3284 unsigned NumElts = VecTy->getNumElements();
3287 for (
unsigned i = 0; i != NumElts; ++i)
3288 Idxs[i] = (i & ~0x3) + ((Imm >> (2 * (i & 0x3))) & 3);
3295 }
else if (IsX86 && (
Name.starts_with(
"avx.vperm2f128.") ||
3296 Name ==
"avx2.vperm2i128")) {
3305 uint8_t Imm = cast<ConstantInt>(CI->
getArgOperand(2))->getZExtValue();
3307 unsigned NumElts = cast<FixedVectorType>(CI->
getType())->getNumElements();
3308 unsigned HalfSize = NumElts / 2;
3320 unsigned StartIndex = (Imm & 0x01) ? HalfSize : 0;
3321 for (
unsigned i = 0; i < HalfSize; ++i)
3322 ShuffleMask[i] = StartIndex + i;
3325 StartIndex = (Imm & 0x10) ? HalfSize : 0;
3326 for (
unsigned i = 0; i < HalfSize; ++i)
3327 ShuffleMask[i + HalfSize] = NumElts + StartIndex + i;
3331 }
else if (IsX86 && (
Name.starts_with(
"avx.vpermil.") ||
3332 Name ==
"sse2.pshuf.d" ||
3333 Name.starts_with(
"avx512.mask.vpermil.p") ||
3334 Name.starts_with(
"avx512.mask.pshuf.d."))) {
3336 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
3337 auto *VecTy = cast<FixedVectorType>(CI->
getType());
3338 unsigned NumElts = VecTy->getNumElements();
3340 unsigned IdxSize = 64 / VecTy->getScalarSizeInBits();
3341 unsigned IdxMask = ((1 << IdxSize) - 1);
3347 for (
unsigned i = 0; i != NumElts; ++i)
3348 Idxs[i] = ((Imm >> ((i * IdxSize) % 8)) & IdxMask) | (i & ~IdxMask);
3355 }
else if (IsX86 && (
Name ==
"sse2.pshufl.w" ||
3356 Name.starts_with(
"avx512.mask.pshufl.w."))) {
3358 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
3359 unsigned NumElts = cast<FixedVectorType>(CI->
getType())->getNumElements();
3362 for (
unsigned l = 0; l != NumElts; l += 8) {
3363 for (
unsigned i = 0; i != 4; ++i)
3364 Idxs[i + l] = ((Imm >> (2 * i)) & 0x3) + l;
3365 for (
unsigned i = 4; i != 8; ++i)
3366 Idxs[i + l] = i + l;
3374 }
else if (IsX86 && (
Name ==
"sse2.pshufh.w" ||
3375 Name.starts_with(
"avx512.mask.pshufh.w."))) {
3377 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
3378 unsigned NumElts = cast<FixedVectorType>(CI->
getType())->getNumElements();
3381 for (
unsigned l = 0; l != NumElts; l += 8) {
3382 for (
unsigned i = 0; i != 4; ++i)
3383 Idxs[i + l] = i + l;
3384 for (
unsigned i = 0; i != 4; ++i)
3385 Idxs[i + l + 4] = ((Imm >> (2 * i)) & 0x3) + 4 + l;
3393 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.shuf.p")) {
3396 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(2))->getZExtValue();
3397 unsigned NumElts = cast<FixedVectorType>(CI->
getType())->getNumElements();
3400 unsigned HalfLaneElts = NumLaneElts / 2;
3403 for (
unsigned i = 0; i != NumElts; ++i) {
3405 Idxs[i] = i - (i % NumLaneElts);
3407 if ((i % NumLaneElts) >= HalfLaneElts)
3411 Idxs[i] += (Imm >> ((i * HalfLaneElts) % 8)) & ((1 << HalfLaneElts) - 1);
3418 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.movddup") ||
3419 Name.starts_with(
"avx512.mask.movshdup") ||
3420 Name.starts_with(
"avx512.mask.movsldup"))) {
3422 unsigned NumElts = cast<FixedVectorType>(CI->
getType())->getNumElements();
3426 if (
Name.starts_with(
"avx512.mask.movshdup."))
3430 for (
unsigned l = 0; l != NumElts; l += NumLaneElts)
3431 for (
unsigned i = 0; i != NumLaneElts; i += 2) {
3432 Idxs[i + l + 0] = i + l +
Offset;
3433 Idxs[i + l + 1] = i + l +
Offset;
3440 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.punpckl") ||
3441 Name.starts_with(
"avx512.mask.unpckl."))) {
3444 int NumElts = cast<FixedVectorType>(CI->
getType())->getNumElements();
3448 for (
int l = 0; l != NumElts; l += NumLaneElts)
3449 for (
int i = 0; i != NumLaneElts; ++i)
3450 Idxs[i + l] = l + (i / 2) + NumElts * (i % 2);
3456 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.punpckh") ||
3457 Name.starts_with(
"avx512.mask.unpckh."))) {
3460 int NumElts = cast<FixedVectorType>(CI->
getType())->getNumElements();
3464 for (
int l = 0; l != NumElts; l += NumLaneElts)
3465 for (
int i = 0; i != NumLaneElts; ++i)
3466 Idxs[i + l] = (NumLaneElts / 2) + l + (i / 2) + NumElts * (i % 2);
3472 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.and.") ||
3473 Name.starts_with(
"avx512.mask.pand."))) {
3475 VectorType *ITy = VectorType::getInteger(FTy);
3481 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.andn.") ||
3482 Name.starts_with(
"avx512.mask.pandn."))) {
3484 VectorType *ITy = VectorType::getInteger(FTy);
3491 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.or.") ||
3492 Name.starts_with(
"avx512.mask.por."))) {
3494 VectorType *ITy = VectorType::getInteger(FTy);
3500 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.xor.") ||
3501 Name.starts_with(
"avx512.mask.pxor."))) {
3503 VectorType *ITy = VectorType::getInteger(FTy);
3509 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.padd.")) {
3513 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.psub.")) {
3517 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.pmull.")) {
3521 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.add.p")) {
3522 if (
Name.ends_with(
".512")) {
3524 if (
Name[17] ==
's')
3525 IID = Intrinsic::x86_avx512_add_ps_512;
3527 IID = Intrinsic::x86_avx512_add_pd_512;
3530 { CI->getArgOperand(0), CI->getArgOperand(1),
3531 CI->getArgOperand(4) });
3537 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.div.p")) {
3538 if (
Name.ends_with(
".512")) {
3540 if (
Name[17] ==
's')
3541 IID = Intrinsic::x86_avx512_div_ps_512;
3543 IID = Intrinsic::x86_avx512_div_pd_512;
3546 { CI->getArgOperand(0), CI->getArgOperand(1),
3547 CI->getArgOperand(4) });
3553 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.mul.p")) {
3554 if (
Name.ends_with(
".512")) {
3556 if (
Name[17] ==
's')
3557 IID = Intrinsic::x86_avx512_mul_ps_512;
3559 IID = Intrinsic::x86_avx512_mul_pd_512;
3562 { CI->getArgOperand(0), CI->getArgOperand(1),
3563 CI->getArgOperand(4) });
3569 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.sub.p")) {
3570 if (
Name.ends_with(
".512")) {
3572 if (
Name[17] ==
's')
3573 IID = Intrinsic::x86_avx512_sub_ps_512;
3575 IID = Intrinsic::x86_avx512_sub_pd_512;
3578 { CI->getArgOperand(0), CI->getArgOperand(1),
3579 CI->getArgOperand(4) });
3585 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.max.p") ||
3586 Name.starts_with(
"avx512.mask.min.p")) &&
3587 Name.drop_front(18) ==
".512") {
3588 bool IsDouble =
Name[17] ==
'd';
3589 bool IsMin =
Name[13] ==
'i';
3591 { Intrinsic::x86_avx512_max_ps_512, Intrinsic::x86_avx512_max_pd_512 },
3592 { Intrinsic::x86_avx512_min_ps_512, Intrinsic::x86_avx512_min_pd_512 }
3597 { CI->getArgOperand(0), CI->getArgOperand(1),
3598 CI->getArgOperand(4) });
3601 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.lzcnt.")) {
3605 { CI->getArgOperand(0), Builder.getInt1(false) });
3608 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.psll")) {
3609 bool IsImmediate =
Name[16] ==
'i' ||
3610 (
Name.size() > 18 &&
Name[18] ==
'i');
3611 bool IsVariable =
Name[16] ==
'v';
3618 if (IsVariable &&
Name[17] !=
'.') {
3619 if (
Size ==
'd' &&
Name[17] ==
'2')
3620 IID = Intrinsic::x86_avx2_psllv_q;
3621 else if (
Size ==
'd' &&
Name[17] ==
'4')
3622 IID = Intrinsic::x86_avx2_psllv_q_256;
3623 else if (
Size ==
's' &&
Name[17] ==
'4')
3624 IID = Intrinsic::x86_avx2_psllv_d;
3625 else if (
Size ==
's' &&
Name[17] ==
'8')
3626 IID = Intrinsic::x86_avx2_psllv_d_256;
3627 else if (
Size ==
'h' &&
Name[17] ==
'8')
3628 IID = Intrinsic::x86_avx512_psllv_w_128;
3629 else if (
Size ==
'h' &&
Name[17] ==
'1')
3630 IID = Intrinsic::x86_avx512_psllv_w_256;
3631 else if (
Name[17] ==
'3' &&
Name[18] ==
'2')
3632 IID = Intrinsic::x86_avx512_psllv_w_512;
3635 }
else if (
Name.ends_with(
".128")) {
3637 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_d
3638 : Intrinsic::x86_sse2_psll_d;
3639 else if (
Size ==
'q')
3640 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_q
3641 : Intrinsic::x86_sse2_psll_q;
3642 else if (
Size ==
'w')
3643 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_w
3644 : Intrinsic::x86_sse2_psll_w;
3647 }
else if (
Name.ends_with(
".256")) {
3649 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_d
3650 : Intrinsic::x86_avx2_psll_d;
3651 else if (
Size ==
'q')
3652 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_q
3653 : Intrinsic::x86_avx2_psll_q;
3654 else if (
Size ==
'w')
3655 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_w
3656 : Intrinsic::x86_avx2_psll_w;
3661 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_d_512 :
3662 IsVariable ? Intrinsic::x86_avx512_psllv_d_512 :
3663 Intrinsic::x86_avx512_psll_d_512;
3664 else if (
Size ==
'q')
3665 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_q_512 :
3666 IsVariable ? Intrinsic::x86_avx512_psllv_q_512 :
3667 Intrinsic::x86_avx512_psll_q_512;
3668 else if (
Size ==
'w')
3669 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_w_512
3670 : Intrinsic::x86_avx512_psll_w_512;
3676 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.psrl")) {
3677 bool IsImmediate =
Name[16] ==
'i' ||
3678 (
Name.size() > 18 &&
Name[18] ==
'i');
3679 bool IsVariable =
Name[16] ==
'v';
3686 if (IsVariable &&
Name[17] !=
'.') {
3687 if (
Size ==
'd' &&
Name[17] ==
'2')
3688 IID = Intrinsic::x86_avx2_psrlv_q;
3689 else if (
Size ==
'd' &&
Name[17] ==
'4')
3690 IID = Intrinsic::x86_avx2_psrlv_q_256;
3691 else if (
Size ==
's' &&
Name[17] ==
'4')
3692 IID = Intrinsic::x86_avx2_psrlv_d;
3693 else if (
Size ==
's' &&
Name[17] ==
'8')
3694 IID = Intrinsic::x86_avx2_psrlv_d_256;
3695 else if (
Size ==
'h' &&
Name[17] ==
'8')
3696 IID = Intrinsic::x86_avx512_psrlv_w_128;
3697 else if (
Size ==
'h' &&
Name[17] ==
'1')
3698 IID = Intrinsic::x86_avx512_psrlv_w_256;
3699 else if (
Name[17] ==
'3' &&
Name[18] ==
'2')
3700 IID = Intrinsic::x86_avx512_psrlv_w_512;
3703 }
else if (
Name.ends_with(
".128")) {
3705 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_d
3706 : Intrinsic::x86_sse2_psrl_d;
3707 else if (
Size ==
'q')
3708 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_q
3709 : Intrinsic::x86_sse2_psrl_q;
3710 else if (
Size ==
'w')
3711 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_w
3712 : Intrinsic::x86_sse2_psrl_w;
3715 }
else if (
Name.ends_with(
".256")) {
3717 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_d
3718 : Intrinsic::x86_avx2_psrl_d;
3719 else if (
Size ==
'q')
3720 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_q
3721 : Intrinsic::x86_avx2_psrl_q;
3722 else if (
Size ==
'w')
3723 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_w
3724 : Intrinsic::x86_avx2_psrl_w;
3729 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_d_512 :
3730 IsVariable ? Intrinsic::x86_avx512_psrlv_d_512 :
3731 Intrinsic::x86_avx512_psrl_d_512;
3732 else if (
Size ==
'q')
3733 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_q_512 :
3734 IsVariable ? Intrinsic::x86_avx512_psrlv_q_512 :
3735 Intrinsic::x86_avx512_psrl_q_512;
3736 else if (
Size ==
'w')
3737 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_w_512
3738 : Intrinsic::x86_avx512_psrl_w_512;
3744 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.psra")) {
3745 bool IsImmediate =
Name[16] ==
'i' ||
3746 (
Name.size() > 18 &&
Name[18] ==
'i');
3747 bool IsVariable =
Name[16] ==
'v';
3754 if (IsVariable &&
Name[17] !=
'.') {
3755 if (
Size ==
's' &&
Name[17] ==
'4')
3756 IID = Intrinsic::x86_avx2_psrav_d;
3757 else if (
Size ==
's' &&
Name[17] ==
'8')
3758 IID = Intrinsic::x86_avx2_psrav_d_256;
3759 else if (
Size ==
'h' &&
Name[17] ==
'8')
3760 IID = Intrinsic::x86_avx512_psrav_w_128;
3761 else if (
Size ==
'h' &&
Name[17] ==
'1')
3762 IID = Intrinsic::x86_avx512_psrav_w_256;
3763 else if (
Name[17] ==
'3' &&
Name[18] ==
'2')
3764 IID = Intrinsic::x86_avx512_psrav_w_512;
3767 }
else if (
Name.ends_with(
".128")) {
3769 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_d
3770 : Intrinsic::x86_sse2_psra_d;
3771 else if (
Size ==
'q')
3772 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_128 :
3773 IsVariable ? Intrinsic::x86_avx512_psrav_q_128 :
3774 Intrinsic::x86_avx512_psra_q_128;
3775 else if (
Size ==
'w')
3776 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_w
3777 : Intrinsic::x86_sse2_psra_w;
3780 }
else if (
Name.ends_with(
".256")) {
3782 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_d
3783 : Intrinsic::x86_avx2_psra_d;
3784 else if (
Size ==
'q')
3785 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_256 :
3786 IsVariable ? Intrinsic::x86_avx512_psrav_q_256 :
3787 Intrinsic::x86_avx512_psra_q_256;
3788 else if (
Size ==
'w')
3789 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_w
3790 : Intrinsic::x86_avx2_psra_w;
3795 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_d_512 :
3796 IsVariable ? Intrinsic::x86_avx512_psrav_d_512 :
3797 Intrinsic::x86_avx512_psra_d_512;
3798 else if (
Size ==
'q')
3799 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_512 :
3800 IsVariable ? Intrinsic::x86_avx512_psrav_q_512 :
3801 Intrinsic::x86_avx512_psra_q_512;
3802 else if (
Size ==
'w')
3803 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_w_512
3804 : Intrinsic::x86_avx512_psra_w_512;
3810 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.move.s")) {
3812 }
else if (IsX86 &&
Name.starts_with(
"avx512.cvtmask2")) {
3814 }
else if (IsX86 &&
Name.ends_with(
".movntdqa")) {
3822 Ptr, PointerType::getUnqual(CI->
getType()),
"cast");
3826 LI->
setMetadata(LLVMContext::MD_nontemporal, Node);
3828 }
else if (IsX86 && (
Name.starts_with(
"fma.vfmadd.") ||
3829 Name.starts_with(
"fma.vfmsub.") ||
3830 Name.starts_with(
"fma.vfnmadd.") ||
3831 Name.starts_with(
"fma.vfnmsub."))) {
3832 bool NegMul =
Name[6] ==
'n';
3833 bool NegAcc = NegMul ?
Name[8] ==
's' :
Name[7] ==
's';
3834 bool IsScalar = NegMul ?
Name[12] ==
's' :
Name[11] ==
's';
3845 if (NegMul && !IsScalar)
3847 if (NegMul && IsScalar)
3860 }
else if (IsX86 &&
Name.starts_with(
"fma4.vfmadd.s")) {
3875 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.vfmadd.s") ||
3876 Name.starts_with(
"avx512.maskz.vfmadd.s") ||
3877 Name.starts_with(
"avx512.mask3.vfmadd.s") ||
3878 Name.starts_with(
"avx512.mask3.vfmsub.s") ||
3879 Name.starts_with(
"avx512.mask3.vfnmsub.s"))) {
3880 bool IsMask3 =
Name[11] ==
'3';
3881 bool IsMaskZ =
Name[11] ==
'z';
3883 Name =
Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
3884 bool NegMul =
Name[2] ==
'n';
3885 bool NegAcc = NegMul ?
Name[4] ==
's' :
Name[3] ==
's';
3891 if (NegMul && (IsMask3 || IsMaskZ))
3893 if (NegMul && !(IsMask3 || IsMaskZ))
3903 cast<ConstantInt>(CI->
getArgOperand(4))->getZExtValue() != 4) {
3907 if (
Name.back() ==
'd')
3908 IID = Intrinsic::x86_avx512_vfmadd_f64;
3910 IID = Intrinsic::x86_avx512_vfmadd_f32;
3925 if (NegAcc && IsMask3)
3932 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.vfmadd.p") ||
3933 Name.starts_with(
"avx512.mask.vfnmadd.p") ||
3934 Name.starts_with(
"avx512.mask.vfnmsub.p") ||
3935 Name.starts_with(
"avx512.mask3.vfmadd.p") ||
3936 Name.starts_with(
"avx512.mask3.vfmsub.p") ||
3937 Name.starts_with(
"avx512.mask3.vfnmsub.p") ||
3938 Name.starts_with(
"avx512.maskz.vfmadd.p"))) {
3939 bool IsMask3 =
Name[11] ==
'3';
3940 bool IsMaskZ =
Name[11] ==
'z';
3942 Name =
Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
3943 bool NegMul =
Name[2] ==
'n';
3944 bool NegAcc = NegMul ?
Name[4] ==
's' :
Name[3] ==
's';
3950 if (NegMul && (IsMask3 || IsMaskZ))
3952 if (NegMul && !(IsMask3 || IsMaskZ))
3959 cast<ConstantInt>(CI->
getArgOperand(4))->getZExtValue() != 4)) {
3963 IID = Intrinsic::x86_avx512_vfmadd_ps_512;
3965 IID = Intrinsic::x86_avx512_vfmadd_pd_512;
3968 { A, B, C, CI->getArgOperand(4) });
3981 }
else if (IsX86 &&
Name.starts_with(
"fma.vfmsubadd.p")) {
3985 if (VecWidth == 128 && EltWidth == 32)
3986 IID = Intrinsic::x86_fma_vfmaddsub_ps;
3987 else if (VecWidth == 256 && EltWidth == 32)
3988 IID = Intrinsic::x86_fma_vfmaddsub_ps_256;
3989 else if (VecWidth == 128 && EltWidth == 64)
3990 IID = Intrinsic::x86_fma_vfmaddsub_pd;
3991 else if (VecWidth == 256 && EltWidth == 64)
3992 IID = Intrinsic::x86_fma_vfmaddsub_pd_256;
4001 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.vfmaddsub.p") ||
4002 Name.starts_with(
"avx512.mask3.vfmaddsub.p") ||
4003 Name.starts_with(
"avx512.maskz.vfmaddsub.p") ||
4004 Name.starts_with(
"avx512.mask3.vfmsubadd.p"))) {
4005 bool IsMask3 =
Name[11] ==
'3';
4006 bool IsMaskZ =
Name[11] ==
'z';
4008 Name =
Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
4009 bool IsSubAdd =
Name[3] ==
's';
4014 IID = Intrinsic::x86_avx512_vfmaddsub_ps_512;
4016 IID = Intrinsic::x86_avx512_vfmaddsub_pd_512;
4026 int NumElts = cast<FixedVectorType>(CI->
getType())->getNumElements();
4041 for (
int i = 0; i != NumElts; ++i)
4042 Idxs[i] = i + (i % 2) * NumElts;
4052 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.pternlog.") ||
4053 Name.starts_with(
"avx512.maskz.pternlog."))) {
4054 bool ZeroMask =
Name[11] ==
'z';
4058 if (VecWidth == 128 && EltWidth == 32)
4059 IID = Intrinsic::x86_avx512_pternlog_d_128;
4060 else if (VecWidth == 256 && EltWidth == 32)
4061 IID = Intrinsic::x86_avx512_pternlog_d_256;
4062 else if (VecWidth == 512 && EltWidth == 32)
4063 IID = Intrinsic::x86_avx512_pternlog_d_512;
4064 else if (VecWidth == 128 && EltWidth == 64)
4065 IID = Intrinsic::x86_avx512_pternlog_q_128;
4066 else if (VecWidth == 256 && EltWidth == 64)
4067 IID = Intrinsic::x86_avx512_pternlog_q_256;
4068 else if (VecWidth == 512 && EltWidth == 64)
4069 IID = Intrinsic::x86_avx512_pternlog_q_512;
4080 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.vpmadd52") ||
4081 Name.starts_with(
"avx512.maskz.vpmadd52"))) {
4082 bool ZeroMask =
Name[11] ==
'z';
4086 if (VecWidth == 128 && !
High)
4087 IID = Intrinsic::x86_avx512_vpmadd52l_uq_128;
4088 else if (VecWidth == 256 && !
High)
4089 IID = Intrinsic::x86_avx512_vpmadd52l_uq_256;
4090 else if (VecWidth == 512 && !
High)
4091 IID = Intrinsic::x86_avx512_vpmadd52l_uq_512;
4092 else if (VecWidth == 128 &&
High)
4093 IID = Intrinsic::x86_avx512_vpmadd52h_uq_128;
4094 else if (VecWidth == 256 &&
High)
4095 IID = Intrinsic::x86_avx512_vpmadd52h_uq_256;
4096 else if (VecWidth == 512 &&
High)
4097 IID = Intrinsic::x86_avx512_vpmadd52h_uq_512;
4108 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.vpermi2var.") ||
4109 Name.starts_with(
"avx512.mask.vpermt2var.") ||
4110 Name.starts_with(
"avx512.maskz.vpermt2var."))) {
4111 bool ZeroMask =
Name[11] ==
'z';
4112 bool IndexForm =
Name[17] ==
'i';
4114 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.vpdpbusd.") ||
4115 Name.starts_with(
"avx512.maskz.vpdpbusd.") ||
4116 Name.starts_with(
"avx512.mask.vpdpbusds.") ||
4117 Name.starts_with(
"avx512.maskz.vpdpbusds."))) {
4118 bool ZeroMask =
Name[11] ==
'z';
4119 bool IsSaturating =
Name[ZeroMask ? 21 : 20] ==
's';
4122 if (VecWidth == 128 && !IsSaturating)
4123 IID = Intrinsic::x86_avx512_vpdpbusd_128;
4124 else if (VecWidth == 256 && !IsSaturating)
4125 IID = Intrinsic::x86_avx512_vpdpbusd_256;
4126 else if (VecWidth == 512 && !IsSaturating)
4127 IID = Intrinsic::x86_avx512_vpdpbusd_512;
4128 else if (VecWidth == 128 && IsSaturating)
4129 IID = Intrinsic::x86_avx512_vpdpbusds_128;
4130 else if (VecWidth == 256 && IsSaturating)
4131 IID = Intrinsic::x86_avx512_vpdpbusds_256;
4132 else if (VecWidth == 512 && IsSaturating)
4133 IID = Intrinsic::x86_avx512_vpdpbusds_512;
4144 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.vpdpwssd.") ||
4145 Name.starts_with(
"avx512.maskz.vpdpwssd.") ||
4146 Name.starts_with(
"avx512.mask.vpdpwssds.") ||
4147 Name.starts_with(
"avx512.maskz.vpdpwssds."))) {
4148 bool ZeroMask =
Name[11] ==
'z';
4149 bool IsSaturating =
Name[ZeroMask ? 21 : 20] ==
's';
4152 if (VecWidth == 128 && !IsSaturating)
4153 IID = Intrinsic::x86_avx512_vpdpwssd_128;
4154 else if (VecWidth == 256 && !IsSaturating)
4155 IID = Intrinsic::x86_avx512_vpdpwssd_256;
4156 else if (VecWidth == 512 && !IsSaturating)
4157 IID = Intrinsic::x86_avx512_vpdpwssd_512;
4158 else if (VecWidth == 128 && IsSaturating)
4159 IID = Intrinsic::x86_avx512_vpdpwssds_128;
4160 else if (VecWidth == 256 && IsSaturating)
4161 IID = Intrinsic::x86_avx512_vpdpwssds_256;
4162 else if (VecWidth == 512 && IsSaturating)
4163 IID = Intrinsic::x86_avx512_vpdpwssds_512;
4174 }
else if (IsX86 && (
Name ==
"addcarryx.u32" ||
Name ==
"addcarryx.u64" ||
4175 Name ==
"addcarry.u32" ||
Name ==
"addcarry.u64" ||
4176 Name ==
"subborrow.u32" ||
Name ==
"subborrow.u64")) {
4178 if (
Name[0] ==
'a' &&
Name.back() ==
'2')
4179 IID = Intrinsic::x86_addcarry_32;
4180 else if (
Name[0] ==
'a' &&
Name.back() ==
'4')
4181 IID = Intrinsic::x86_addcarry_64;
4182 else if (
Name[0] ==
's' &&
Name.back() ==
'2')
4183 IID = Intrinsic::x86_subborrow_32;
4184 else if (
Name[0] ==
's' &&
Name.back() ==
'4')
4185 IID = Intrinsic::x86_subborrow_64;
4207 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.") &&
4210 }
else if (IsNVVM && (
Name ==
"abs.i" ||
Name ==
"abs.ll")) {
4216 }
else if (IsNVVM && (
Name.starts_with(
"atomic.load.add.f32.p") ||
4217 Name.starts_with(
"atomic.load.add.f64.p"))) {
4221 AtomicOrdering::SequentiallyConsistent);
4222 }
else if (IsNVVM &&
Name.consume_front(
"max.") &&
4231 }
else if (IsNVVM &&
Name.consume_front(
"min.") &&
4240 }
else if (IsNVVM &&
Name ==
"clz.ll") {
4246 {Arg, Builder.getFalse()},
"ctlz");
4248 }
else if (IsNVVM &&
Name ==
"popc.ll") {
4257 }
else if (IsNVVM) {
4258 if (
Name ==
"h2f") {
4261 F->getParent(), Intrinsic::convert_from_fp16,
4262 {Builder.getFloatTy()}),
4267 !
F->getReturnType()->getScalarType()->isBFloatTy()) {
4281 if (
F->getReturnType()->isIntegerTy())
4287 }
else if (IsAMDGCN) {
4299 FallthroughToDefaultUpgrade =
true;
4307 if (!FallthroughToDefaultUpgrade) {
4315 const auto &DefaultCase = [&]() ->
void {
4320 "Unknown function for CallBase upgrade and isn't just a name change");
4326 if (
auto *OldST = dyn_cast<StructType>(CI->
getType())) {
4328 "Return type must have changed");
4329 assert(OldST->getNumElements() ==
4330 cast<StructType>(NewFn->
getReturnType())->getNumElements() &&
4331 "Must have same number of elements");
4336 for (
unsigned Idx = 0;
Idx < OldST->getNumElements(); ++
Idx) {
4357 case Intrinsic::arm_neon_vst1:
4358 case Intrinsic::arm_neon_vst2:
4359 case Intrinsic::arm_neon_vst3:
4360 case Intrinsic::arm_neon_vst4:
4361 case Intrinsic::arm_neon_vst2lane:
4362 case Intrinsic::arm_neon_vst3lane:
4363 case Intrinsic::arm_neon_vst4lane: {
4368 case Intrinsic::aarch64_sve_bfmlalb_lane_v2:
4369 case Intrinsic::aarch64_sve_bfmlalt_lane_v2:
4370 case Intrinsic::aarch64_sve_bfdot_lane_v2: {
4374 cast<ConstantInt>(Args[3])->getZExtValue());
4378 case Intrinsic::aarch64_sve_ld3_sret:
4379 case Intrinsic::aarch64_sve_ld4_sret:
4380 case Intrinsic::aarch64_sve_ld2_sret: {
4389 dyn_cast<ScalableVectorType>(
F->getReturnType());
4390 unsigned MinElts =
RetTy->getMinNumElements() /
N;
4394 for (
unsigned I = 0;
I <
N;
I++) {
4399 NewCall = dyn_cast<CallInst>(Ret);
4403 case Intrinsic::coro_end: {
4410 case Intrinsic::vector_extract: {
4413 if (!
Name.starts_with(
"aarch64.sve.tuple.get")) {
4418 dyn_cast<ScalableVectorType>(
F->getReturnType());
4419 unsigned MinElts =
RetTy->getMinNumElements();
4420 unsigned I = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
4426 case Intrinsic::vector_insert: {
4429 if (!
Name.starts_with(
"aarch64.sve.tuple")) {
4433 if (
Name.starts_with(
"aarch64.sve.tuple.set")) {
4434 unsigned I = dyn_cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
4443 if (
Name.starts_with(
"aarch64.sve.tuple.create")) {
4449 assert(
N > 1 &&
"Create is expected to be between 2-4");
4451 dyn_cast<ScalableVectorType>(
F->getReturnType());
4453 unsigned MinElts =
RetTy->getMinNumElements() /
N;
4454 for (
unsigned I = 0;
I <
N;
I++) {
4459 NewCall = dyn_cast<CallInst>(Ret);
4464 case Intrinsic::arm_neon_bfdot:
4465 case Intrinsic::arm_neon_bfmmla:
4466 case Intrinsic::arm_neon_bfmlalb:
4467 case Intrinsic::arm_neon_bfmlalt:
4468 case Intrinsic::aarch64_neon_bfdot:
4469 case Intrinsic::aarch64_neon_bfmmla:
4470 case Intrinsic::aarch64_neon_bfmlalb:
4471 case Intrinsic::aarch64_neon_bfmlalt: {
4474 "Mismatch between function args and call args");
4475 size_t OperandWidth =
4477 assert((OperandWidth == 64 || OperandWidth == 128) &&
4478 "Unexpected operand width");
4480 auto Iter = CI->
args().begin();
4481 Args.push_back(*Iter++);
4488 case Intrinsic::bitreverse:
4492 case Intrinsic::ctlz:
4493 case Intrinsic::cttz:
4495 "Mismatch between function args and call args");
4500 case Intrinsic::objectsize: {
4501 Value *NullIsUnknownSize =
4510 case Intrinsic::ctpop:
4514 case Intrinsic::convert_from_fp16:
4518 case Intrinsic::dbg_value: {
4522 if (
Name.starts_with(
"dbg.addr")) {
4524 cast<MetadataAsValue>(CI->
getArgOperand(2))->getMetadata());
4536 if (
Offset->isZeroValue()) {
4546 case Intrinsic::ptr_annotation:
4563 case Intrinsic::var_annotation:
4579 case Intrinsic::riscv_aes32dsi:
4580 case Intrinsic::riscv_aes32dsmi:
4581 case Intrinsic::riscv_aes32esi:
4582 case Intrinsic::riscv_aes32esmi:
4583 case Intrinsic::riscv_sm4ks:
4584 case Intrinsic::riscv_sm4ed: {
4599 cast<ConstantInt>(Arg2)->getZExtValue());
4601 NewCall = Builder.
CreateCall(NewFn, {Arg0, Arg1, Arg2});
4602 Value *Res = NewCall;
4610 case Intrinsic::riscv_sha256sig0:
4611 case Intrinsic::riscv_sha256sig1:
4612 case Intrinsic::riscv_sha256sum0:
4613 case Intrinsic::riscv_sha256sum1:
4614 case Intrinsic::riscv_sm3p0:
4615 case Intrinsic::riscv_sm3p1: {
4633 case Intrinsic::x86_xop_vfrcz_ss:
4634 case Intrinsic::x86_xop_vfrcz_sd:
4638 case Intrinsic::x86_xop_vpermil2pd:
4639 case Intrinsic::x86_xop_vpermil2ps:
4640 case Intrinsic::x86_xop_vpermil2pd_256:
4641 case Intrinsic::x86_xop_vpermil2ps_256: {
4644 VectorType *IntIdxTy = VectorType::getInteger(FltIdxTy);
4650 case Intrinsic::x86_sse41_ptestc:
4651 case Intrinsic::x86_sse41_ptestz:
4652 case Intrinsic::x86_sse41_ptestnzc: {
4669 NewCall = Builder.
CreateCall(NewFn, {BC0, BC1});
4673 case Intrinsic::x86_rdtscp: {
4695 case Intrinsic::x86_sse41_insertps:
4696 case Intrinsic::x86_sse41_dppd:
4697 case Intrinsic::x86_sse41_dpps:
4698 case Intrinsic::x86_sse41_mpsadbw:
4699 case Intrinsic::x86_avx_dp_ps_256:
4700 case Intrinsic::x86_avx2_mpsadbw: {
4711 case Intrinsic::x86_avx512_mask_cmp_pd_128:
4712 case Intrinsic::x86_avx512_mask_cmp_pd_256:
4713 case Intrinsic::x86_avx512_mask_cmp_pd_512:
4714 case Intrinsic::x86_avx512_mask_cmp_ps_128:
4715 case Intrinsic::x86_avx512_mask_cmp_ps_256:
4716 case Intrinsic::x86_avx512_mask_cmp_ps_512: {
4719 cast<FixedVectorType>(Args[0]->
getType())->getNumElements();
4731 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128:
4732 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256:
4733 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512:
4734 case Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128:
4735 case Intrinsic::x86_avx512bf16_cvtneps2bf16_256:
4736 case Intrinsic::x86_avx512bf16_cvtneps2bf16_512: {
4738 unsigned NumElts = cast<FixedVectorType>(CI->
getType())->getNumElements();
4740 Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
4753 case Intrinsic::x86_avx512bf16_dpbf16ps_128:
4754 case Intrinsic::x86_avx512bf16_dpbf16ps_256:
4755 case Intrinsic::x86_avx512bf16_dpbf16ps_512:{
4758 cast<FixedVectorType>(CI->
getType())->getNumElements() * 2;
4768 case Intrinsic::thread_pointer: {
4773 case Intrinsic::memcpy:
4774 case Intrinsic::memmove:
4775 case Intrinsic::memset: {
4795 {OldAttrs.getParamAttrs(0), OldAttrs.getParamAttrs(1),
4796 OldAttrs.getParamAttrs(2), OldAttrs.getParamAttrs(4)});
4798 auto *MemCI = cast<MemIntrinsic>(NewCall);
4801 MemCI->setDestAlignment(
Align->getMaybeAlignValue());
4803 if (
auto *MTI = dyn_cast<MemTransferInst>(MemCI))
4804 MTI->setSourceAlignment(
Align->getMaybeAlignValue());
4808 assert(NewCall &&
"Should have either set this variable or returned through "
4809 "the default case");
4816 assert(
F &&
"Illegal attempt to upgrade a non-existent intrinsic.");
4825 if (
CallBase *CB = dyn_cast<CallBase>(U))
4829 F->eraseFromParent();
4835 if (NumOperands == 0)
4839 if (isa<MDNode>(MD.
getOperand(0)) && NumOperands >= 3)
4843 if (NumOperands == 3) {
4847 Metadata *Elts2[] = {ScalarType, ScalarType,
4861 if (Opc != Instruction::BitCast)
4865 Type *SrcTy = V->getType();
4882 if (Opc != Instruction::BitCast)
4885 Type *SrcTy =
C->getType();
4909 bool BrokenDebugInfo =
false;
4912 if (!BrokenDebugInfo)
4918 M.getContext().diagnose(Diag);
4925 M.getContext().diagnose(DiagVersion);
4933 bool Changed =
false;
4934 const char *MarkerKey =
"clang.arc.retainAutoreleasedReturnValueMarker";
4935 NamedMDNode *ModRetainReleaseMarker = M.getNamedMetadata(MarkerKey);
4936 if (ModRetainReleaseMarker) {
4939 MDString *
ID = dyn_cast_or_null<MDString>(
Op->getOperand(0));
4942 ID->getString().split(ValueComp,
"#");
4943 if (ValueComp.
size() == 2) {
4944 std::string NewValue = ValueComp[0].str() +
";" + ValueComp[1].str();
4948 M.eraseNamedMetadata(ModRetainReleaseMarker);
4959 auto UpgradeToIntrinsic = [&](
const char *OldFunc,
4961 Function *Fn = M.getFunction(OldFunc);
4969 CallInst *CI = dyn_cast<CallInst>(U);
4984 bool InvalidCast =
false;
4986 for (
unsigned I = 0, E = CI->
arg_size();
I != E; ++
I) {
4991 if (I < NewFuncTy->getNumParams()) {
5001 Args.push_back(Arg);
5026 UpgradeToIntrinsic(
"clang.arc.use", llvm::Intrinsic::objc_clang_arc_use);
5034 std::pair<const char *, llvm::Intrinsic::ID> RuntimeFuncs[] = {
5035 {
"objc_autorelease", llvm::Intrinsic::objc_autorelease},
5036 {
"objc_autoreleasePoolPop", llvm::Intrinsic::objc_autoreleasePoolPop},
5037 {
"objc_autoreleasePoolPush", llvm::Intrinsic::objc_autoreleasePoolPush},
5038 {
"objc_autoreleaseReturnValue",
5039 llvm::Intrinsic::objc_autoreleaseReturnValue},
5040 {
"objc_copyWeak", llvm::Intrinsic::objc_copyWeak},
5041 {
"objc_destroyWeak", llvm::Intrinsic::objc_destroyWeak},
5042 {
"objc_initWeak", llvm::Intrinsic::objc_initWeak},
5043 {
"objc_loadWeak", llvm::Intrinsic::objc_loadWeak},
5044 {
"objc_loadWeakRetained", llvm::Intrinsic::objc_loadWeakRetained},
5045 {
"objc_moveWeak", llvm::Intrinsic::objc_moveWeak},
5046 {
"objc_release", llvm::Intrinsic::objc_release},
5047 {
"objc_retain", llvm::Intrinsic::objc_retain},
5048 {
"objc_retainAutorelease", llvm::Intrinsic::objc_retainAutorelease},
5049 {
"objc_retainAutoreleaseReturnValue",
5050 llvm::Intrinsic::objc_retainAutoreleaseReturnValue},
5051 {
"objc_retainAutoreleasedReturnValue",
5052 llvm::Intrinsic::objc_retainAutoreleasedReturnValue},
5053 {
"objc_retainBlock", llvm::Intrinsic::objc_retainBlock},
5054 {
"objc_storeStrong", llvm::Intrinsic::objc_storeStrong},
5055 {
"objc_storeWeak", llvm::Intrinsic::objc_storeWeak},
5056 {
"objc_unsafeClaimAutoreleasedReturnValue",
5057 llvm::Intrinsic::objc_unsafeClaimAutoreleasedReturnValue},
5058 {
"objc_retainedObject", llvm::Intrinsic::objc_retainedObject},
5059 {
"objc_unretainedObject", llvm::Intrinsic::objc_unretainedObject},
5060 {
"objc_unretainedPointer", llvm::Intrinsic::objc_unretainedPointer},
5061 {
"objc_retain_autorelease", llvm::Intrinsic::objc_retain_autorelease},
5062 {
"objc_sync_enter", llvm::Intrinsic::objc_sync_enter},
5063 {
"objc_sync_exit", llvm::Intrinsic::objc_sync_exit},
5064 {
"objc_arc_annotation_topdown_bbstart",
5065 llvm::Intrinsic::objc_arc_annotation_topdown_bbstart},
5066 {
"objc_arc_annotation_topdown_bbend",
5067 llvm::Intrinsic::objc_arc_annotation_topdown_bbend},
5068 {
"objc_arc_annotation_bottomup_bbstart",
5069 llvm::Intrinsic::objc_arc_annotation_bottomup_bbstart},
5070 {
"objc_arc_annotation_bottomup_bbend",
5071 llvm::Intrinsic::objc_arc_annotation_bottomup_bbend}};
5073 for (
auto &
I : RuntimeFuncs)
5074 UpgradeToIntrinsic(
I.first,
I.second);
5078 NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
5082 bool HasObjCFlag =
false, HasClassProperties =
false, Changed =
false;
5083 bool HasSwiftVersionFlag =
false;
5084 uint8_t SwiftMajorVersion, SwiftMinorVersion;
5093 MDString *
ID = dyn_cast_or_null<MDString>(
Op->getOperand(1));
5105 if (
ID->getString() ==
"Objective-C Image Info Version")
5107 if (
ID->getString() ==
"Objective-C Class Properties")
5108 HasClassProperties =
true;
5110 if (
ID->getString() ==
"PIC Level") {
5111 if (
auto *Behavior =
5112 mdconst::dyn_extract_or_null<ConstantInt>(
Op->getOperand(0))) {
5113 uint64_t V = Behavior->getLimitedValue();
5119 if (
ID->getString() ==
"PIE Level")
5120 if (
auto *Behavior =
5121 mdconst::dyn_extract_or_null<ConstantInt>(
Op->getOperand(0)))
5127 if (
ID->getString() ==
"branch-target-enforcement" ||
5128 ID->getString().starts_with(
"sign-return-address")) {
5129 if (
auto *Behavior =
5130 mdconst::dyn_extract_or_null<ConstantInt>(
Op->getOperand(0))) {
5135 Op->getOperand(1),
Op->getOperand(2)};
5145 if (
ID->getString() ==
"Objective-C Image Info Section") {
5146 if (
auto *
Value = dyn_cast_or_null<MDString>(
Op->getOperand(2))) {
5148 Value->getString().split(ValueComp,
" ");
5149 if (ValueComp.
size() != 1) {
5150 std::string NewValue;
5151 for (
auto &S : ValueComp)
5152 NewValue += S.str();
5153 Metadata *Ops[3] = {
Op->getOperand(0),
Op->getOperand(1),
5163 if (
ID->getString() ==
"Objective-C Garbage Collection") {
5164 auto Md = dyn_cast<ConstantAsMetadata>(
Op->getOperand(2));
5166 assert(Md->getValue() &&
"Expected non-empty metadata");
5167 auto Type = Md->getValue()->getType();
5170 unsigned Val = Md->getValue()->getUniqueInteger().getZExtValue();
5171 if ((Val & 0xff) != Val) {
5172 HasSwiftVersionFlag =
true;
5173 SwiftABIVersion = (Val & 0xff00) >> 8;
5174 SwiftMajorVersion = (Val & 0xff000000) >> 24;
5175 SwiftMinorVersion = (Val & 0xff0000) >> 16;
5186 if (
ID->getString() ==
"amdgpu_code_object_version") {
5189 MDString::get(M.getContext(),
"amdhsa_code_object_version"),
5201 if (HasObjCFlag && !HasClassProperties) {
5207 if (HasSwiftVersionFlag) {
5211 ConstantInt::get(Int8Ty, SwiftMajorVersion));
5213 ConstantInt::get(Int8Ty, SwiftMinorVersion));
5221 auto TrimSpaces = [](
StringRef Section) -> std::string {
5223 Section.split(Components,
',');
5228 for (
auto Component : Components)
5229 OS <<
',' << Component.trim();
5231 return std::string(
OS.str().substr(1));
5234 for (
auto &GV : M.globals()) {
5235 if (!GV.hasSection())
5240 if (!Section.starts_with(
"__DATA, __objc_catlist"))
5245 GV.setSection(TrimSpaces(Section));
5261struct StrictFPUpgradeVisitor :
public InstVisitor<StrictFPUpgradeVisitor> {
5262 StrictFPUpgradeVisitor() =
default;
5265 if (!Call.isStrictFP())
5267 if (isa<ConstrainedFPIntrinsic>(&Call))
5271 Call.removeFnAttr(Attribute::StrictFP);
5272 Call.addFnAttr(Attribute::NoBuiltin);
5280 if (!
F.isDeclaration() && !
F.hasFnAttribute(Attribute::StrictFP)) {
5281 StrictFPUpgradeVisitor SFPV;
5287 for (
auto &Arg :
F.args())
5292 if (
Attribute A =
F.getFnAttribute(
"implicit-section-name");
5293 A.isValid() &&
A.isStringAttribute()) {
5294 F.setSection(
A.getValueAsString());
5295 F.removeFnAttr(
"implicit-section-name");
5300 auto *
T = dyn_cast_or_null<MDTuple>(MD);
5303 if (
T->getNumOperands() < 1)
5305 auto *S = dyn_cast_or_null<MDString>(
T->getOperand(0));
5308 return S->getString().starts_with(
"llvm.vectorizer.");
5312 StringRef OldPrefix =
"llvm.vectorizer.";
5315 if (OldTag ==
"llvm.vectorizer.unroll")
5324 auto *
T = dyn_cast_or_null<MDTuple>(MD);
5327 if (
T->getNumOperands() < 1)
5329 auto *OldTag = dyn_cast_or_null<MDString>(
T->getOperand(0));
5332 if (!OldTag->getString().starts_with(
"llvm.vectorizer."))
5339 for (
unsigned I = 1, E =
T->getNumOperands();
I != E; ++
I)
5346 auto *
T = dyn_cast<MDTuple>(&
N);
5365 if (((
T.isAMDGPU() && !
T.isAMDGCN()) ||
5366 (
T.isSPIR() || (
T.isSPIRV() && !
T.isSPIRVLogical()))) &&
5367 !
DL.contains(
"-G") && !
DL.starts_with(
"G")) {
5368 return DL.
empty() ? std::string(
"G1") : (
DL +
"-G1").str();
5371 if (
T.isRISCV64()) {
5373 auto I =
DL.find(
"-n64-");
5375 return (
DL.take_front(
I) +
"-n32:64-" +
DL.drop_front(
I + 5)).str();
5379 std::string Res =
DL.str();
5383 if (!
DL.contains(
"-G") && !
DL.starts_with(
"G"))
5384 Res.append(Res.empty() ?
"G1" :
"-G1");
5389 if (!
DL.contains(
"-ni") && !
DL.starts_with(
"ni"))
5390 Res.append(
"-ni:7:8:9");
5392 if (
DL.ends_with(
"ni:7"))
5394 if (
DL.ends_with(
"ni:7:8"))
5399 if (!
DL.contains(
"-p7") && !
DL.starts_with(
"p7"))
5400 Res.append(
"-p7:160:256:256:32");
5401 if (!
DL.contains(
"-p8") && !
DL.starts_with(
"p8"))
5402 Res.append(
"-p8:128:128");
5403 if (!
DL.contains(
"-p9") && !
DL.starts_with(
"p9"))
5404 Res.append(
"-p9:192:256:256:32");
5410 if (
T.isAArch64()) {
5412 if (!
DL.
empty() && !
DL.contains(
"-Fn32"))
5413 Res.append(
"-Fn32");
5422 std::string AddrSpaces =
"-p270:32:32-p271:32:32-p272:64:64";
5425 Regex R(
"(e-m:[a-z](-p:32:32)?)(-[if]64:.*$)");
5426 if (R.match(Res, &
Groups))
5436 if (!
T.isOSIAMCU()) {
5437 std::string I128 =
"-i128:128";
5440 Regex R(
"^(e(-[mpi][^-]*)*)((-[^mpi][^-]*)*)$");
5441 if (R.match(Res, &
Groups))
5449 if (
T.isWindowsMSVCEnvironment() && !
T.isArch64Bit()) {
5451 auto I =
Ref.find(
"-f80:32-");
5453 Res = (
Ref.take_front(
I) +
"-f80:128-" +
Ref.drop_front(
I + 8)).str();
5461 Attribute A =
B.getAttribute(
"no-frame-pointer-elim");
5464 FramePointer =
A.getValueAsString() ==
"true" ?
"all" :
"none";
5465 B.removeAttribute(
"no-frame-pointer-elim");
5467 if (
B.contains(
"no-frame-pointer-elim-non-leaf")) {
5469 if (FramePointer !=
"all")
5470 FramePointer =
"non-leaf";
5471 B.removeAttribute(
"no-frame-pointer-elim-non-leaf");
5473 if (!FramePointer.
empty())
5474 B.addAttribute(
"frame-pointer", FramePointer);
5476 A =
B.getAttribute(
"null-pointer-is-valid");
5479 bool NullPointerIsValid =
A.getValueAsString() ==
"true";
5480 B.removeAttribute(
"null-pointer-is-valid");
5481 if (NullPointerIsValid)
5482 B.addAttribute(Attribute::NullPointerIsValid);
5492 return OBD.
getTag() ==
"clang.arc.attachedcall" &&
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu AMDGPU Register Bank Select
static Value * upgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallBase &CI, bool ZeroMask, bool IndexForm)
static Metadata * upgradeLoopArgument(Metadata *MD)
static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn, bool CanUpgradeDebugIntrinsicsToRecords)
static Value * upgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, Value *Op, unsigned Shift)
static bool upgradeRetainReleaseMarker(Module &M)
This checks for objc retain release marker which should be upgraded.
static Value * upgradeX86vpcom(IRBuilder<> &Builder, CallBase &CI, unsigned Imm, bool IsSigned)
static Value * upgradeMaskToInt(IRBuilder<> &Builder, CallBase &CI)
static Value * upgradeX86Rotate(IRBuilder<> &Builder, CallBase &CI, bool IsRotateRight)
static Intrinsic::ID shouldUpgradeNVPTXBF16Intrinsic(StringRef Name)
static MDString * upgradeLoopTag(LLVMContext &C, StringRef OldTag)
static bool upgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID, Function *&NewFn)
static Value * upgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0, Value *Op1, Value *Shift, Value *Passthru, Value *Mask, bool IsVALIGN)
static Value * upgradeAbs(IRBuilder<> &Builder, CallBase &CI)
static Value * emitX86Select(IRBuilder<> &Builder, Value *Mask, Value *Op0, Value *Op1)
static Value * upgradeMaskedMove(IRBuilder<> &Builder, CallBase &CI)
static bool upgradeX86IntrinsicFunction(Function *F, StringRef Name, Function *&NewFn)
static Value * applyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec, Value *Mask)
static bool shouldUpgradeX86Intrinsic(Function *F, StringRef Name)
static Value * upgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op, unsigned Shift)
static bool isOldLoopArgument(Metadata *MD)
static Value * upgradeARMIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
static bool upgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID, Function *&NewFn)
static Value * upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
static Value * upgradeMaskedLoad(IRBuilder<> &Builder, Value *Ptr, Value *Passthru, Value *Mask, bool Aligned)
static bool upgradeX86BF16Intrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
static MDType * unwrapMAVOp(CallBase *CI, unsigned Op)
Helper to unwrap intrinsic call MetadataAsValue operands.
static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F, StringRef Name, Function *&NewFn)
static Value * getX86MaskVec(IRBuilder<> &Builder, Value *Mask, unsigned NumElts)
static Value * emitX86ScalarSelect(IRBuilder<> &Builder, Value *Mask, Value *Op0, Value *Op1)
static Value * upgradeX86ConcatShift(IRBuilder<> &Builder, CallBase &CI, bool IsShiftRight, bool ZeroMask)
static void rename(GlobalValue *GV)
static bool upgradePTESTIntrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
static bool upgradeX86BF16DPIntrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
static cl::opt< bool > DisableAutoUpgradeDebugInfo("disable-auto-upgrade-debug-info", cl::desc("Disable autoupgrade of debug info"))
static Value * upgradeMaskedCompare(IRBuilder<> &Builder, CallBase &CI, unsigned CC, bool Signed)
static Value * upgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallBase &CI, Intrinsic::ID IID)
static Value * upgradeX86MaskedShift(IRBuilder<> &Builder, CallBase &CI, Intrinsic::ID IID)
static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder, CallBase &CI, Value *&Rep)
static void upgradeDbgIntrinsicToDbgRecord(StringRef Name, CallBase *CI)
Convert debug intrinsic calls to non-instruction debug records.
static Value * upgradePMULDQ(IRBuilder<> &Builder, CallBase &CI, bool IsSigned)
static Value * upgradeMaskedStore(IRBuilder<> &Builder, Value *Ptr, Value *Data, Value *Mask, bool Aligned)
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file contains constants used for implementing Dwarf debug support.
Module.h This file contains the declarations for the Module class.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements the StringSwitch template, which mimics a switch() statement whose cases are str...
static SymbolRef::Type getType(const Symbol *Sym)
static const X86InstrFMA3Group Groups[]
Class for arbitrary precision integers.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Class to represent array types.
Type * getElementType() const
an instruction that atomically reads a memory location, combines it with another value,...
void setVolatile(bool V)
Specify whether this is a volatile RMW or not.
BinOp
This enumeration lists the possible modifications atomicrmw can make.
@ UIncWrap
Increment one up to a maximum value.
@ UDecWrap
Decrement one until a minimum value or zero.
AttributeSet getFnAttrs() const
The function attributes are returned.
static AttributeList get(LLVMContext &C, ArrayRef< std::pair< unsigned, Attribute > > Attrs)
Create an AttributeList with the specified parameters in it.
AttributeSet getRetAttrs() const
The attributes for the ret value are returned.
void insertDbgRecordBefore(DbgRecord *DR, InstListType::iterator Here)
Insert a DbgRecord into a block at the position given by Here.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Value * getCalledOperand() const
void setAttributes(AttributeList A)
Set the parameter attributes for this call.
Value * getArgOperand(unsigned i) const
FunctionType * getFunctionType() const
Intrinsic::ID getIntrinsicID() const
Returns the intrinsic ID of the intrinsic called or Intrinsic::not_intrinsic if the called function i...
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
void setCalledOperand(Value *V)
unsigned arg_size() const
AttributeList getAttributes() const
Return the parameter attributes for this call.
void setCalledFunction(Function *Fn)
Sets the function called, including updating the function type.
This class represents a function call, abstracting a target machine's calling convention.
void setTailCallKind(TailCallKind TCK)
static CastInst * Create(Instruction::CastOps, Value *S, Type *Ty, const Twine &Name, BasicBlock::iterator InsertBefore)
Provides a way to construct any of the CastInst subclasses using an opcode instead of the subclass's ...
static bool castIsValid(Instruction::CastOps op, Type *SrcTy, Type *DstTy)
This method can be used to determine if a cast from SrcTy to DstTy using Opcode op is valid or not.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
static ConstantAggregateZero * get(Type *Ty)
static Constant * get(ArrayType *T, ArrayRef< Constant * > V)
static Constant * getIntToPtr(Constant *C, Type *Ty, bool OnlyIfReduced=false)
static Constant * getPointerCast(Constant *C, Type *Ty)
Create a BitCast, AddrSpaceCast, or a PtrToInt cast constant expression.
static Constant * getPtrToInt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
This is the shared class of boolean and integer constants.
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
static Constant * get(StructType *T, ArrayRef< Constant * > V)
static ConstantTokenNone * get(LLVMContext &Context)
Return the ConstantTokenNone.
This is an important base class in LLVM.
static Constant * getAllOnesValue(Type *Ty)
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
static DIExpression * append(const DIExpression *Expr, ArrayRef< uint64_t > Ops)
Append the opcodes Ops to DIExpr.
This class represents an Operation in the Expression.
uint64_t getNumOperands() const
Records a position in IR for a source label (DILabel).
Base class for non-instruction debug metadata records that have positions within IR.
Record of a variable value-assignment, aka a non instruction representation of the dbg....
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
Class to represent function types.
Type * getParamType(unsigned i) const
Parameter type accessors.
Type * getReturnType() const
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
void eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing module and deletes it.
Type * getReturnType() const
Returns the type of the ret val.
Argument * getArg(unsigned i) const
LinkageTypes getLinkage() const
Type * getValueType() const
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
bool hasInitializer() const
Definitions have initializers, declarations don't.
ConstantInt * getInt1(bool V)
Get a constant value representing either true or false.
Value * CreateFSub(Value *L, Value *R, const Twine &Name="", MDNode *FPMD=nullptr)
Value * CreateInsertElement(Type *VecTy, Value *NewElt, Value *Idx, const Twine &Name="")
IntegerType * getInt1Ty()
Fetch the type representing a single bit.
Value * CreateInsertValue(Value *Agg, Value *Val, ArrayRef< unsigned > Idxs, const Twine &Name="")
Value * CreateFDiv(Value *L, Value *R, const Twine &Name="", MDNode *FPMD=nullptr)
CallInst * CreateInsertVector(Type *DstType, Value *SrcVec, Value *SubVec, Value *Idx, const Twine &Name="")
Create a call to the vector.insert intrinsic.
Value * CreateSIToFP(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
IntegerType * getIntNTy(unsigned N)
Fetch the type representing an N-bit integer.
LoadInst * CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align, const char *Name)
Value * CreateFAdd(Value *L, Value *R, const Twine &Name="", MDNode *FPMD=nullptr)
Value * CreatePointerCast(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name="")
Return a vector value that contains.
Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")
CallInst * CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment, Value *Mask, Value *PassThru=nullptr, const Twine &Name="")
Create a call to Masked Load intrinsic.
Value * CreateICmpSGE(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
Value * CreateSExt(Value *V, Type *DestTy, const Twine &Name="")
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
ConstantInt * getInt8(uint8_t C)
Get a constant 8-bit value.
Value * CreateUIToFP(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)
IntegerType * getInt16Ty()
Fetch the type representing a 16-bit integer.
Value * CreateNeg(Value *V, const Twine &Name="", bool HasNSW=false)
Value * CreateICmpSLE(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateNot(Value *V, const Twine &Name="")
Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)
Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of converting the string to 'bool...
Value * CreateShl(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")
CallInst * CreateMaskedStore(Value *Val, Value *Ptr, Align Alignment, Value *Mask)
Create a call to Masked Store intrinsic.
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
ConstantInt * getFalse()
Get the constant value for i1 false.
Value * CreateIsNotNull(Value *Arg, const Twine &Name="")
Return a boolean value testing if Arg != 0.
AtomicRMWInst * CreateAtomicRMW(AtomicRMWInst::BinOp Op, Value *Ptr, Value *Val, MaybeAlign Align, AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System)
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="", bool IsNUW=false, bool IsNSW=false)
Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="")
PointerType * getPtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer.
Value * CreateICmpUGE(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateIntCast(Value *V, Type *DestTy, bool isSigned, const Twine &Name="")
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
StoreInst * CreateAlignedStore(Value *Val, Value *Ptr, MaybeAlign Align, bool isVolatile=false)
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateAShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
Value * CreateFPExt(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateXor(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateICmpULE(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateFMul(Value *L, Value *R, const Twine &Name="", MDNode *FPMD=nullptr)
Value * CreateFNeg(Value *V, const Twine &Name="", MDNode *FPMathTag=nullptr)
IntegerType * getInt8Ty()
Fetch the type representing an 8-bit integer.
Type * getBFloatTy()
Fetch the type representing a 16-bit brain floating point value.
Value * CreateMul(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Base class for instruction visitors.
RetTy visitCallBase(CallBase &I)
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
const BasicBlock * getParent() const
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
unsigned getBitWidth() const
Get the number of bits in this IntegerType.
This is an important class for using LLVM in a threaded context.
An instruction for reading from memory.
const MDOperand & getOperand(unsigned I) const
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
unsigned getNumOperands() const
Return number of MDNode operands.
LLVMContext & getContext() const
static MDString * get(LLVMContext &Context, StringRef Str)
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
A Module instance is used to store all the information related to an LLVM module.
ModFlagBehavior
This enumeration defines the supported behaviors of module flags.
@ Override
Uses the specified value, regardless of the behavior or value of the other module.
@ Error
Emits an error if two values disagree, otherwise the resulting value is that of the operands.
@ Min
Takes the min of the two values, which are required to be integers.
@ Max
Takes the max of the two values, which are required to be integers.
bool IsNewDbgInfoFormat
Is this Module using intrinsics to record the position of debugging information, or non-intrinsic rec...
void setOperand(unsigned I, MDNode *New)
MDNode * getOperand(unsigned i) const
unsigned getNumOperands() const
A container for an operand bundle being viewed as a set of values rather than a set of uses.
ArrayRef< InputTy > inputs() const
Class to represent pointers.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
bool match(StringRef String, SmallVectorImpl< StringRef > *Matches=nullptr, std::string *Error=nullptr) const
matches - Match the regex against a given String.
Class to represent scalable SIMD vectors.
uint64_t getMinNumElements() const
Get the minimum number of elements in this vector.
ArrayRef< int > getShuffleMask() const
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
void reserve(size_type N)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
StringRef - Represent a constant reference to a string, i.e.
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
constexpr bool empty() const
empty - Check if the string is empty.
StringRef drop_front(size_t N=1) const
Return a StringRef equal to 'this' but with the first N elements dropped.
constexpr size_t size() const
size - Get the string size.
static constexpr size_t npos
A switch()-like statement whose cases are string literals.
StringSwitch & Case(StringLiteral S, T Value)
StringSwitch & StartsWith(StringLiteral S, T Value)
StringSwitch & Cases(StringLiteral S0, StringLiteral S1, T Value)
Class to represent struct types.
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
unsigned getNumElements() const
Random access to the elements.
Type * getElementType(unsigned N) const
Triple - Helper class for working with autoconf configuration names.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
static Type * getHalfTy(LLVMContext &C)
static Type * getBFloatTy(LLVMContext &C)
bool isBFloatTy() const
Return true if this is 'bfloat', a 16-bit bfloat type.
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
static IntegerType * getInt8Ty(LLVMContext &C)
bool isPtrOrPtrVectorTy() const
Return true if this is a pointer type or a vector of pointer types.
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
static Type * getFloatTy(LLVMContext &C)
bool isIntegerTy() const
True if this is an instance of IntegerType.
bool isFPOrFPVectorTy() const
Return true if this is a FP type or a vector of FP.
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
void setName(const Twine &Name)
Change the name of the value.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
iterator_range< user_iterator > users()
LLVMContext & getContext() const
All values hold a context through their type.
StringRef getName() const
Return a constant reference to the value's name.
void takeName(Value *V)
Transfer the name from V to this value.
Base class of all SIMD vector types.
constexpr ScalarTy getFixedValue() const
self_iterator getIterator()
A raw_ostream that writes to an SmallVector or SmallString.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
AttributeMask typeIncompatible(Type *Ty, AttributeSafetyKind ASK=ASK_ALL)
Which attributes cannot be applied to a type.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ C
The default llvm calling convention, compatible with C.
void getIntrinsicInfoTableEntries(ID id, SmallVectorImpl< IITDescriptor > &T)
Return the IIT table descriptor for the specified intrinsic into an array of IITDescriptors.
std::optional< Function * > remangleIntrinsicFunction(Function *F)
StringRef getName(ID id)
Return the LLVM name for an intrinsic, such as "llvm.ppc.altivec.lvx".
AttributeList getAttributes(LLVMContext &C, ID id)
Return the attributes for an intrinsic.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
This is an optimization pass for GlobalISel generic memory operations.
void UpgradeIntrinsicCall(CallBase *CB, Function *NewFn)
This is the complement to the above, replacing a specific call to an intrinsic function with a call t...
void UpgradeSectionAttributes(Module &M)
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
void UpgradeInlineAsmString(std::string *AsmStr)
Upgrade comment in call to inline asm that represents an objc retain release marker.
bool isValidAtomicOrdering(Int I)
bool UpgradeIntrinsicFunction(Function *F, Function *&NewFn, bool CanUpgradeDebugIntrinsicsToRecords=true)
This is a more granular function that simply checks an intrinsic function for upgrading,...
MDNode * upgradeInstructionLoopAttachment(MDNode &N)
Upgrade the loop attachment metadata node.
void UpgradeAttributes(AttrBuilder &B)
Upgrade attributes that changed format or kind.
void UpgradeCallsToIntrinsic(Function *F)
This is an auto-upgrade hook for any old intrinsic function syntaxes which need to have both the func...
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
bool UpgradeModuleFlags(Module &M)
This checks for module flags which should be upgraded.
void UpgradeOperandBundles(std::vector< OperandBundleDef > &OperandBundles)
Upgrade operand bundles (without knowing about their user instruction).
Constant * UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy)
This is an auto-upgrade for bitcast constant expression between pointers with different address space...
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
std::string UpgradeDataLayoutString(StringRef DL, StringRef Triple)
Upgrade the datalayout string by adding a section for address space pointers.
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
GlobalVariable * UpgradeGlobalVariable(GlobalVariable *GV)
This checks for global variables which should be upgraded.
unsigned getDebugMetadataVersionFromModule(const Module &M)
Return Debug Info Metadata Version by checking module flags.
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
bool StripDebugInfo(Module &M)
Strip debug info in the module if it exists.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ Ref
The access may reference the value stored in memory.
Instruction * UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy, Instruction *&Temp)
This is an auto-upgrade for bitcast between pointers with different address spaces: the instruction i...
@ Dynamic
Denotes mode unknown at compile time.
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
bool UpgradeDebugInfo(Module &M)
Check the debug info version number, if it is out-dated, drop the debug info.
void UpgradeFunctionAttributes(Function &F)
Correct any IR that is relying on old function attribute behavior.
MDNode * UpgradeTBAANode(MDNode &TBAANode)
If the given TBAA tag uses the scalar TBAA format, create a new node corresponding to the upgrade to ...
void UpgradeARCRuntime(Module &M)
Convert calls to ARC runtime functions to intrinsic calls and upgrade the old retain release marker t...
bool verifyModule(const Module &M, raw_ostream *OS=nullptr, bool *BrokenDebugInfo=nullptr)
Check a module for errors.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.
Description of the encoding of one expression Op.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.