LLVM 19.0.0git
SelectionDAGBuilder.cpp
Go to the documentation of this file.
1//===- SelectionDAGBuilder.cpp - Selection-DAG building -------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements routines for translating from LLVM IR into SelectionDAG IR.
10//
11//===----------------------------------------------------------------------===//
12
13#include "SelectionDAGBuilder.h"
14#include "SDNodeDbgValue.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/BitVector.h"
18#include "llvm/ADT/STLExtras.h"
20#include "llvm/ADT/SmallSet.h"
21#include "llvm/ADT/StringRef.h"
22#include "llvm/ADT/Twine.h"
26#include "llvm/Analysis/Loads.h"
58#include "llvm/IR/Argument.h"
59#include "llvm/IR/Attributes.h"
60#include "llvm/IR/BasicBlock.h"
61#include "llvm/IR/CFG.h"
62#include "llvm/IR/CallingConv.h"
63#include "llvm/IR/Constant.h"
65#include "llvm/IR/Constants.h"
66#include "llvm/IR/DataLayout.h"
67#include "llvm/IR/DebugInfo.h"
72#include "llvm/IR/Function.h"
74#include "llvm/IR/InlineAsm.h"
75#include "llvm/IR/InstrTypes.h"
78#include "llvm/IR/Intrinsics.h"
79#include "llvm/IR/IntrinsicsAArch64.h"
80#include "llvm/IR/IntrinsicsAMDGPU.h"
81#include "llvm/IR/IntrinsicsWebAssembly.h"
82#include "llvm/IR/LLVMContext.h"
84#include "llvm/IR/Metadata.h"
85#include "llvm/IR/Module.h"
86#include "llvm/IR/Operator.h"
88#include "llvm/IR/Statepoint.h"
89#include "llvm/IR/Type.h"
90#include "llvm/IR/User.h"
91#include "llvm/IR/Value.h"
92#include "llvm/MC/MCContext.h"
97#include "llvm/Support/Debug.h"
106#include <cstddef>
107#include <iterator>
108#include <limits>
109#include <optional>
110#include <tuple>
111
112using namespace llvm;
113using namespace PatternMatch;
114using namespace SwitchCG;
115
116#define DEBUG_TYPE "isel"
117
118/// LimitFloatPrecision - Generate low-precision inline sequences for
119/// some float libcalls (6, 8 or 12 bits).
120static unsigned LimitFloatPrecision;
121
122static cl::opt<bool>
123 InsertAssertAlign("insert-assert-align", cl::init(true),
124 cl::desc("Insert the experimental `assertalign` node."),
126
128 LimitFPPrecision("limit-float-precision",
129 cl::desc("Generate low-precision inline sequences "
130 "for some float libcalls"),
132 cl::init(0));
133
135 "switch-peel-threshold", cl::Hidden, cl::init(66),
136 cl::desc("Set the case probability threshold for peeling the case from a "
137 "switch statement. A value greater than 100 will void this "
138 "optimization"));
139
140// Limit the width of DAG chains. This is important in general to prevent
141// DAG-based analysis from blowing up. For example, alias analysis and
142// load clustering may not complete in reasonable time. It is difficult to
143// recognize and avoid this situation within each individual analysis, and
144// future analyses are likely to have the same behavior. Limiting DAG width is
145// the safe approach and will be especially important with global DAGs.
146//
147// MaxParallelChains default is arbitrarily high to avoid affecting
148// optimization, but could be lowered to improve compile time. Any ld-ld-st-st
149// sequence over this should have been converted to llvm.memcpy by the
150// frontend. It is easy to induce this behavior with .ll code such as:
151// %buffer = alloca [4096 x i8]
152// %data = load [4096 x i8]* %argPtr
153// store [4096 x i8] %data, [4096 x i8]* %buffer
154static const unsigned MaxParallelChains = 64;
155
157 const SDValue *Parts, unsigned NumParts,
158 MVT PartVT, EVT ValueVT, const Value *V,
159 SDValue InChain,
160 std::optional<CallingConv::ID> CC);
161
162/// getCopyFromParts - Create a value that contains the specified legal parts
163/// combined into the value they represent. If the parts combine to a type
164/// larger than ValueVT then AssertOp can be used to specify whether the extra
165/// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
166/// (ISD::AssertSext).
167static SDValue
168getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts,
169 unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V,
170 SDValue InChain,
171 std::optional<CallingConv::ID> CC = std::nullopt,
172 std::optional<ISD::NodeType> AssertOp = std::nullopt) {
173 // Let the target assemble the parts if it wants to
174 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
175 if (SDValue Val = TLI.joinRegisterPartsIntoValue(DAG, DL, Parts, NumParts,
176 PartVT, ValueVT, CC))
177 return Val;
178
179 if (ValueVT.isVector())
180 return getCopyFromPartsVector(DAG, DL, Parts, NumParts, PartVT, ValueVT, V,
181 InChain, CC);
182
183 assert(NumParts > 0 && "No parts to assemble!");
184 SDValue Val = Parts[0];
185
186 if (NumParts > 1) {
187 // Assemble the value from multiple parts.
188 if (ValueVT.isInteger()) {
189 unsigned PartBits = PartVT.getSizeInBits();
190 unsigned ValueBits = ValueVT.getSizeInBits();
191
192 // Assemble the power of 2 part.
193 unsigned RoundParts = llvm::bit_floor(NumParts);
194 unsigned RoundBits = PartBits * RoundParts;
195 EVT RoundVT = RoundBits == ValueBits ?
196 ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);
197 SDValue Lo, Hi;
198
199 EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);
200
201 if (RoundParts > 2) {
202 Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2, PartVT, HalfVT, V,
203 InChain);
204 Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2, RoundParts / 2,
205 PartVT, HalfVT, V, InChain);
206 } else {
207 Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);
208 Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);
209 }
210
211 if (DAG.getDataLayout().isBigEndian())
212 std::swap(Lo, Hi);
213
214 Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);
215
216 if (RoundParts < NumParts) {
217 // Assemble the trailing non-power-of-2 part.
218 unsigned OddParts = NumParts - RoundParts;
219 EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);
220 Hi = getCopyFromParts(DAG, DL, Parts + RoundParts, OddParts, PartVT,
221 OddVT, V, InChain, CC);
222
223 // Combine the round and odd parts.
224 Lo = Val;
225 if (DAG.getDataLayout().isBigEndian())
226 std::swap(Lo, Hi);
227 EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
228 Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);
229 Hi = DAG.getNode(ISD::SHL, DL, TotalVT, Hi,
230 DAG.getConstant(Lo.getValueSizeInBits(), DL,
232 TotalVT, DAG.getDataLayout())));
233 Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);
234 Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);
235 }
236 } else if (PartVT.isFloatingPoint()) {
237 // FP split into multiple FP parts (for ppcf128)
238 assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
239 "Unexpected split");
240 SDValue Lo, Hi;
241 Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);
242 Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);
243 if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
244 std::swap(Lo, Hi);
245 Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);
246 } else {
247 // FP split into integer parts (soft fp)
248 assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&
249 !PartVT.isVector() && "Unexpected split");
250 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
251 Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V,
252 InChain, CC);
253 }
254 }
255
256 // There is now one part, held in Val. Correct it to match ValueVT.
257 // PartEVT is the type of the register class that holds the value.
258 // ValueVT is the type of the inline asm operation.
259 EVT PartEVT = Val.getValueType();
260
261 if (PartEVT == ValueVT)
262 return Val;
263
264 if (PartEVT.isInteger() && ValueVT.isFloatingPoint() &&
265 ValueVT.bitsLT(PartEVT)) {
266 // For an FP value in an integer part, we need to truncate to the right
267 // width first.
268 PartEVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
269 Val = DAG.getNode(ISD::TRUNCATE, DL, PartEVT, Val);
270 }
271
272 // Handle types that have the same size.
273 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
274 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
275
276 // Handle types with different sizes.
277 if (PartEVT.isInteger() && ValueVT.isInteger()) {
278 if (ValueVT.bitsLT(PartEVT)) {
279 // For a truncate, see if we have any information to
280 // indicate whether the truncated bits will always be
281 // zero or sign-extension.
282 if (AssertOp)
283 Val = DAG.getNode(*AssertOp, DL, PartEVT, Val,
284 DAG.getValueType(ValueVT));
285 return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
286 }
287 return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
288 }
289
290 if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
291 // FP_ROUND's are always exact here.
292 if (ValueVT.bitsLT(Val.getValueType())) {
293
294 SDValue NoChange =
296
298 llvm::Attribute::StrictFP)) {
299 return DAG.getNode(ISD::STRICT_FP_ROUND, DL,
300 DAG.getVTList(ValueVT, MVT::Other), InChain, Val,
301 NoChange);
302 }
303
304 return DAG.getNode(ISD::FP_ROUND, DL, ValueVT, Val, NoChange);
305 }
306
307 return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);
308 }
309
310 // Handle MMX to a narrower integer type by bitcasting MMX to integer and
311 // then truncating.
312 if (PartEVT == MVT::x86mmx && ValueVT.isInteger() &&
313 ValueVT.bitsLT(PartEVT)) {
314 Val = DAG.getNode(ISD::BITCAST, DL, MVT::i64, Val);
315 return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
316 }
317
318 report_fatal_error("Unknown mismatch in getCopyFromParts!");
319}
320
322 const Twine &ErrMsg) {
323 const Instruction *I = dyn_cast_or_null<Instruction>(V);
324 if (!V)
325 return Ctx.emitError(ErrMsg);
326
327 const char *AsmError = ", possible invalid constraint for vector type";
328 if (const CallInst *CI = dyn_cast<CallInst>(I))
329 if (CI->isInlineAsm())
330 return Ctx.emitError(I, ErrMsg + AsmError);
331
332 return Ctx.emitError(I, ErrMsg);
333}
334
335/// getCopyFromPartsVector - Create a value that contains the specified legal
336/// parts combined into the value they represent. If the parts combine to a
337/// type larger than ValueVT then AssertOp can be used to specify whether the
338/// extra bits are known to be zero (ISD::AssertZext) or sign extended from
339/// ValueVT (ISD::AssertSext).
341 const SDValue *Parts, unsigned NumParts,
342 MVT PartVT, EVT ValueVT, const Value *V,
343 SDValue InChain,
344 std::optional<CallingConv::ID> CallConv) {
345 assert(ValueVT.isVector() && "Not a vector value");
346 assert(NumParts > 0 && "No parts to assemble!");
347 const bool IsABIRegCopy = CallConv.has_value();
348
349 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
350 SDValue Val = Parts[0];
351
352 // Handle a multi-element vector.
353 if (NumParts > 1) {
354 EVT IntermediateVT;
355 MVT RegisterVT;
356 unsigned NumIntermediates;
357 unsigned NumRegs;
358
359 if (IsABIRegCopy) {
361 *DAG.getContext(), *CallConv, ValueVT, IntermediateVT,
362 NumIntermediates, RegisterVT);
363 } else {
364 NumRegs =
365 TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
366 NumIntermediates, RegisterVT);
367 }
368
369 assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
370 NumParts = NumRegs; // Silence a compiler warning.
371 assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
372 assert(RegisterVT.getSizeInBits() ==
373 Parts[0].getSimpleValueType().getSizeInBits() &&
374 "Part type sizes don't match!");
375
376 // Assemble the parts into intermediate operands.
377 SmallVector<SDValue, 8> Ops(NumIntermediates);
378 if (NumIntermediates == NumParts) {
379 // If the register was not expanded, truncate or copy the value,
380 // as appropriate.
381 for (unsigned i = 0; i != NumParts; ++i)
382 Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1, PartVT, IntermediateVT,
383 V, InChain, CallConv);
384 } else if (NumParts > 0) {
385 // If the intermediate type was expanded, build the intermediate
386 // operands from the parts.
387 assert(NumParts % NumIntermediates == 0 &&
388 "Must expand into a divisible number of parts!");
389 unsigned Factor = NumParts / NumIntermediates;
390 for (unsigned i = 0; i != NumIntermediates; ++i)
391 Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor, PartVT,
392 IntermediateVT, V, InChain, CallConv);
393 }
394
395 // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
396 // intermediate operands.
397 EVT BuiltVectorTy =
398 IntermediateVT.isVector()
400 *DAG.getContext(), IntermediateVT.getScalarType(),
401 IntermediateVT.getVectorElementCount() * NumParts)
403 IntermediateVT.getScalarType(),
404 NumIntermediates);
405 Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS
407 DL, BuiltVectorTy, Ops);
408 }
409
410 // There is now one part, held in Val. Correct it to match ValueVT.
411 EVT PartEVT = Val.getValueType();
412
413 if (PartEVT == ValueVT)
414 return Val;
415
416 if (PartEVT.isVector()) {
417 // Vector/Vector bitcast.
418 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
419 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
420
421 // If the parts vector has more elements than the value vector, then we
422 // have a vector widening case (e.g. <2 x float> -> <4 x float>).
423 // Extract the elements we want.
424 if (PartEVT.getVectorElementCount() != ValueVT.getVectorElementCount()) {
427 (PartEVT.getVectorElementCount().isScalable() ==
428 ValueVT.getVectorElementCount().isScalable()) &&
429 "Cannot narrow, it would be a lossy transformation");
430 PartEVT =
432 ValueVT.getVectorElementCount());
433 Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, PartEVT, Val,
434 DAG.getVectorIdxConstant(0, DL));
435 if (PartEVT == ValueVT)
436 return Val;
437 if (PartEVT.isInteger() && ValueVT.isFloatingPoint())
438 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
439
440 // Vector/Vector bitcast (e.g. <2 x bfloat> -> <2 x half>).
441 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
442 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
443 }
444
445 // Promoted vector extract
446 return DAG.getAnyExtOrTrunc(Val, DL, ValueVT);
447 }
448
449 // Trivial bitcast if the types are the same size and the destination
450 // vector type is legal.
451 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
452 TLI.isTypeLegal(ValueVT))
453 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
454
455 if (ValueVT.getVectorNumElements() != 1) {
456 // Certain ABIs require that vectors are passed as integers. For vectors
457 // are the same size, this is an obvious bitcast.
458 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) {
459 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
460 } else if (ValueVT.bitsLT(PartEVT)) {
461 const uint64_t ValueSize = ValueVT.getFixedSizeInBits();
462 EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
463 // Drop the extra bits.
464 Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val);
465 return DAG.getBitcast(ValueVT, Val);
466 }
467
469 *DAG.getContext(), V, "non-trivial scalar-to-vector conversion");
470 return DAG.getUNDEF(ValueVT);
471 }
472
473 // Handle cases such as i8 -> <1 x i1>
474 EVT ValueSVT = ValueVT.getVectorElementType();
475 if (ValueVT.getVectorNumElements() == 1 && ValueSVT != PartEVT) {
476 unsigned ValueSize = ValueSVT.getSizeInBits();
477 if (ValueSize == PartEVT.getSizeInBits()) {
478 Val = DAG.getNode(ISD::BITCAST, DL, ValueSVT, Val);
479 } else if (ValueSVT.isFloatingPoint() && PartEVT.isInteger()) {
480 // It's possible a scalar floating point type gets softened to integer and
481 // then promoted to a larger integer. If PartEVT is the larger integer
482 // we need to truncate it and then bitcast to the FP type.
483 assert(ValueSVT.bitsLT(PartEVT) && "Unexpected types");
484 EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
485 Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val);
486 Val = DAG.getBitcast(ValueSVT, Val);
487 } else {
488 Val = ValueVT.isFloatingPoint()
489 ? DAG.getFPExtendOrRound(Val, DL, ValueSVT)
490 : DAG.getAnyExtOrTrunc(Val, DL, ValueSVT);
491 }
492 }
493
494 return DAG.getBuildVector(ValueVT, DL, Val);
495}
496
497static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl,
498 SDValue Val, SDValue *Parts, unsigned NumParts,
499 MVT PartVT, const Value *V,
500 std::optional<CallingConv::ID> CallConv);
501
502/// getCopyToParts - Create a series of nodes that contain the specified value
503/// split into legal parts. If the parts contain more bits than Val, then, for
504/// integers, ExtendKind can be used to specify how to generate the extra bits.
505static void
507 unsigned NumParts, MVT PartVT, const Value *V,
508 std::optional<CallingConv::ID> CallConv = std::nullopt,
509 ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {
510 // Let the target split the parts if it wants to
511 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
512 if (TLI.splitValueIntoRegisterParts(DAG, DL, Val, Parts, NumParts, PartVT,
513 CallConv))
514 return;
515 EVT ValueVT = Val.getValueType();
516
517 // Handle the vector case separately.
518 if (ValueVT.isVector())
519 return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V,
520 CallConv);
521
522 unsigned OrigNumParts = NumParts;
524 "Copying to an illegal type!");
525
526 if (NumParts == 0)
527 return;
528
529 assert(!ValueVT.isVector() && "Vector case handled elsewhere");
530 EVT PartEVT = PartVT;
531 if (PartEVT == ValueVT) {
532 assert(NumParts == 1 && "No-op copy with multiple parts!");
533 Parts[0] = Val;
534 return;
535 }
536
537 unsigned PartBits = PartVT.getSizeInBits();
538 if (NumParts * PartBits > ValueVT.getSizeInBits()) {
539 // If the parts cover more bits than the value has, promote the value.
540 if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
541 assert(NumParts == 1 && "Do not know what to promote to!");
542 Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
543 } else {
544 if (ValueVT.isFloatingPoint()) {
545 // FP values need to be bitcast, then extended if they are being put
546 // into a larger container.
547 ValueVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
548 Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
549 }
550 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
551 ValueVT.isInteger() &&
552 "Unknown mismatch!");
553 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
554 Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);
555 if (PartVT == MVT::x86mmx)
556 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
557 }
558 } else if (PartBits == ValueVT.getSizeInBits()) {
559 // Different types of the same size.
560 assert(NumParts == 1 && PartEVT != ValueVT);
561 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
562 } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
563 // If the parts cover less bits than value has, truncate the value.
564 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
565 ValueVT.isInteger() &&
566 "Unknown mismatch!");
567 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
568 Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
569 if (PartVT == MVT::x86mmx)
570 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
571 }
572
573 // The value may have changed - recompute ValueVT.
574 ValueVT = Val.getValueType();
575 assert(NumParts * PartBits == ValueVT.getSizeInBits() &&
576 "Failed to tile the value with PartVT!");
577
578 if (NumParts == 1) {
579 if (PartEVT != ValueVT) {
581 "scalar-to-vector conversion failed");
582 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
583 }
584
585 Parts[0] = Val;
586 return;
587 }
588
589 // Expand the value into multiple parts.
590 if (NumParts & (NumParts - 1)) {
591 // The number of parts is not a power of 2. Split off and copy the tail.
592 assert(PartVT.isInteger() && ValueVT.isInteger() &&
593 "Do not know what to expand to!");
594 unsigned RoundParts = llvm::bit_floor(NumParts);
595 unsigned RoundBits = RoundParts * PartBits;
596 unsigned OddParts = NumParts - RoundParts;
597 SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
598 DAG.getShiftAmountConstant(RoundBits, ValueVT, DL));
599
600 getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V,
601 CallConv);
602
603 if (DAG.getDataLayout().isBigEndian())
604 // The odd parts were reversed by getCopyToParts - unreverse them.
605 std::reverse(Parts + RoundParts, Parts + NumParts);
606
607 NumParts = RoundParts;
608 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
609 Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
610 }
611
612 // The number of parts is a power of 2. Repeatedly bisect the value using
613 // EXTRACT_ELEMENT.
614 Parts[0] = DAG.getNode(ISD::BITCAST, DL,
616 ValueVT.getSizeInBits()),
617 Val);
618
619 for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
620 for (unsigned i = 0; i < NumParts; i += StepSize) {
621 unsigned ThisBits = StepSize * PartBits / 2;
622 EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);
623 SDValue &Part0 = Parts[i];
624 SDValue &Part1 = Parts[i+StepSize/2];
625
626 Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
627 ThisVT, Part0, DAG.getIntPtrConstant(1, DL));
628 Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
629 ThisVT, Part0, DAG.getIntPtrConstant(0, DL));
630
631 if (ThisBits == PartBits && ThisVT != PartVT) {
632 Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);
633 Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);
634 }
635 }
636 }
637
638 if (DAG.getDataLayout().isBigEndian())
639 std::reverse(Parts, Parts + OrigNumParts);
640}
641
643 const SDLoc &DL, EVT PartVT) {
644 if (!PartVT.isVector())
645 return SDValue();
646
647 EVT ValueVT = Val.getValueType();
648 EVT PartEVT = PartVT.getVectorElementType();
649 EVT ValueEVT = ValueVT.getVectorElementType();
650 ElementCount PartNumElts = PartVT.getVectorElementCount();
651 ElementCount ValueNumElts = ValueVT.getVectorElementCount();
652
653 // We only support widening vectors with equivalent element types and
654 // fixed/scalable properties. If a target needs to widen a fixed-length type
655 // to a scalable one, it should be possible to use INSERT_SUBVECTOR below.
656 if (ElementCount::isKnownLE(PartNumElts, ValueNumElts) ||
657 PartNumElts.isScalable() != ValueNumElts.isScalable())
658 return SDValue();
659
660 // Have a try for bf16 because some targets share its ABI with fp16.
661 if (ValueEVT == MVT::bf16 && PartEVT == MVT::f16) {
663 "Cannot widen to illegal type");
664 Val = DAG.getNode(ISD::BITCAST, DL,
665 ValueVT.changeVectorElementType(MVT::f16), Val);
666 } else if (PartEVT != ValueEVT) {
667 return SDValue();
668 }
669
670 // Widening a scalable vector to another scalable vector is done by inserting
671 // the vector into a larger undef one.
672 if (PartNumElts.isScalable())
673 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, PartVT, DAG.getUNDEF(PartVT),
674 Val, DAG.getVectorIdxConstant(0, DL));
675
676 // Vector widening case, e.g. <2 x float> -> <4 x float>. Shuffle in
677 // undef elements.
679 DAG.ExtractVectorElements(Val, Ops);
680 SDValue EltUndef = DAG.getUNDEF(PartEVT);
681 Ops.append((PartNumElts - ValueNumElts).getFixedValue(), EltUndef);
682
683 // FIXME: Use CONCAT for 2x -> 4x.
684 return DAG.getBuildVector(PartVT, DL, Ops);
685}
686
687/// getCopyToPartsVector - Create a series of nodes that contain the specified
688/// value split into legal parts.
689static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,
690 SDValue Val, SDValue *Parts, unsigned NumParts,
691 MVT PartVT, const Value *V,
692 std::optional<CallingConv::ID> CallConv) {
693 EVT ValueVT = Val.getValueType();
694 assert(ValueVT.isVector() && "Not a vector");
695 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
696 const bool IsABIRegCopy = CallConv.has_value();
697
698 if (NumParts == 1) {
699 EVT PartEVT = PartVT;
700 if (PartEVT == ValueVT) {
701 // Nothing to do.
702 } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
703 // Bitconvert vector->vector case.
704 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
705 } else if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT)) {
706 Val = Widened;
707 } else if (PartVT.isVector() &&
709 ValueVT.getVectorElementType()) &&
710 PartEVT.getVectorElementCount() ==
711 ValueVT.getVectorElementCount()) {
712
713 // Promoted vector extract
714 Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
715 } else if (PartEVT.isVector() &&
716 PartEVT.getVectorElementType() !=
717 ValueVT.getVectorElementType() &&
718 TLI.getTypeAction(*DAG.getContext(), ValueVT) ==
719 TargetLowering::TypeWidenVector) {
720 // Combination of widening and promotion.
721 EVT WidenVT =
723 PartVT.getVectorElementCount());
724 SDValue Widened = widenVectorToPartType(DAG, Val, DL, WidenVT);
725 Val = DAG.getAnyExtOrTrunc(Widened, DL, PartVT);
726 } else {
727 // Don't extract an integer from a float vector. This can happen if the
728 // FP type gets softened to integer and then promoted. The promotion
729 // prevents it from being picked up by the earlier bitcast case.
730 if (ValueVT.getVectorElementCount().isScalar() &&
731 (!ValueVT.isFloatingPoint() || !PartVT.isInteger())) {
732 Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val,
733 DAG.getVectorIdxConstant(0, DL));
734 } else {
735 uint64_t ValueSize = ValueVT.getFixedSizeInBits();
736 assert(PartVT.getFixedSizeInBits() > ValueSize &&
737 "lossy conversion of vector to scalar type");
738 EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
739 Val = DAG.getBitcast(IntermediateType, Val);
740 Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
741 }
742 }
743
744 assert(Val.getValueType() == PartVT && "Unexpected vector part value type");
745 Parts[0] = Val;
746 return;
747 }
748
749 // Handle a multi-element vector.
750 EVT IntermediateVT;
751 MVT RegisterVT;
752 unsigned NumIntermediates;
753 unsigned NumRegs;
754 if (IsABIRegCopy) {
756 *DAG.getContext(), *CallConv, ValueVT, IntermediateVT, NumIntermediates,
757 RegisterVT);
758 } else {
759 NumRegs =
760 TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
761 NumIntermediates, RegisterVT);
762 }
763
764 assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
765 NumParts = NumRegs; // Silence a compiler warning.
766 assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
767
768 assert(IntermediateVT.isScalableVector() == ValueVT.isScalableVector() &&
769 "Mixing scalable and fixed vectors when copying in parts");
770
771 std::optional<ElementCount> DestEltCnt;
772
773 if (IntermediateVT.isVector())
774 DestEltCnt = IntermediateVT.getVectorElementCount() * NumIntermediates;
775 else
776 DestEltCnt = ElementCount::getFixed(NumIntermediates);
777
778 EVT BuiltVectorTy = EVT::getVectorVT(
779 *DAG.getContext(), IntermediateVT.getScalarType(), *DestEltCnt);
780
781 if (ValueVT == BuiltVectorTy) {
782 // Nothing to do.
783 } else if (ValueVT.getSizeInBits() == BuiltVectorTy.getSizeInBits()) {
784 // Bitconvert vector->vector case.
785 Val = DAG.getNode(ISD::BITCAST, DL, BuiltVectorTy, Val);
786 } else {
787 if (BuiltVectorTy.getVectorElementType().bitsGT(
788 ValueVT.getVectorElementType())) {
789 // Integer promotion.
790 ValueVT = EVT::getVectorVT(*DAG.getContext(),
791 BuiltVectorTy.getVectorElementType(),
792 ValueVT.getVectorElementCount());
793 Val = DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
794 }
795
796 if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, BuiltVectorTy)) {
797 Val = Widened;
798 }
799 }
800
801 assert(Val.getValueType() == BuiltVectorTy && "Unexpected vector value type");
802
803 // Split the vector into intermediate operands.
804 SmallVector<SDValue, 8> Ops(NumIntermediates);
805 for (unsigned i = 0; i != NumIntermediates; ++i) {
806 if (IntermediateVT.isVector()) {
807 // This does something sensible for scalable vectors - see the
808 // definition of EXTRACT_SUBVECTOR for further details.
809 unsigned IntermediateNumElts = IntermediateVT.getVectorMinNumElements();
810 Ops[i] =
811 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val,
812 DAG.getVectorIdxConstant(i * IntermediateNumElts, DL));
813 } else {
814 Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val,
815 DAG.getVectorIdxConstant(i, DL));
816 }
817 }
818
819 // Split the intermediate operands into legal parts.
820 if (NumParts == NumIntermediates) {
821 // If the register was not expanded, promote or copy the value,
822 // as appropriate.
823 for (unsigned i = 0; i != NumParts; ++i)
824 getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V, CallConv);
825 } else if (NumParts > 0) {
826 // If the intermediate type was expanded, split each the value into
827 // legal parts.
828 assert(NumIntermediates != 0 && "division by zero");
829 assert(NumParts % NumIntermediates == 0 &&
830 "Must expand into a divisible number of parts!");
831 unsigned Factor = NumParts / NumIntermediates;
832 for (unsigned i = 0; i != NumIntermediates; ++i)
833 getCopyToParts(DAG, DL, Ops[i], &Parts[i * Factor], Factor, PartVT, V,
834 CallConv);
835 }
836}
837
839 EVT valuevt, std::optional<CallingConv::ID> CC)
840 : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs),
841 RegCount(1, regs.size()), CallConv(CC) {}
842
844 const DataLayout &DL, unsigned Reg, Type *Ty,
845 std::optional<CallingConv::ID> CC) {
846 ComputeValueVTs(TLI, DL, Ty, ValueVTs);
847
848 CallConv = CC;
849
850 for (EVT ValueVT : ValueVTs) {
851 unsigned NumRegs =
854 : TLI.getNumRegisters(Context, ValueVT);
855 MVT RegisterVT =
858 : TLI.getRegisterType(Context, ValueVT);
859 for (unsigned i = 0; i != NumRegs; ++i)
860 Regs.push_back(Reg + i);
861 RegVTs.push_back(RegisterVT);
862 RegCount.push_back(NumRegs);
863 Reg += NumRegs;
864 }
865}
866
868 FunctionLoweringInfo &FuncInfo,
869 const SDLoc &dl, SDValue &Chain,
870 SDValue *Glue, const Value *V) const {
871 // A Value with type {} or [0 x %t] needs no registers.
872 if (ValueVTs.empty())
873 return SDValue();
874
875 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
876
877 // Assemble the legal parts into the final values.
878 SmallVector<SDValue, 4> Values(ValueVTs.size());
880 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
881 // Copy the legal parts from the registers.
882 EVT ValueVT = ValueVTs[Value];
883 unsigned NumRegs = RegCount[Value];
884 MVT RegisterVT = isABIMangled()
886 *DAG.getContext(), *CallConv, RegVTs[Value])
887 : RegVTs[Value];
888
889 Parts.resize(NumRegs);
890 for (unsigned i = 0; i != NumRegs; ++i) {
891 SDValue P;
892 if (!Glue) {
893 P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);
894 } else {
895 P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Glue);
896 *Glue = P.getValue(2);
897 }
898
899 Chain = P.getValue(1);
900 Parts[i] = P;
901
902 // If the source register was virtual and if we know something about it,
903 // add an assert node.
904 if (!Register::isVirtualRegister(Regs[Part + i]) ||
905 !RegisterVT.isInteger())
906 continue;
907
909 FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);
910 if (!LOI)
911 continue;
912
913 unsigned RegSize = RegisterVT.getScalarSizeInBits();
914 unsigned NumSignBits = LOI->NumSignBits;
915 unsigned NumZeroBits = LOI->Known.countMinLeadingZeros();
916
917 if (NumZeroBits == RegSize) {
918 // The current value is a zero.
919 // Explicitly express that as it would be easier for
920 // optimizations to kick in.
921 Parts[i] = DAG.getConstant(0, dl, RegisterVT);
922 continue;
923 }
924
925 // FIXME: We capture more information than the dag can represent. For
926 // now, just use the tightest assertzext/assertsext possible.
927 bool isSExt;
928 EVT FromVT(MVT::Other);
929 if (NumZeroBits) {
930 FromVT = EVT::getIntegerVT(*DAG.getContext(), RegSize - NumZeroBits);
931 isSExt = false;
932 } else if (NumSignBits > 1) {
933 FromVT =
934 EVT::getIntegerVT(*DAG.getContext(), RegSize - NumSignBits + 1);
935 isSExt = true;
936 } else {
937 continue;
938 }
939 // Add an assertion node.
940 assert(FromVT != MVT::Other);
941 Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,
942 RegisterVT, P, DAG.getValueType(FromVT));
943 }
944
945 Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(), NumRegs,
946 RegisterVT, ValueVT, V, Chain, CallConv);
947 Part += NumRegs;
948 Parts.clear();
949 }
950
951 return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);
952}
953
955 const SDLoc &dl, SDValue &Chain, SDValue *Glue,
956 const Value *V,
957 ISD::NodeType PreferredExtendType) const {
958 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
959 ISD::NodeType ExtendKind = PreferredExtendType;
960
961 // Get the list of the values's legal parts.
962 unsigned NumRegs = Regs.size();
963 SmallVector<SDValue, 8> Parts(NumRegs);
964 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
965 unsigned NumParts = RegCount[Value];
966
967 MVT RegisterVT = isABIMangled()
969 *DAG.getContext(), *CallConv, RegVTs[Value])
970 : RegVTs[Value];
971
972 if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT))
973 ExtendKind = ISD::ZERO_EXTEND;
974
975 getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value), &Parts[Part],
976 NumParts, RegisterVT, V, CallConv, ExtendKind);
977 Part += NumParts;
978 }
979
980 // Copy the parts into the registers.
981 SmallVector<SDValue, 8> Chains(NumRegs);
982 for (unsigned i = 0; i != NumRegs; ++i) {
983 SDValue Part;
984 if (!Glue) {
985 Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);
986 } else {
987 Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Glue);
988 *Glue = Part.getValue(1);
989 }
990
991 Chains[i] = Part.getValue(0);
992 }
993
994 if (NumRegs == 1 || Glue)
995 // If NumRegs > 1 && Glue is used then the use of the last CopyToReg is
996 // flagged to it. That is the CopyToReg nodes and the user are considered
997 // a single scheduling unit. If we create a TokenFactor and return it as
998 // chain, then the TokenFactor is both a predecessor (operand) of the
999 // user as well as a successor (the TF operands are flagged to the user).
1000 // c1, f1 = CopyToReg
1001 // c2, f2 = CopyToReg
1002 // c3 = TokenFactor c1, c2
1003 // ...
1004 // = op c3, ..., f2
1005 Chain = Chains[NumRegs-1];
1006 else
1007 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
1008}
1009
1011 unsigned MatchingIdx, const SDLoc &dl,
1012 SelectionDAG &DAG,
1013 std::vector<SDValue> &Ops) const {
1014 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1015
1016 InlineAsm::Flag Flag(Code, Regs.size());
1017 if (HasMatching)
1018 Flag.setMatchingOp(MatchingIdx);
1019 else if (!Regs.empty() && Register::isVirtualRegister(Regs.front())) {
1020 // Put the register class of the virtual registers in the flag word. That
1021 // way, later passes can recompute register class constraints for inline
1022 // assembly as well as normal instructions.
1023 // Don't do this for tied operands that can use the regclass information
1024 // from the def.
1026 const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());
1027 Flag.setRegClass(RC->getID());
1028 }
1029
1030 SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32);
1031 Ops.push_back(Res);
1032
1033 if (Code == InlineAsm::Kind::Clobber) {
1034 // Clobbers should always have a 1:1 mapping with registers, and may
1035 // reference registers that have illegal (e.g. vector) types. Hence, we
1036 // shouldn't try to apply any sort of splitting logic to them.
1037 assert(Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() &&
1038 "No 1:1 mapping from clobbers to regs?");
1040 (void)SP;
1041 for (unsigned I = 0, E = ValueVTs.size(); I != E; ++I) {
1042 Ops.push_back(DAG.getRegister(Regs[I], RegVTs[I]));
1043 assert(
1044 (Regs[I] != SP ||
1046 "If we clobbered the stack pointer, MFI should know about it.");
1047 }
1048 return;
1049 }
1050
1051 for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
1052 MVT RegisterVT = RegVTs[Value];
1053 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value],
1054 RegisterVT);
1055 for (unsigned i = 0; i != NumRegs; ++i) {
1056 assert(Reg < Regs.size() && "Mismatch in # registers expected");
1057 unsigned TheReg = Regs[Reg++];
1058 Ops.push_back(DAG.getRegister(TheReg, RegisterVT));
1059 }
1060 }
1061}
1062
1066 unsigned I = 0;
1067 for (auto CountAndVT : zip_first(RegCount, RegVTs)) {
1068 unsigned RegCount = std::get<0>(CountAndVT);
1069 MVT RegisterVT = std::get<1>(CountAndVT);
1070 TypeSize RegisterSize = RegisterVT.getSizeInBits();
1071 for (unsigned E = I + RegCount; I != E; ++I)
1072 OutVec.push_back(std::make_pair(Regs[I], RegisterSize));
1073 }
1074 return OutVec;
1075}
1076
1078 AssumptionCache *ac,
1079 const TargetLibraryInfo *li) {
1080 AA = aa;
1081 AC = ac;
1082 GFI = gfi;
1083 LibInfo = li;
1085 LPadToCallSiteMap.clear();
1087 AssignmentTrackingEnabled = isAssignmentTrackingEnabled(
1089}
1090
1092 NodeMap.clear();
1093 UnusedArgNodeMap.clear();
1094 PendingLoads.clear();
1095 PendingExports.clear();
1096 PendingConstrainedFP.clear();
1097 PendingConstrainedFPStrict.clear();
1098 CurInst = nullptr;
1099 HasTailCall = false;
1100 SDNodeOrder = LowestSDNodeOrder;
1102}
1103
1105 DanglingDebugInfoMap.clear();
1106}
1107
1108// Update DAG root to include dependencies on Pending chains.
1109SDValue SelectionDAGBuilder::updateRoot(SmallVectorImpl<SDValue> &Pending) {
1110 SDValue Root = DAG.getRoot();
1111
1112 if (Pending.empty())
1113 return Root;
1114
1115 // Add current root to PendingChains, unless we already indirectly
1116 // depend on it.
1117 if (Root.getOpcode() != ISD::EntryToken) {
1118 unsigned i = 0, e = Pending.size();
1119 for (; i != e; ++i) {
1120 assert(Pending[i].getNode()->getNumOperands() > 1);
1121 if (Pending[i].getNode()->getOperand(0) == Root)
1122 break; // Don't add the root if we already indirectly depend on it.
1123 }
1124
1125 if (i == e)
1126 Pending.push_back(Root);
1127 }
1128
1129 if (Pending.size() == 1)
1130 Root = Pending[0];
1131 else
1132 Root = DAG.getTokenFactor(getCurSDLoc(), Pending);
1133
1134 DAG.setRoot(Root);
1135 Pending.clear();
1136 return Root;
1137}
1138
1140 return updateRoot(PendingLoads);
1141}
1142
1144 // Chain up all pending constrained intrinsics together with all
1145 // pending loads, by simply appending them to PendingLoads and
1146 // then calling getMemoryRoot().
1147 PendingLoads.reserve(PendingLoads.size() +
1148 PendingConstrainedFP.size() +
1149 PendingConstrainedFPStrict.size());
1150 PendingLoads.append(PendingConstrainedFP.begin(),
1151 PendingConstrainedFP.end());
1152 PendingLoads.append(PendingConstrainedFPStrict.begin(),
1153 PendingConstrainedFPStrict.end());
1154 PendingConstrainedFP.clear();
1155 PendingConstrainedFPStrict.clear();
1156 return getMemoryRoot();
1157}
1158
1160 // We need to emit pending fpexcept.strict constrained intrinsics,
1161 // so append them to the PendingExports list.
1162 PendingExports.append(PendingConstrainedFPStrict.begin(),
1163 PendingConstrainedFPStrict.end());
1164 PendingConstrainedFPStrict.clear();
1165 return updateRoot(PendingExports);
1166}
1167
1169 DILocalVariable *Variable,
1171 DebugLoc DL) {
1172 assert(Variable && "Missing variable");
1173
1174 // Check if address has undef value.
1175 if (!Address || isa<UndefValue>(Address) ||
1176 (Address->use_empty() && !isa<Argument>(Address))) {
1177 LLVM_DEBUG(
1178 dbgs()
1179 << "dbg_declare: Dropping debug info (bad/undef/unused-arg address)\n");
1180 return;
1181 }
1182
1183 bool IsParameter = Variable->isParameter() || isa<Argument>(Address);
1184
1185 SDValue &N = NodeMap[Address];
1186 if (!N.getNode() && isa<Argument>(Address))
1187 // Check unused arguments map.
1188 N = UnusedArgNodeMap[Address];
1189 SDDbgValue *SDV;
1190 if (N.getNode()) {
1191 if (const BitCastInst *BCI = dyn_cast<BitCastInst>(Address))
1192 Address = BCI->getOperand(0);
1193 // Parameters are handled specially.
1194 auto *FINode = dyn_cast<FrameIndexSDNode>(N.getNode());
1195 if (IsParameter && FINode) {
1196 // Byval parameter. We have a frame index at this point.
1197 SDV = DAG.getFrameIndexDbgValue(Variable, Expression, FINode->getIndex(),
1198 /*IsIndirect*/ true, DL, SDNodeOrder);
1199 } else if (isa<Argument>(Address)) {
1200 // Address is an argument, so try to emit its dbg value using
1201 // virtual register info from the FuncInfo.ValueMap.
1202 EmitFuncArgumentDbgValue(Address, Variable, Expression, DL,
1203 FuncArgumentDbgValueKind::Declare, N);
1204 return;
1205 } else {
1206 SDV = DAG.getDbgValue(Variable, Expression, N.getNode(), N.getResNo(),
1207 true, DL, SDNodeOrder);
1208 }
1209 DAG.AddDbgValue(SDV, IsParameter);
1210 } else {
1211 // If Address is an argument then try to emit its dbg value using
1212 // virtual register info from the FuncInfo.ValueMap.
1213 if (!EmitFuncArgumentDbgValue(Address, Variable, Expression, DL,
1214 FuncArgumentDbgValueKind::Declare, N)) {
1215 LLVM_DEBUG(dbgs() << "dbg_declare: Dropping debug info"
1216 << " (could not emit func-arg dbg_value)\n");
1217 }
1218 }
1219 return;
1220}
1221
1223 // Add SDDbgValue nodes for any var locs here. Do so before updating
1224 // SDNodeOrder, as this mapping is {Inst -> Locs BEFORE Inst}.
1225 if (FunctionVarLocs const *FnVarLocs = DAG.getFunctionVarLocs()) {
1226 // Add SDDbgValue nodes for any var locs here. Do so before updating
1227 // SDNodeOrder, as this mapping is {Inst -> Locs BEFORE Inst}.
1228 for (auto It = FnVarLocs->locs_begin(&I), End = FnVarLocs->locs_end(&I);
1229 It != End; ++It) {
1230 auto *Var = FnVarLocs->getDILocalVariable(It->VariableID);
1231 dropDanglingDebugInfo(Var, It->Expr);
1232 if (It->Values.isKillLocation(It->Expr)) {
1233 handleKillDebugValue(Var, It->Expr, It->DL, SDNodeOrder);
1234 continue;
1235 }
1236 SmallVector<Value *> Values(It->Values.location_ops());
1237 if (!handleDebugValue(Values, Var, It->Expr, It->DL, SDNodeOrder,
1238 It->Values.hasArgList())) {
1240 for (Value *V : It->Values.location_ops())
1241 Vals.push_back(V);
1243 FnVarLocs->getDILocalVariable(It->VariableID),
1244 It->Expr, Vals.size() > 1, It->DL, SDNodeOrder);
1245 }
1246 }
1247 }
1248
1249 // We must skip DbgVariableRecords if they've already been processed above as
1250 // we have just emitted the debug values resulting from assignment tracking
1251 // analysis, making any existing DbgVariableRecords redundant (and probably
1252 // less correct). We still need to process DbgLabelRecords. This does sink
1253 // DbgLabelRecords to the bottom of the group of debug records. That sholdn't
1254 // be important as it does so deterministcally and ordering between
1255 // DbgLabelRecords and DbgVariableRecords is immaterial (other than for MIR/IR
1256 // printing).
1257 bool SkipDbgVariableRecords = DAG.getFunctionVarLocs();
1258 // Is there is any debug-info attached to this instruction, in the form of
1259 // DbgRecord non-instruction debug-info records.
1260 for (DbgRecord &DR : I.getDbgRecordRange()) {
1261 if (DbgLabelRecord *DLR = dyn_cast<DbgLabelRecord>(&DR)) {
1262 assert(DLR->getLabel() && "Missing label");
1263 SDDbgLabel *SDV =
1264 DAG.getDbgLabel(DLR->getLabel(), DLR->getDebugLoc(), SDNodeOrder);
1265 DAG.AddDbgLabel(SDV);
1266 continue;
1267 }
1268
1269 if (SkipDbgVariableRecords)
1270 continue;
1271 DbgVariableRecord &DVR = cast<DbgVariableRecord>(DR);
1272 DILocalVariable *Variable = DVR.getVariable();
1275
1277 if (FuncInfo.PreprocessedDVRDeclares.contains(&DVR))
1278 continue;
1279 LLVM_DEBUG(dbgs() << "SelectionDAG visiting dbg_declare: " << DVR
1280 << "\n");
1282 DVR.getDebugLoc());
1283 continue;
1284 }
1285
1286 // A DbgVariableRecord with no locations is a kill location.
1288 if (Values.empty()) {
1290 SDNodeOrder);
1291 continue;
1292 }
1293
1294 // A DbgVariableRecord with an undef or absent location is also a kill
1295 // location.
1296 if (llvm::any_of(Values,
1297 [](Value *V) { return !V || isa<UndefValue>(V); })) {
1299 SDNodeOrder);
1300 continue;
1301 }
1302
1303 bool IsVariadic = DVR.hasArgList();
1304 if (!handleDebugValue(Values, Variable, Expression, DVR.getDebugLoc(),
1305 SDNodeOrder, IsVariadic)) {
1306 addDanglingDebugInfo(Values, Variable, Expression, IsVariadic,
1307 DVR.getDebugLoc(), SDNodeOrder);
1308 }
1309 }
1310}
1311
1313 visitDbgInfo(I);
1314
1315 // Set up outgoing PHI node register values before emitting the terminator.
1316 if (I.isTerminator()) {
1317 HandlePHINodesInSuccessorBlocks(I.getParent());
1318 }
1319
1320 // Increase the SDNodeOrder if dealing with a non-debug instruction.
1321 if (!isa<DbgInfoIntrinsic>(I))
1322 ++SDNodeOrder;
1323
1324 CurInst = &I;
1325
1326 // Set inserted listener only if required.
1327 bool NodeInserted = false;
1328 std::unique_ptr<SelectionDAG::DAGNodeInsertedListener> InsertedListener;
1329 MDNode *PCSectionsMD = I.getMetadata(LLVMContext::MD_pcsections);
1330 MDNode *MMRA = I.getMetadata(LLVMContext::MD_mmra);
1331 if (PCSectionsMD || MMRA) {
1332 InsertedListener = std::make_unique<SelectionDAG::DAGNodeInsertedListener>(
1333 DAG, [&](SDNode *) { NodeInserted = true; });
1334 }
1335
1336 visit(I.getOpcode(), I);
1337
1338 if (!I.isTerminator() && !HasTailCall &&
1339 !isa<GCStatepointInst>(I)) // statepoints handle their exports internally
1341
1342 // Handle metadata.
1343 if (PCSectionsMD || MMRA) {
1344 auto It = NodeMap.find(&I);
1345 if (It != NodeMap.end()) {
1346 if (PCSectionsMD)
1347 DAG.addPCSections(It->second.getNode(), PCSectionsMD);
1348 if (MMRA)
1349 DAG.addMMRAMetadata(It->second.getNode(), MMRA);
1350 } else if (NodeInserted) {
1351 // This should not happen; if it does, don't let it go unnoticed so we can
1352 // fix it. Relevant visit*() function is probably missing a setValue().
1353 errs() << "warning: loosing !pcsections and/or !mmra metadata ["
1354 << I.getModule()->getName() << "]\n";
1355 LLVM_DEBUG(I.dump());
1356 assert(false);
1357 }
1358 }
1359
1360 CurInst = nullptr;
1361}
1362
1363void SelectionDAGBuilder::visitPHI(const PHINode &) {
1364 llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!");
1365}
1366
1367void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
1368 // Note: this doesn't use InstVisitor, because it has to work with
1369 // ConstantExpr's in addition to instructions.
1370 switch (Opcode) {
1371 default: llvm_unreachable("Unknown instruction type encountered!");
1372 // Build the switch statement using the Instruction.def file.
1373#define HANDLE_INST(NUM, OPCODE, CLASS) \
1374 case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
1375#include "llvm/IR/Instruction.def"
1376 }
1377}
1378
1380 DILocalVariable *Variable,
1381 DebugLoc DL, unsigned Order,
1384 // For variadic dbg_values we will now insert an undef.
1385 // FIXME: We can potentially recover these!
1387 for (const Value *V : Values) {
1388 auto *Undef = UndefValue::get(V->getType());
1390 }
1391 SDDbgValue *SDV = DAG.getDbgValueList(Variable, Expression, Locs, {},
1392 /*IsIndirect=*/false, DL, Order,
1393 /*IsVariadic=*/true);
1394 DAG.AddDbgValue(SDV, /*isParameter=*/false);
1395 return true;
1396}
1397
1399 DILocalVariable *Var,
1400 DIExpression *Expr,
1401 bool IsVariadic, DebugLoc DL,
1402 unsigned Order) {
1403 if (IsVariadic) {
1404 handleDanglingVariadicDebugInfo(DAG, Var, DL, Order, Values, Expr);
1405 return;
1406 }
1407 // TODO: Dangling debug info will eventually either be resolved or produce
1408 // an Undef DBG_VALUE. However in the resolution case, a gap may appear
1409 // between the original dbg.value location and its resolved DBG_VALUE,
1410 // which we should ideally fill with an extra Undef DBG_VALUE.
1411 assert(Values.size() == 1);
1412 DanglingDebugInfoMap[Values[0]].emplace_back(Var, Expr, DL, Order);
1413}
1414
1416 const DIExpression *Expr) {
1417 auto isMatchingDbgValue = [&](DanglingDebugInfo &DDI) {
1418 DIVariable *DanglingVariable = DDI.getVariable();
1419 DIExpression *DanglingExpr = DDI.getExpression();
1420 if (DanglingVariable == Variable && Expr->fragmentsOverlap(DanglingExpr)) {
1421 LLVM_DEBUG(dbgs() << "Dropping dangling debug info for "
1422 << printDDI(nullptr, DDI) << "\n");
1423 return true;
1424 }
1425 return false;
1426 };
1427
1428 for (auto &DDIMI : DanglingDebugInfoMap) {
1429 DanglingDebugInfoVector &DDIV = DDIMI.second;
1430
1431 // If debug info is to be dropped, run it through final checks to see
1432 // whether it can be salvaged.
1433 for (auto &DDI : DDIV)
1434 if (isMatchingDbgValue(DDI))
1435 salvageUnresolvedDbgValue(DDIMI.first, DDI);
1436
1437 erase_if(DDIV, isMatchingDbgValue);
1438 }
1439}
1440
1441// resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
1442// generate the debug data structures now that we've seen its definition.
1444 SDValue Val) {
1445 auto DanglingDbgInfoIt = DanglingDebugInfoMap.find(V);
1446 if (DanglingDbgInfoIt == DanglingDebugInfoMap.end())
1447 return;
1448
1449 DanglingDebugInfoVector &DDIV = DanglingDbgInfoIt->second;
1450 for (auto &DDI : DDIV) {
1451 DebugLoc DL = DDI.getDebugLoc();
1452 unsigned ValSDNodeOrder = Val.getNode()->getIROrder();
1453 unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
1454 DILocalVariable *Variable = DDI.getVariable();
1455 DIExpression *Expr = DDI.getExpression();
1457 "Expected inlined-at fields to agree");
1458 SDDbgValue *SDV;
1459 if (Val.getNode()) {
1460 // FIXME: I doubt that it is correct to resolve a dangling DbgValue as a
1461 // FuncArgumentDbgValue (it would be hoisted to the function entry, and if
1462 // we couldn't resolve it directly when examining the DbgValue intrinsic
1463 // in the first place we should not be more successful here). Unless we
1464 // have some test case that prove this to be correct we should avoid
1465 // calling EmitFuncArgumentDbgValue here.
1466 if (!EmitFuncArgumentDbgValue(V, Variable, Expr, DL,
1467 FuncArgumentDbgValueKind::Value, Val)) {
1468 LLVM_DEBUG(dbgs() << "Resolve dangling debug info for "
1469 << printDDI(V, DDI) << "\n");
1470 LLVM_DEBUG(dbgs() << " By mapping to:\n "; Val.dump());
1471 // Increase the SDNodeOrder for the DbgValue here to make sure it is
1472 // inserted after the definition of Val when emitting the instructions
1473 // after ISel. An alternative could be to teach
1474 // ScheduleDAGSDNodes::EmitSchedule to delay the insertion properly.
1475 LLVM_DEBUG(if (ValSDNodeOrder > DbgSDNodeOrder) dbgs()
1476 << "changing SDNodeOrder from " << DbgSDNodeOrder << " to "
1477 << ValSDNodeOrder << "\n");
1478 SDV = getDbgValue(Val, Variable, Expr, DL,
1479 std::max(DbgSDNodeOrder, ValSDNodeOrder));
1480 DAG.AddDbgValue(SDV, false);
1481 } else
1482 LLVM_DEBUG(dbgs() << "Resolved dangling debug info for "
1483 << printDDI(V, DDI)
1484 << " in EmitFuncArgumentDbgValue\n");
1485 } else {
1486 LLVM_DEBUG(dbgs() << "Dropping debug info for " << printDDI(V, DDI)
1487 << "\n");
1488 auto Undef = UndefValue::get(V->getType());
1489 auto SDV =
1490 DAG.getConstantDbgValue(Variable, Expr, Undef, DL, DbgSDNodeOrder);
1491 DAG.AddDbgValue(SDV, false);
1492 }
1493 }
1494 DDIV.clear();
1495}
1496
1498 DanglingDebugInfo &DDI) {
1499 // TODO: For the variadic implementation, instead of only checking the fail
1500 // state of `handleDebugValue`, we need know specifically which values were
1501 // invalid, so that we attempt to salvage only those values when processing
1502 // a DIArgList.
1503 const Value *OrigV = V;
1504 DILocalVariable *Var = DDI.getVariable();
1505 DIExpression *Expr = DDI.getExpression();
1506 DebugLoc DL = DDI.getDebugLoc();
1507 unsigned SDOrder = DDI.getSDNodeOrder();
1508
1509 // Currently we consider only dbg.value intrinsics -- we tell the salvager
1510 // that DW_OP_stack_value is desired.
1511 bool StackValue = true;
1512
1513 // Can this Value can be encoded without any further work?
1514 if (handleDebugValue(V, Var, Expr, DL, SDOrder, /*IsVariadic=*/false))
1515 return;
1516
1517 // Attempt to salvage back through as many instructions as possible. Bail if
1518 // a non-instruction is seen, such as a constant expression or global
1519 // variable. FIXME: Further work could recover those too.
1520 while (isa<Instruction>(V)) {
1521 const Instruction &VAsInst = *cast<const Instruction>(V);
1522 // Temporary "0", awaiting real implementation.
1524 SmallVector<Value *, 4> AdditionalValues;
1525 V = salvageDebugInfoImpl(const_cast<Instruction &>(VAsInst),
1526 Expr->getNumLocationOperands(), Ops,
1527 AdditionalValues);
1528 // If we cannot salvage any further, and haven't yet found a suitable debug
1529 // expression, bail out.
1530 if (!V)
1531 break;
1532
1533 // TODO: If AdditionalValues isn't empty, then the salvage can only be
1534 // represented with a DBG_VALUE_LIST, so we give up. When we have support
1535 // here for variadic dbg_values, remove that condition.
1536 if (!AdditionalValues.empty())
1537 break;
1538
1539 // New value and expr now represent this debuginfo.
1540 Expr = DIExpression::appendOpsToArg(Expr, Ops, 0, StackValue);
1541
1542 // Some kind of simplification occurred: check whether the operand of the
1543 // salvaged debug expression can be encoded in this DAG.
1544 if (handleDebugValue(V, Var, Expr, DL, SDOrder, /*IsVariadic=*/false)) {
1545 LLVM_DEBUG(
1546 dbgs() << "Salvaged debug location info for:\n " << *Var << "\n"
1547 << *OrigV << "\nBy stripping back to:\n " << *V << "\n");
1548 return;
1549 }
1550 }
1551
1552 // This was the final opportunity to salvage this debug information, and it
1553 // couldn't be done. Place an undef DBG_VALUE at this location to terminate
1554 // any earlier variable location.
1555 assert(OrigV && "V shouldn't be null");
1556 auto *Undef = UndefValue::get(OrigV->getType());
1557 auto *SDV = DAG.getConstantDbgValue(Var, Expr, Undef, DL, SDNodeOrder);
1558 DAG.AddDbgValue(SDV, false);
1559 LLVM_DEBUG(dbgs() << "Dropping debug value info for:\n "
1560 << printDDI(OrigV, DDI) << "\n");
1561}
1562
1564 DIExpression *Expr,
1565 DebugLoc DbgLoc,
1566 unsigned Order) {
1570 handleDebugValue(Poison, Var, NewExpr, DbgLoc, Order,
1571 /*IsVariadic*/ false);
1572}
1573
1575 DILocalVariable *Var,
1576 DIExpression *Expr, DebugLoc DbgLoc,
1577 unsigned Order, bool IsVariadic) {
1578 if (Values.empty())
1579 return true;
1580
1581 // Filter EntryValue locations out early.
1582 if (visitEntryValueDbgValue(Values, Var, Expr, DbgLoc))
1583 return true;
1584
1585 SmallVector<SDDbgOperand> LocationOps;
1586 SmallVector<SDNode *> Dependencies;
1587 for (const Value *V : Values) {
1588 // Constant value.
1589 if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V) ||
1590 isa<ConstantPointerNull>(V)) {
1591 LocationOps.emplace_back(SDDbgOperand::fromConst(V));
1592 continue;
1593 }
1594
1595 // Look through IntToPtr constants.
1596 if (auto *CE = dyn_cast<ConstantExpr>(V))
1597 if (CE->getOpcode() == Instruction::IntToPtr) {
1598 LocationOps.emplace_back(SDDbgOperand::fromConst(CE->getOperand(0)));
1599 continue;
1600 }
1601
1602 // If the Value is a frame index, we can create a FrameIndex debug value
1603 // without relying on the DAG at all.
1604 if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1605 auto SI = FuncInfo.StaticAllocaMap.find(AI);
1606 if (SI != FuncInfo.StaticAllocaMap.end()) {
1607 LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(SI->second));
1608 continue;
1609 }
1610 }
1611
1612 // Do not use getValue() in here; we don't want to generate code at
1613 // this point if it hasn't been done yet.
1614 SDValue N = NodeMap[V];
1615 if (!N.getNode() && isa<Argument>(V)) // Check unused arguments map.
1616 N = UnusedArgNodeMap[V];
1617 if (N.getNode()) {
1618 // Only emit func arg dbg value for non-variadic dbg.values for now.
1619 if (!IsVariadic &&
1620 EmitFuncArgumentDbgValue(V, Var, Expr, DbgLoc,
1621 FuncArgumentDbgValueKind::Value, N))
1622 return true;
1623 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) {
1624 // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can
1625 // describe stack slot locations.
1626 //
1627 // Consider "int x = 0; int *px = &x;". There are two kinds of
1628 // interesting debug values here after optimization:
1629 //
1630 // dbg.value(i32* %px, !"int *px", !DIExpression()), and
1631 // dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))
1632 //
1633 // Both describe the direct values of their associated variables.
1634 Dependencies.push_back(N.getNode());
1635 LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(FISDN->getIndex()));
1636 continue;
1637 }
1638 LocationOps.emplace_back(
1639 SDDbgOperand::fromNode(N.getNode(), N.getResNo()));
1640 continue;
1641 }
1642
1644 // Special rules apply for the first dbg.values of parameter variables in a
1645 // function. Identify them by the fact they reference Argument Values, that
1646 // they're parameters, and they are parameters of the current function. We
1647 // need to let them dangle until they get an SDNode.
1648 bool IsParamOfFunc =
1649 isa<Argument>(V) && Var->isParameter() && !DbgLoc.getInlinedAt();
1650 if (IsParamOfFunc)
1651 return false;
1652
1653 // The value is not used in this block yet (or it would have an SDNode).
1654 // We still want the value to appear for the user if possible -- if it has
1655 // an associated VReg, we can refer to that instead.
1656 auto VMI = FuncInfo.ValueMap.find(V);
1657 if (VMI != FuncInfo.ValueMap.end()) {
1658 unsigned Reg = VMI->second;
1659 // If this is a PHI node, it may be split up into several MI PHI nodes
1660 // (in FunctionLoweringInfo::set).
1661 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg,
1662 V->getType(), std::nullopt);
1663 if (RFV.occupiesMultipleRegs()) {
1664 // FIXME: We could potentially support variadic dbg_values here.
1665 if (IsVariadic)
1666 return false;
1667 unsigned Offset = 0;
1668 unsigned BitsToDescribe = 0;
1669 if (auto VarSize = Var->getSizeInBits())
1670 BitsToDescribe = *VarSize;
1671 if (auto Fragment = Expr->getFragmentInfo())
1672 BitsToDescribe = Fragment->SizeInBits;
1673 for (const auto &RegAndSize : RFV.getRegsAndSizes()) {
1674 // Bail out if all bits are described already.
1675 if (Offset >= BitsToDescribe)
1676 break;
1677 // TODO: handle scalable vectors.
1678 unsigned RegisterSize = RegAndSize.second;
1679 unsigned FragmentSize = (Offset + RegisterSize > BitsToDescribe)
1680 ? BitsToDescribe - Offset
1681 : RegisterSize;
1682 auto FragmentExpr = DIExpression::createFragmentExpression(
1683 Expr, Offset, FragmentSize);
1684 if (!FragmentExpr)
1685 continue;
1687 Var, *FragmentExpr, RegAndSize.first, false, DbgLoc, SDNodeOrder);
1688 DAG.AddDbgValue(SDV, false);
1689 Offset += RegisterSize;
1690 }
1691 return true;
1692 }
1693 // We can use simple vreg locations for variadic dbg_values as well.
1694 LocationOps.emplace_back(SDDbgOperand::fromVReg(Reg));
1695 continue;
1696 }
1697 // We failed to create a SDDbgOperand for V.
1698 return false;
1699 }
1700
1701 // We have created a SDDbgOperand for each Value in Values.
1702 // Should use Order instead of SDNodeOrder?
1703 assert(!LocationOps.empty());
1704 SDDbgValue *SDV = DAG.getDbgValueList(Var, Expr, LocationOps, Dependencies,
1705 /*IsIndirect=*/false, DbgLoc,
1706 SDNodeOrder, IsVariadic);
1707 DAG.AddDbgValue(SDV, /*isParameter=*/false);
1708 return true;
1709}
1710
1712 // Try to fixup any remaining dangling debug info -- and drop it if we can't.
1713 for (auto &Pair : DanglingDebugInfoMap)
1714 for (auto &DDI : Pair.second)
1715 salvageUnresolvedDbgValue(const_cast<Value *>(Pair.first), DDI);
1717}
1718
1719/// getCopyFromRegs - If there was virtual register allocated for the value V
1720/// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.
1723 SDValue Result;
1724
1725 if (It != FuncInfo.ValueMap.end()) {
1726 Register InReg = It->second;
1727
1729 DAG.getDataLayout(), InReg, Ty,
1730 std::nullopt); // This is not an ABI copy.
1731 SDValue Chain = DAG.getEntryNode();
1732 Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr,
1733 V);
1734 resolveDanglingDebugInfo(V, Result);
1735 }
1736
1737 return Result;
1738}
1739
1740/// getValue - Return an SDValue for the given Value.
1742 // If we already have an SDValue for this value, use it. It's important
1743 // to do this first, so that we don't create a CopyFromReg if we already
1744 // have a regular SDValue.
1745 SDValue &N = NodeMap[V];
1746 if (N.getNode()) return N;
1747
1748 // If there's a virtual register allocated and initialized for this
1749 // value, use it.
1750 if (SDValue copyFromReg = getCopyFromRegs(V, V->getType()))
1751 return copyFromReg;
1752
1753 // Otherwise create a new SDValue and remember it.
1754 SDValue Val = getValueImpl(V);
1755 NodeMap[V] = Val;
1757 return Val;
1758}
1759
1760/// getNonRegisterValue - Return an SDValue for the given Value, but
1761/// don't look in FuncInfo.ValueMap for a virtual register.
1763 // If we already have an SDValue for this value, use it.
1764 SDValue &N = NodeMap[V];
1765 if (N.getNode()) {
1766 if (isIntOrFPConstant(N)) {
1767 // Remove the debug location from the node as the node is about to be used
1768 // in a location which may differ from the original debug location. This
1769 // is relevant to Constant and ConstantFP nodes because they can appear
1770 // as constant expressions inside PHI nodes.
1771 N->setDebugLoc(DebugLoc());
1772 }
1773 return N;
1774 }
1775
1776 // Otherwise create a new SDValue and remember it.
1777 SDValue Val = getValueImpl(V);
1778 NodeMap[V] = Val;
1780 return Val;
1781}
1782
1783/// getValueImpl - Helper function for getValue and getNonRegisterValue.
1784/// Create an SDValue for the given value.
1787
1788 if (const Constant *C = dyn_cast<Constant>(V)) {
1789 EVT VT = TLI.getValueType(DAG.getDataLayout(), V->getType(), true);
1790
1791 if (const ConstantInt *CI = dyn_cast<ConstantInt>(C))
1792 return DAG.getConstant(*CI, getCurSDLoc(), VT);
1793
1794 if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
1795 return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);
1796
1797 if (isa<ConstantPointerNull>(C)) {
1798 unsigned AS = V->getType()->getPointerAddressSpace();
1799 return DAG.getConstant(0, getCurSDLoc(),
1800 TLI.getPointerTy(DAG.getDataLayout(), AS));
1801 }
1802
1803 if (match(C, m_VScale()))
1804 return DAG.getVScale(getCurSDLoc(), VT, APInt(VT.getSizeInBits(), 1));
1805
1806 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
1807 return DAG.getConstantFP(*CFP, getCurSDLoc(), VT);
1808
1809 if (isa<UndefValue>(C) && !V->getType()->isAggregateType())
1810 return DAG.getUNDEF(VT);
1811
1812 if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
1813 visit(CE->getOpcode(), *CE);
1814 SDValue N1 = NodeMap[V];
1815 assert(N1.getNode() && "visit didn't populate the NodeMap!");
1816 return N1;
1817 }
1818
1819 if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) {
1821 for (const Use &U : C->operands()) {
1822 SDNode *Val = getValue(U).getNode();
1823 // If the operand is an empty aggregate, there are no values.
1824 if (!Val) continue;
1825 // Add each leaf value from the operand to the Constants list
1826 // to form a flattened list of all the values.
1827 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1828 Constants.push_back(SDValue(Val, i));
1829 }
1830
1832 }
1833
1834 if (const ConstantDataSequential *CDS =
1835 dyn_cast<ConstantDataSequential>(C)) {
1837 for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
1838 SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();
1839 // Add each leaf value from the operand to the Constants list
1840 // to form a flattened list of all the values.
1841 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1842 Ops.push_back(SDValue(Val, i));
1843 }
1844
1845 if (isa<ArrayType>(CDS->getType()))
1846 return DAG.getMergeValues(Ops, getCurSDLoc());
1847 return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1848 }
1849
1850 if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
1851 assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&
1852 "Unknown struct or array constant!");
1853
1854 SmallVector<EVT, 4> ValueVTs;
1855 ComputeValueVTs(TLI, DAG.getDataLayout(), C->getType(), ValueVTs);
1856 unsigned NumElts = ValueVTs.size();
1857 if (NumElts == 0)
1858 return SDValue(); // empty struct
1860 for (unsigned i = 0; i != NumElts; ++i) {
1861 EVT EltVT = ValueVTs[i];
1862 if (isa<UndefValue>(C))
1863 Constants[i] = DAG.getUNDEF(EltVT);
1864 else if (EltVT.isFloatingPoint())
1865 Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1866 else
1867 Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT);
1868 }
1869
1871 }
1872
1873 if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
1874 return DAG.getBlockAddress(BA, VT);
1875
1876 if (const auto *Equiv = dyn_cast<DSOLocalEquivalent>(C))
1877 return getValue(Equiv->getGlobalValue());
1878
1879 if (const auto *NC = dyn_cast<NoCFIValue>(C))
1880 return getValue(NC->getGlobalValue());
1881
1882 if (VT == MVT::aarch64svcount) {
1883 assert(C->isNullValue() && "Can only zero this target type!");
1884 return DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT,
1885 DAG.getConstant(0, getCurSDLoc(), MVT::nxv16i1));
1886 }
1887
1888 VectorType *VecTy = cast<VectorType>(V->getType());
1889
1890 // Now that we know the number and type of the elements, get that number of
1891 // elements into the Ops array based on what kind of constant it is.
1892 if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {
1894 unsigned NumElements = cast<FixedVectorType>(VecTy)->getNumElements();
1895 for (unsigned i = 0; i != NumElements; ++i)
1896 Ops.push_back(getValue(CV->getOperand(i)));
1897
1898 return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1899 }
1900
1901 if (isa<ConstantAggregateZero>(C)) {
1902 EVT EltVT =
1904
1905 SDValue Op;
1906 if (EltVT.isFloatingPoint())
1907 Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1908 else
1909 Op = DAG.getConstant(0, getCurSDLoc(), EltVT);
1910
1911 return NodeMap[V] = DAG.getSplat(VT, getCurSDLoc(), Op);
1912 }
1913
1914 llvm_unreachable("Unknown vector constant");
1915 }
1916
1917 // If this is a static alloca, generate it as the frameindex instead of
1918 // computation.
1919 if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1921 FuncInfo.StaticAllocaMap.find(AI);
1922 if (SI != FuncInfo.StaticAllocaMap.end())
1923 return DAG.getFrameIndex(
1924 SI->second, TLI.getValueType(DAG.getDataLayout(), AI->getType()));
1925 }
1926
1927 // If this is an instruction which fast-isel has deferred, select it now.
1928 if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
1930
1931 RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg,
1932 Inst->getType(), std::nullopt);
1933 SDValue Chain = DAG.getEntryNode();
1934 return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
1935 }
1936
1937 if (const MetadataAsValue *MD = dyn_cast<MetadataAsValue>(V))
1938 return DAG.getMDNode(cast<MDNode>(MD->getMetadata()));
1939
1940 if (const auto *BB = dyn_cast<BasicBlock>(V))
1941 return DAG.getBasicBlock(FuncInfo.MBBMap[BB]);
1942
1943 llvm_unreachable("Can't get register for value!");
1944}
1945
1946void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) {
1948 bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX;
1949 bool IsCoreCLR = Pers == EHPersonality::CoreCLR;
1950 bool IsSEH = isAsynchronousEHPersonality(Pers);
1951 MachineBasicBlock *CatchPadMBB = FuncInfo.MBB;
1952 if (!IsSEH)
1953 CatchPadMBB->setIsEHScopeEntry();
1954 // In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues.
1955 if (IsMSVCCXX || IsCoreCLR)
1956 CatchPadMBB->setIsEHFuncletEntry();
1957}
1958
1959void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) {
1960 // Update machine-CFG edge.
1961 MachineBasicBlock *TargetMBB = FuncInfo.MBBMap[I.getSuccessor()];
1962 FuncInfo.MBB->addSuccessor(TargetMBB);
1963 TargetMBB->setIsEHCatchretTarget(true);
1965
1967 bool IsSEH = isAsynchronousEHPersonality(Pers);
1968 if (IsSEH) {
1969 // If this is not a fall-through branch or optimizations are switched off,
1970 // emit the branch.
1971 if (TargetMBB != NextBlock(FuncInfo.MBB) ||
1973 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
1974 getControlRoot(), DAG.getBasicBlock(TargetMBB)));
1975 return;
1976 }
1977
1978 // Figure out the funclet membership for the catchret's successor.
1979 // This will be used by the FuncletLayout pass to determine how to order the
1980 // BB's.
1981 // A 'catchret' returns to the outer scope's color.
1982 Value *ParentPad = I.getCatchSwitchParentPad();
1983 const BasicBlock *SuccessorColor;
1984 if (isa<ConstantTokenNone>(ParentPad))
1985 SuccessorColor = &FuncInfo.Fn->getEntryBlock();
1986 else
1987 SuccessorColor = cast<Instruction>(ParentPad)->getParent();
1988 assert(SuccessorColor && "No parent funclet for catchret!");
1989 MachineBasicBlock *SuccessorColorMBB = FuncInfo.MBBMap[SuccessorColor];
1990 assert(SuccessorColorMBB && "No MBB for SuccessorColor!");
1991
1992 // Create the terminator node.
1994 getControlRoot(), DAG.getBasicBlock(TargetMBB),
1995 DAG.getBasicBlock(SuccessorColorMBB));
1996 DAG.setRoot(Ret);
1997}
1998
1999void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {
2000 // Don't emit any special code for the cleanuppad instruction. It just marks
2001 // the start of an EH scope/funclet.
2004 if (Pers != EHPersonality::Wasm_CXX) {
2007 }
2008}
2009
2010// In wasm EH, even though a catchpad may not catch an exception if a tag does
2011// not match, it is OK to add only the first unwind destination catchpad to the
2012// successors, because there will be at least one invoke instruction within the
2013// catch scope that points to the next unwind destination, if one exists, so
2014// CFGSort cannot mess up with BB sorting order.
2015// (All catchpads with 'catch (type)' clauses have a 'llvm.rethrow' intrinsic
2016// call within them, and catchpads only consisting of 'catch (...)' have a
2017// '__cxa_end_catch' call within them, both of which generate invokes in case
2018// the next unwind destination exists, i.e., the next unwind destination is not
2019// the caller.)
2020//
2021// Having at most one EH pad successor is also simpler and helps later
2022// transformations.
2023//
2024// For example,
2025// current:
2026// invoke void @foo to ... unwind label %catch.dispatch
2027// catch.dispatch:
2028// %0 = catchswitch within ... [label %catch.start] unwind label %next
2029// catch.start:
2030// ...
2031// ... in this BB or some other child BB dominated by this BB there will be an
2032// invoke that points to 'next' BB as an unwind destination
2033//
2034// next: ; We don't need to add this to 'current' BB's successor
2035// ...
2037 FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
2038 BranchProbability Prob,
2039 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
2040 &UnwindDests) {
2041 while (EHPadBB) {
2042 const Instruction *Pad = EHPadBB->getFirstNonPHI();
2043 if (isa<CleanupPadInst>(Pad)) {
2044 // Stop on cleanup pads.
2045 UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
2046 UnwindDests.back().first->setIsEHScopeEntry();
2047 break;
2048 } else if (const auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
2049 // Add the catchpad handlers to the possible destinations. We don't
2050 // continue to the unwind destination of the catchswitch for wasm.
2051 for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
2052 UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);
2053 UnwindDests.back().first->setIsEHScopeEntry();
2054 }
2055 break;
2056 } else {
2057 continue;
2058 }
2059 }
2060}
2061
2062/// When an invoke or a cleanupret unwinds to the next EH pad, there are
2063/// many places it could ultimately go. In the IR, we have a single unwind
2064/// destination, but in the machine CFG, we enumerate all the possible blocks.
2065/// This function skips over imaginary basic blocks that hold catchswitch
2066/// instructions, and finds all the "real" machine
2067/// basic block destinations. As those destinations may not be successors of
2068/// EHPadBB, here we also calculate the edge probability to those destinations.
2069/// The passed-in Prob is the edge probability to EHPadBB.
2071 FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
2072 BranchProbability Prob,
2073 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
2074 &UnwindDests) {
2075 EHPersonality Personality =
2077 bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
2078 bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
2079 bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
2080 bool IsSEH = isAsynchronousEHPersonality(Personality);
2081
2082 if (IsWasmCXX) {
2083 findWasmUnwindDestinations(FuncInfo, EHPadBB, Prob, UnwindDests);
2084 assert(UnwindDests.size() <= 1 &&
2085 "There should be at most one unwind destination for wasm");
2086 return;
2087 }
2088
2089 while (EHPadBB) {
2090 const Instruction *Pad = EHPadBB->getFirstNonPHI();
2091 BasicBlock *NewEHPadBB = nullptr;
2092 if (isa<LandingPadInst>(Pad)) {
2093 // Stop on landingpads. They are not funclets.
2094 UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
2095 break;
2096 } else if (isa<CleanupPadInst>(Pad)) {
2097 // Stop on cleanup pads. Cleanups are always funclet entries for all known
2098 // personalities.
2099 UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
2100 UnwindDests.back().first->setIsEHScopeEntry();
2101 UnwindDests.back().first->setIsEHFuncletEntry();
2102 break;
2103 } else if (const auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
2104 // Add the catchpad handlers to the possible destinations.
2105 for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
2106 UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);
2107 // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
2108 if (IsMSVCCXX || IsCoreCLR)
2109 UnwindDests.back().first->setIsEHFuncletEntry();
2110 if (!IsSEH)
2111 UnwindDests.back().first->setIsEHScopeEntry();
2112 }
2113 NewEHPadBB = CatchSwitch->getUnwindDest();
2114 } else {
2115 continue;
2116 }
2117
2118 BranchProbabilityInfo *BPI = FuncInfo.BPI;
2119 if (BPI && NewEHPadBB)
2120 Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);
2121 EHPadBB = NewEHPadBB;
2122 }
2123}
2124
2125void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) {
2126 // Update successor info.
2128 auto UnwindDest = I.getUnwindDest();
2130 BranchProbability UnwindDestProb =
2131 (BPI && UnwindDest)
2132 ? BPI->getEdgeProbability(FuncInfo.MBB->getBasicBlock(), UnwindDest)
2134 findUnwindDestinations(FuncInfo, UnwindDest, UnwindDestProb, UnwindDests);
2135 for (auto &UnwindDest : UnwindDests) {
2136 UnwindDest.first->setIsEHPad();
2137 addSuccessorWithProb(FuncInfo.MBB, UnwindDest.first, UnwindDest.second);
2138 }
2140
2141 // Create the terminator node.
2142 SDValue Ret =
2144 DAG.setRoot(Ret);
2145}
2146
2147void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) {
2148 report_fatal_error("visitCatchSwitch not yet implemented!");
2149}
2150
2151void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
2153 auto &DL = DAG.getDataLayout();
2154 SDValue Chain = getControlRoot();
2157
2158 // Calls to @llvm.experimental.deoptimize don't generate a return value, so
2159 // lower
2160 //
2161 // %val = call <ty> @llvm.experimental.deoptimize()
2162 // ret <ty> %val
2163 //
2164 // differently.
2165 if (I.getParent()->getTerminatingDeoptimizeCall()) {
2167 return;
2168 }
2169
2170 if (!FuncInfo.CanLowerReturn) {
2171 unsigned DemoteReg = FuncInfo.DemoteRegister;
2172 const Function *F = I.getParent()->getParent();
2173
2174 // Emit a store of the return value through the virtual register.
2175 // Leave Outs empty so that LowerReturn won't try to load return
2176 // registers the usual way.
2177 SmallVector<EVT, 1> PtrValueVTs;
2178 ComputeValueVTs(TLI, DL,
2179 PointerType::get(F->getContext(),
2181 PtrValueVTs);
2182
2183 SDValue RetPtr =
2184 DAG.getCopyFromReg(Chain, getCurSDLoc(), DemoteReg, PtrValueVTs[0]);
2185 SDValue RetOp = getValue(I.getOperand(0));
2186
2187 SmallVector<EVT, 4> ValueVTs, MemVTs;
2189 ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &MemVTs,
2190 &Offsets, 0);
2191 unsigned NumValues = ValueVTs.size();
2192
2193 SmallVector<SDValue, 4> Chains(NumValues);
2194 Align BaseAlign = DL.getPrefTypeAlign(I.getOperand(0)->getType());
2195 for (unsigned i = 0; i != NumValues; ++i) {
2196 // An aggregate return value cannot wrap around the address space, so
2197 // offsets to its parts don't wrap either.
2199 TypeSize::getFixed(Offsets[i]));
2200
2201 SDValue Val = RetOp.getValue(RetOp.getResNo() + i);
2202 if (MemVTs[i] != ValueVTs[i])
2203 Val = DAG.getPtrExtOrTrunc(Val, getCurSDLoc(), MemVTs[i]);
2204 Chains[i] = DAG.getStore(
2205 Chain, getCurSDLoc(), Val,
2206 // FIXME: better loc info would be nice.
2208 commonAlignment(BaseAlign, Offsets[i]));
2209 }
2210
2212 MVT::Other, Chains);
2213 } else if (I.getNumOperands() != 0) {
2214 SmallVector<EVT, 4> ValueVTs;
2215 ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs);
2216 unsigned NumValues = ValueVTs.size();
2217 if (NumValues) {
2218 SDValue RetOp = getValue(I.getOperand(0));
2219
2220 const Function *F = I.getParent()->getParent();
2221
2222 bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters(
2223 I.getOperand(0)->getType(), F->getCallingConv(),
2224 /*IsVarArg*/ false, DL);
2225
2226 ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
2227 if (F->getAttributes().hasRetAttr(Attribute::SExt))
2228 ExtendKind = ISD::SIGN_EXTEND;
2229 else if (F->getAttributes().hasRetAttr(Attribute::ZExt))
2230 ExtendKind = ISD::ZERO_EXTEND;
2231
2232 LLVMContext &Context = F->getContext();
2233 bool RetInReg = F->getAttributes().hasRetAttr(Attribute::InReg);
2234
2235 for (unsigned j = 0; j != NumValues; ++j) {
2236 EVT VT = ValueVTs[j];
2237
2238 if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
2239 VT = TLI.getTypeForExtReturn(Context, VT, ExtendKind);
2240
2241 CallingConv::ID CC = F->getCallingConv();
2242
2243 unsigned NumParts = TLI.getNumRegistersForCallingConv(Context, CC, VT);
2244 MVT PartVT = TLI.getRegisterTypeForCallingConv(Context, CC, VT);
2245 SmallVector<SDValue, 4> Parts(NumParts);
2247 SDValue(RetOp.getNode(), RetOp.getResNo() + j),
2248 &Parts[0], NumParts, PartVT, &I, CC, ExtendKind);
2249
2250 // 'inreg' on function refers to return value
2252 if (RetInReg)
2253 Flags.setInReg();
2254
2255 if (I.getOperand(0)->getType()->isPointerTy()) {
2256 Flags.setPointer();
2257 Flags.setPointerAddrSpace(
2258 cast<PointerType>(I.getOperand(0)->getType())->getAddressSpace());
2259 }
2260
2261 if (NeedsRegBlock) {
2262 Flags.setInConsecutiveRegs();
2263 if (j == NumValues - 1)
2264 Flags.setInConsecutiveRegsLast();
2265 }
2266
2267 // Propagate extension type if any
2268 if (ExtendKind == ISD::SIGN_EXTEND)
2269 Flags.setSExt();
2270 else if (ExtendKind == ISD::ZERO_EXTEND)
2271 Flags.setZExt();
2272
2273 for (unsigned i = 0; i < NumParts; ++i) {
2274 Outs.push_back(ISD::OutputArg(Flags,
2275 Parts[i].getValueType().getSimpleVT(),
2276 VT, /*isfixed=*/true, 0, 0));
2277 OutVals.push_back(Parts[i]);
2278 }
2279 }
2280 }
2281 }
2282
2283 // Push in swifterror virtual register as the last element of Outs. This makes
2284 // sure swifterror virtual register will be returned in the swifterror
2285 // physical register.
2286 const Function *F = I.getParent()->getParent();
2287 if (TLI.supportSwiftError() &&
2288 F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) {
2289 assert(SwiftError.getFunctionArg() && "Need a swift error argument");
2291 Flags.setSwiftError();
2293 Flags, /*vt=*/TLI.getPointerTy(DL), /*argvt=*/EVT(TLI.getPointerTy(DL)),
2294 /*isfixed=*/true, /*origidx=*/1, /*partOffs=*/0));
2295 // Create SDNode for the swifterror virtual register.
2296 OutVals.push_back(
2299 EVT(TLI.getPointerTy(DL))));
2300 }
2301
2302 bool isVarArg = DAG.getMachineFunction().getFunction().isVarArg();
2303 CallingConv::ID CallConv =
2306 Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);
2307
2308 // Verify that the target's LowerReturn behaved as expected.
2309 assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&
2310 "LowerReturn didn't return a valid chain!");
2311
2312 // Update the DAG with the new chain value resulting from return lowering.
2313 DAG.setRoot(Chain);
2314}
2315
2316/// CopyToExportRegsIfNeeded - If the given value has virtual registers
2317/// created for it, emit nodes to copy the value into the virtual
2318/// registers.
2320 // Skip empty types
2321 if (V->getType()->isEmptyTy())
2322 return;
2323
2325 if (VMI != FuncInfo.ValueMap.end()) {
2326 assert((!V->use_empty() || isa<CallBrInst>(V)) &&
2327 "Unused value assigned virtual registers!");
2328 CopyValueToVirtualRegister(V, VMI->second);
2329 }
2330}
2331
2332/// ExportFromCurrentBlock - If this condition isn't known to be exported from
2333/// the current basic block, add it to ValueMap now so that we'll get a
2334/// CopyTo/FromReg.
2336 // No need to export constants.
2337 if (!isa<Instruction>(V) && !isa<Argument>(V)) return;
2338
2339 // Already exported?
2340 if (FuncInfo.isExportedInst(V)) return;
2341
2344}
2345
2347 const BasicBlock *FromBB) {
2348 // The operands of the setcc have to be in this block. We don't know
2349 // how to export them from some other block.
2350 if (const Instruction *VI = dyn_cast<Instruction>(V)) {
2351 // Can export from current BB.
2352 if (VI->getParent() == FromBB)
2353 return true;
2354
2355 // Is already exported, noop.
2356 return FuncInfo.isExportedInst(V);
2357 }
2358
2359 // If this is an argument, we can export it if the BB is the entry block or
2360 // if it is already exported.
2361 if (isa<Argument>(V)) {
2362 if (FromBB->isEntryBlock())
2363 return true;
2364
2365 // Otherwise, can only export this if it is already exported.
2366 return FuncInfo.isExportedInst(V);
2367 }
2368
2369 // Otherwise, constants can always be exported.
2370 return true;
2371}
2372
2373/// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
2375SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src,
2376 const MachineBasicBlock *Dst) const {
2378 const BasicBlock *SrcBB = Src->getBasicBlock();
2379 const BasicBlock *DstBB = Dst->getBasicBlock();
2380 if (!BPI) {
2381 // If BPI is not available, set the default probability as 1 / N, where N is
2382 // the number of successors.
2383 auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1);
2384 return BranchProbability(1, SuccSize);
2385 }
2386 return BPI->getEdgeProbability(SrcBB, DstBB);
2387}
2388
2389void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src,
2390 MachineBasicBlock *Dst,
2391 BranchProbability Prob) {
2392 if (!FuncInfo.BPI)
2393 Src->addSuccessorWithoutProb(Dst);
2394 else {
2395 if (Prob.isUnknown())
2396 Prob = getEdgeProbability(Src, Dst);
2397 Src->addSuccessor(Dst, Prob);
2398 }
2399}
2400
2401static bool InBlock(const Value *V, const BasicBlock *BB) {
2402 if (const Instruction *I = dyn_cast<Instruction>(V))
2403 return I->getParent() == BB;
2404 return true;
2405}
2406
2407/// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
2408/// This function emits a branch and is used at the leaves of an OR or an
2409/// AND operator tree.
2410void
2413 MachineBasicBlock *FBB,
2414 MachineBasicBlock *CurBB,
2415 MachineBasicBlock *SwitchBB,
2416 BranchProbability TProb,
2417 BranchProbability FProb,
2418 bool InvertCond) {
2419 const BasicBlock *BB = CurBB->getBasicBlock();
2420
2421 // If the leaf of the tree is a comparison, merge the condition into
2422 // the caseblock.
2423 if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
2424 // The operands of the cmp have to be in this block. We don't know
2425 // how to export them from some other block. If this is the first block
2426 // of the sequence, no exporting is needed.
2427 if (CurBB == SwitchBB ||
2428 (isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&
2429 isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {
2430 ISD::CondCode Condition;
2431 if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
2432 ICmpInst::Predicate Pred =
2433 InvertCond ? IC->getInversePredicate() : IC->getPredicate();
2434 Condition = getICmpCondCode(Pred);
2435 } else {
2436 const FCmpInst *FC = cast<FCmpInst>(Cond);
2437 FCmpInst::Predicate Pred =
2438 InvertCond ? FC->getInversePredicate() : FC->getPredicate();
2439 Condition = getFCmpCondCode(Pred);
2440 if (TM.Options.NoNaNsFPMath)
2441 Condition = getFCmpCodeWithoutNaN(Condition);
2442 }
2443
2444 CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,
2445 TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2446 SL->SwitchCases.push_back(CB);
2447 return;
2448 }
2449 }
2450
2451 // Create a CaseBlock record representing this branch.
2452 ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ;
2454 nullptr, TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2455 SL->SwitchCases.push_back(CB);
2456}
2457
2458// Collect dependencies on V recursively. This is used for the cost analysis in
2459// `shouldKeepJumpConditionsTogether`.
2463 unsigned Depth = 0) {
2464 // Return false if we have an incomplete count.
2466 return false;
2467
2468 auto *I = dyn_cast<Instruction>(V);
2469 if (I == nullptr)
2470 return true;
2471
2472 if (Necessary != nullptr) {
2473 // This instruction is necessary for the other side of the condition so
2474 // don't count it.
2475 if (Necessary->contains(I))
2476 return true;
2477 }
2478
2479 // Already added this dep.
2480 if (!Deps->try_emplace(I, false).second)
2481 return true;
2482
2483 for (unsigned OpIdx = 0, E = I->getNumOperands(); OpIdx < E; ++OpIdx)
2484 if (!collectInstructionDeps(Deps, I->getOperand(OpIdx), Necessary,
2485 Depth + 1))
2486 return false;
2487 return true;
2488}
2489
2491 const FunctionLoweringInfo &FuncInfo, const BranchInst &I,
2492 Instruction::BinaryOps Opc, const Value *Lhs, const Value *Rhs,
2494 if (I.getNumSuccessors() != 2)
2495 return false;
2496
2497 if (!I.isConditional())
2498 return false;
2499
2500 if (Params.BaseCost < 0)
2501 return false;
2502
2503 // Baseline cost.
2504 InstructionCost CostThresh = Params.BaseCost;
2505
2506 BranchProbabilityInfo *BPI = nullptr;
2507 if (Params.LikelyBias || Params.UnlikelyBias)
2508 BPI = FuncInfo.BPI;
2509 if (BPI != nullptr) {
2510 // See if we are either likely to get an early out or compute both lhs/rhs
2511 // of the condition.
2512 BasicBlock *IfFalse = I.getSuccessor(0);
2513 BasicBlock *IfTrue = I.getSuccessor(1);
2514
2515 std::optional<bool> Likely;
2516 if (BPI->isEdgeHot(I.getParent(), IfTrue))
2517 Likely = true;
2518 else if (BPI->isEdgeHot(I.getParent(), IfFalse))
2519 Likely = false;
2520
2521 if (Likely) {
2522 if (Opc == (*Likely ? Instruction::And : Instruction::Or))
2523 // Its likely we will have to compute both lhs and rhs of condition
2524 CostThresh += Params.LikelyBias;
2525 else {
2526 if (Params.UnlikelyBias < 0)
2527 return false;
2528 // Its likely we will get an early out.
2529 CostThresh -= Params.UnlikelyBias;
2530 }
2531 }
2532 }
2533
2534 if (CostThresh <= 0)
2535 return false;
2536
2537 // Collect "all" instructions that lhs condition is dependent on.
2538 // Use map for stable iteration (to avoid non-determanism of iteration of
2539 // SmallPtrSet). The `bool` value is just a dummy.
2541 collectInstructionDeps(&LhsDeps, Lhs);
2542 // Collect "all" instructions that rhs condition is dependent on AND are
2543 // dependencies of lhs. This gives us an estimate on which instructions we
2544 // stand to save by splitting the condition.
2545 if (!collectInstructionDeps(&RhsDeps, Rhs, &LhsDeps))
2546 return false;
2547 // Add the compare instruction itself unless its a dependency on the LHS.
2548 if (const auto *RhsI = dyn_cast<Instruction>(Rhs))
2549 if (!LhsDeps.contains(RhsI))
2550 RhsDeps.try_emplace(RhsI, false);
2551
2552 const auto &TLI = DAG.getTargetLoweringInfo();
2553 const auto &TTI =
2554 TLI.getTargetMachine().getTargetTransformInfo(*I.getFunction());
2555
2556 InstructionCost CostOfIncluding = 0;
2557 // See if this instruction will need to computed independently of whether RHS
2558 // is.
2559 Value *BrCond = I.getCondition();
2560 auto ShouldCountInsn = [&RhsDeps, &BrCond](const Instruction *Ins) {
2561 for (const auto *U : Ins->users()) {
2562 // If user is independent of RHS calculation we don't need to count it.
2563 if (auto *UIns = dyn_cast<Instruction>(U))
2564 if (UIns != BrCond && !RhsDeps.contains(UIns))
2565 return false;
2566 }
2567 return true;
2568 };
2569
2570 // Prune instructions from RHS Deps that are dependencies of unrelated
2571 // instructions. The value (SelectionDAG::MaxRecursionDepth) is fairly
2572 // arbitrary and just meant to cap the how much time we spend in the pruning
2573 // loop. Its highly unlikely to come into affect.
2574 const unsigned MaxPruneIters = SelectionDAG::MaxRecursionDepth;
2575 // Stop after a certain point. No incorrectness from including too many
2576 // instructions.
2577 for (unsigned PruneIters = 0; PruneIters < MaxPruneIters; ++PruneIters) {
2578 const Instruction *ToDrop = nullptr;
2579 for (const auto &InsPair : RhsDeps) {
2580 if (!ShouldCountInsn(InsPair.first)) {
2581 ToDrop = InsPair.first;
2582 break;
2583 }
2584 }
2585 if (ToDrop == nullptr)
2586 break;
2587 RhsDeps.erase(ToDrop);
2588 }
2589
2590 for (const auto &InsPair : RhsDeps) {
2591 // Finally accumulate latency that we can only attribute to computing the
2592 // RHS condition. Use latency because we are essentially trying to calculate
2593 // the cost of the dependency chain.
2594 // Possible TODO: We could try to estimate ILP and make this more precise.
2595 CostOfIncluding +=
2597
2598 if (CostOfIncluding > CostThresh)
2599 return false;
2600 }
2601 return true;
2602}
2603
2606 MachineBasicBlock *FBB,
2607 MachineBasicBlock *CurBB,
2608 MachineBasicBlock *SwitchBB,
2610 BranchProbability TProb,
2611 BranchProbability FProb,
2612 bool InvertCond) {
2613 // Skip over not part of the tree and remember to invert op and operands at
2614 // next level.
2615 Value *NotCond;
2616 if (match(Cond, m_OneUse(m_Not(m_Value(NotCond)))) &&
2617 InBlock(NotCond, CurBB->getBasicBlock())) {
2618 FindMergedConditions(NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
2619 !InvertCond);
2620 return;
2621 }
2622
2623 const Instruction *BOp = dyn_cast<Instruction>(Cond);
2624 const Value *BOpOp0, *BOpOp1;
2625 // Compute the effective opcode for Cond, taking into account whether it needs
2626 // to be inverted, e.g.
2627 // and (not (or A, B)), C
2628 // gets lowered as
2629 // and (and (not A, not B), C)
2631 if (BOp) {
2632 BOpc = match(BOp, m_LogicalAnd(m_Value(BOpOp0), m_Value(BOpOp1)))
2633 ? Instruction::And
2634 : (match(BOp, m_LogicalOr(m_Value(BOpOp0), m_Value(BOpOp1)))
2635 ? Instruction::Or
2637 if (InvertCond) {
2638 if (BOpc == Instruction::And)
2639 BOpc = Instruction::Or;
2640 else if (BOpc == Instruction::Or)
2641 BOpc = Instruction::And;
2642 }
2643 }
2644
2645 // If this node is not part of the or/and tree, emit it as a branch.
2646 // Note that all nodes in the tree should have same opcode.
2647 bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->hasOneUse();
2648 if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() ||
2649 !InBlock(BOpOp0, CurBB->getBasicBlock()) ||
2650 !InBlock(BOpOp1, CurBB->getBasicBlock())) {
2651 EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
2652 TProb, FProb, InvertCond);
2653 return;
2654 }
2655
2656 // Create TmpBB after CurBB.
2657 MachineFunction::iterator BBI(CurBB);
2660 CurBB->getParent()->insert(++BBI, TmpBB);
2661
2662 if (Opc == Instruction::Or) {
2663 // Codegen X | Y as:
2664 // BB1:
2665 // jmp_if_X TBB
2666 // jmp TmpBB
2667 // TmpBB:
2668 // jmp_if_Y TBB
2669 // jmp FBB
2670 //
2671
2672 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2673 // The requirement is that
2674 // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
2675 // = TrueProb for original BB.
2676 // Assuming the original probabilities are A and B, one choice is to set
2677 // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
2678 // A/(1+B) and 2B/(1+B). This choice assumes that
2679 // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
2680 // Another choice is to assume TrueProb for BB1 equals to TrueProb for
2681 // TmpBB, but the math is more complicated.
2682
2683 auto NewTrueProb = TProb / 2;
2684 auto NewFalseProb = TProb / 2 + FProb;
2685 // Emit the LHS condition.
2686 FindMergedConditions(BOpOp0, TBB, TmpBB, CurBB, SwitchBB, Opc, NewTrueProb,
2687 NewFalseProb, InvertCond);
2688
2689 // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
2690 SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
2691 BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
2692 // Emit the RHS condition into TmpBB.
2693 FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
2694 Probs[1], InvertCond);
2695 } else {
2696 assert(Opc == Instruction::And && "Unknown merge op!");
2697 // Codegen X & Y as:
2698 // BB1:
2699 // jmp_if_X TmpBB
2700 // jmp FBB
2701 // TmpBB:
2702 // jmp_if_Y TBB
2703 // jmp FBB
2704 //
2705 // This requires creation of TmpBB after CurBB.
2706
2707 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2708 // The requirement is that
2709 // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
2710 // = FalseProb for original BB.
2711 // Assuming the original probabilities are A and B, one choice is to set
2712 // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
2713 // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
2714 // TrueProb for BB1 * FalseProb for TmpBB.
2715
2716 auto NewTrueProb = TProb + FProb / 2;
2717 auto NewFalseProb = FProb / 2;
2718 // Emit the LHS condition.
2719 FindMergedConditions(BOpOp0, TmpBB, FBB, CurBB, SwitchBB, Opc, NewTrueProb,
2720 NewFalseProb, InvertCond);
2721
2722 // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
2723 SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
2724 BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
2725 // Emit the RHS condition into TmpBB.
2726 FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
2727 Probs[1], InvertCond);
2728 }
2729}
2730
2731/// If the set of cases should be emitted as a series of branches, return true.
2732/// If we should emit this as a bunch of and/or'd together conditions, return
2733/// false.
2734bool
2735SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
2736 if (Cases.size() != 2) return true;
2737
2738 // If this is two comparisons of the same values or'd or and'd together, they
2739 // will get folded into a single comparison, so don't emit two blocks.
2740 if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
2741 Cases[0].CmpRHS == Cases[1].CmpRHS) ||
2742 (Cases[0].CmpRHS == Cases[1].CmpLHS &&
2743 Cases[0].CmpLHS == Cases[1].CmpRHS)) {
2744 return false;
2745 }
2746
2747 // Handle: (X != null) | (Y != null) --> (X|Y) != 0
2748 // Handle: (X == null) & (Y == null) --> (X|Y) == 0
2749 if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
2750 Cases[0].CC == Cases[1].CC &&
2751 isa<Constant>(Cases[0].CmpRHS) &&
2752 cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
2753 if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
2754 return false;
2755 if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
2756 return false;
2757 }
2758
2759 return true;
2760}
2761
2762void SelectionDAGBuilder::visitBr(const BranchInst &I) {
2764
2765 // Update machine-CFG edges.
2766 MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)];
2767
2768 if (I.isUnconditional()) {
2769 // Update machine-CFG edges.
2770 BrMBB->addSuccessor(Succ0MBB);
2771
2772 // If this is not a fall-through branch or optimizations are switched off,
2773 // emit the branch.
2774 if (Succ0MBB != NextBlock(BrMBB) ||
2776 auto Br = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
2777 getControlRoot(), DAG.getBasicBlock(Succ0MBB));
2778 setValue(&I, Br);
2779 DAG.setRoot(Br);
2780 }
2781
2782 return;
2783 }
2784
2785 // If this condition is one of the special cases we handle, do special stuff
2786 // now.
2787 const Value *CondVal = I.getCondition();
2788 MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)];
2789
2790 // If this is a series of conditions that are or'd or and'd together, emit
2791 // this as a sequence of branches instead of setcc's with and/or operations.
2792 // As long as jumps are not expensive (exceptions for multi-use logic ops,
2793 // unpredictable branches, and vector extracts because those jumps are likely
2794 // expensive for any target), this should improve performance.
2795 // For example, instead of something like:
2796 // cmp A, B
2797 // C = seteq
2798 // cmp D, E
2799 // F = setle
2800 // or C, F
2801 // jnz foo
2802 // Emit:
2803 // cmp A, B
2804 // je foo
2805 // cmp D, E
2806 // jle foo
2807 const Instruction *BOp = dyn_cast<Instruction>(CondVal);
2808 if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp &&
2809 BOp->hasOneUse() && !I.hasMetadata(LLVMContext::MD_unpredictable)) {
2810 Value *Vec;
2811 const Value *BOp0, *BOp1;
2813 if (match(BOp, m_LogicalAnd(m_Value(BOp0), m_Value(BOp1))))
2814 Opcode = Instruction::And;
2815 else if (match(BOp, m_LogicalOr(m_Value(BOp0), m_Value(BOp1))))
2816 Opcode = Instruction::Or;
2817
2818 if (Opcode &&
2819 !(match(BOp0, m_ExtractElt(m_Value(Vec), m_Value())) &&
2820 match(BOp1, m_ExtractElt(m_Specific(Vec), m_Value()))) &&
2822 FuncInfo, I, Opcode, BOp0, BOp1,
2824 Opcode, BOp0, BOp1))) {
2825 FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB, Opcode,
2826 getEdgeProbability(BrMBB, Succ0MBB),
2827 getEdgeProbability(BrMBB, Succ1MBB),
2828 /*InvertCond=*/false);
2829 // If the compares in later blocks need to use values not currently
2830 // exported from this block, export them now. This block should always
2831 // be the first entry.
2832 assert(SL->SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!");
2833
2834 // Allow some cases to be rejected.
2835 if (ShouldEmitAsBranches(SL->SwitchCases)) {
2836 for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i) {
2837 ExportFromCurrentBlock(SL->SwitchCases[i].CmpLHS);
2838 ExportFromCurrentBlock(SL->SwitchCases[i].CmpRHS);
2839 }
2840
2841 // Emit the branch for this block.
2842 visitSwitchCase(SL->SwitchCases[0], BrMBB);
2843 SL->SwitchCases.erase(SL->SwitchCases.begin());
2844 return;
2845 }
2846
2847 // Okay, we decided not to do this, remove any inserted MBB's and clear
2848 // SwitchCases.
2849 for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i)
2850 FuncInfo.MF->erase(SL->SwitchCases[i].ThisBB);
2851
2852 SL->SwitchCases.clear();
2853 }
2854 }
2855
2856 // Create a CaseBlock record representing this branch.
2858 nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc());
2859
2860 // Use visitSwitchCase to actually insert the fast branch sequence for this
2861 // cond branch.
2862 visitSwitchCase(CB, BrMBB);
2863}
2864
2865/// visitSwitchCase - Emits the necessary code to represent a single node in
2866/// the binary search tree resulting from lowering a switch instruction.
2868 MachineBasicBlock *SwitchBB) {
2869 SDValue Cond;
2870 SDValue CondLHS = getValue(CB.CmpLHS);
2871 SDLoc dl = CB.DL;
2872
2873 if (CB.CC == ISD::SETTRUE) {
2874 // Branch or fall through to TrueBB.
2875 addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
2876 SwitchBB->normalizeSuccProbs();
2877 if (CB.TrueBB != NextBlock(SwitchBB)) {
2878 DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, getControlRoot(),
2879 DAG.getBasicBlock(CB.TrueBB)));
2880 }
2881 return;
2882 }
2883
2884 auto &TLI = DAG.getTargetLoweringInfo();
2885 EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), CB.CmpLHS->getType());
2886
2887 // Build the setcc now.
2888 if (!CB.CmpMHS) {
2889 // Fold "(X == true)" to X and "(X == false)" to !X to
2890 // handle common cases produced by branch lowering.
2891 if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&
2892 CB.CC == ISD::SETEQ)
2893 Cond = CondLHS;
2894 else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&
2895 CB.CC == ISD::SETEQ) {
2896 SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType());
2897 Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);
2898 } else {
2899 SDValue CondRHS = getValue(CB.CmpRHS);
2900
2901 // If a pointer's DAG type is larger than its memory type then the DAG
2902 // values are zero-extended. This breaks signed comparisons so truncate
2903 // back to the underlying type before doing the compare.
2904 if (CondLHS.getValueType() != MemVT) {
2905 CondLHS = DAG.getPtrExtOrTrunc(CondLHS, getCurSDLoc(), MemVT);
2906 CondRHS = DAG.getPtrExtOrTrunc(CondRHS, getCurSDLoc(), MemVT);
2907 }
2908 Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, CondRHS, CB.CC);
2909 }
2910 } else {
2911 assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now");
2912
2913 const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
2914 const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
2915
2916 SDValue CmpOp = getValue(CB.CmpMHS);
2917 EVT VT = CmpOp.getValueType();
2918
2919 if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
2920 Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT),
2921 ISD::SETLE);
2922 } else {
2923 SDValue SUB = DAG.getNode(ISD::SUB, dl,
2924 VT, CmpOp, DAG.getConstant(Low, dl, VT));
2925 Cond = DAG.getSetCC(dl, MVT::i1, SUB,
2926 DAG.getConstant(High-Low, dl, VT), ISD::SETULE);
2927 }
2928 }
2929
2930 // Update successor info
2931 addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
2932 // TrueBB and FalseBB are always different unless the incoming IR is
2933 // degenerate. This only happens when running llc on weird IR.
2934 if (CB.TrueBB != CB.FalseBB)
2935 addSuccessorWithProb(SwitchBB, CB.FalseBB, CB.FalseProb);
2936 SwitchBB->normalizeSuccProbs();
2937
2938 // If the lhs block is the next block, invert the condition so that we can
2939 // fall through to the lhs instead of the rhs block.
2940 if (CB.TrueBB == NextBlock(SwitchBB)) {
2941 std::swap(CB.TrueBB, CB.FalseBB);
2942 SDValue True = DAG.getConstant(1, dl, Cond.getValueType());
2943 Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
2944 }
2945
2946 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2947 MVT::Other, getControlRoot(), Cond,
2949
2950 setValue(CurInst, BrCond);
2951
2952 // Insert the false branch. Do this even if it's a fall through branch,
2953 // this makes it easier to do DAG optimizations which require inverting
2954 // the branch condition.
2955 BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
2957
2958 DAG.setRoot(BrCond);
2959}
2960
2961/// visitJumpTable - Emit JumpTable node in the current MBB
2963 // Emit the code for the jump table
2964 assert(JT.SL && "Should set SDLoc for SelectionDAG!");
2965 assert(JT.Reg != -1U && "Should lower JT Header first!");
2967 SDValue Index = DAG.getCopyFromReg(getControlRoot(), *JT.SL, JT.Reg, PTy);
2968 SDValue Table = DAG.getJumpTable(JT.JTI, PTy);
2969 SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, *JT.SL, MVT::Other,
2970 Index.getValue(1), Table, Index);
2971 DAG.setRoot(BrJumpTable);
2972}
2973
2974/// visitJumpTableHeader - This function emits necessary code to produce index
2975/// in the JumpTable from switch case.
2977 JumpTableHeader &JTH,
2978 MachineBasicBlock *SwitchBB) {
2979 assert(JT.SL && "Should set SDLoc for SelectionDAG!");
2980 const SDLoc &dl = *JT.SL;
2981
2982 // Subtract the lowest switch case value from the value being switched on.
2983 SDValue SwitchOp = getValue(JTH.SValue);
2984 EVT VT = SwitchOp.getValueType();
2985 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
2986 DAG.getConstant(JTH.First, dl, VT));
2987
2988 // The SDNode we just created, which holds the value being switched on minus
2989 // the smallest case value, needs to be copied to a virtual register so it
2990 // can be used as an index into the jump table in a subsequent basic block.
2991 // This value may be smaller or larger than the target's pointer type, and
2992 // therefore require extension or truncating.
2994 SwitchOp = DAG.getZExtOrTrunc(Sub, dl, TLI.getPointerTy(DAG.getDataLayout()));
2995
2996 unsigned JumpTableReg =
2998 SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl,
2999 JumpTableReg, SwitchOp);
3000 JT.Reg = JumpTableReg;
3001
3002 if (!JTH.FallthroughUnreachable) {
3003 // Emit the range check for the jump table, and branch to the default block
3004 // for the switch statement if the value being switched on exceeds the
3005 // largest case in the switch.
3006 SDValue CMP = DAG.getSetCC(
3008 Sub.getValueType()),
3009 Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT);
3010
3011 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
3012 MVT::Other, CopyTo, CMP,
3013 DAG.getBasicBlock(JT.Default));
3014
3015 // Avoid emitting unnecessary branches to the next block.
3016 if (JT.MBB != NextBlock(SwitchBB))
3017 BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
3018 DAG.getBasicBlock(JT.MBB));
3019
3020 DAG.setRoot(BrCond);
3021 } else {
3022 // Avoid emitting unnecessary branches to the next block.
3023 if (JT.MBB != NextBlock(SwitchBB))
3024 DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, CopyTo,
3025 DAG.getBasicBlock(JT.MBB)));
3026 else
3027 DAG.setRoot(CopyTo);
3028 }
3029}
3030
3031/// Create a LOAD_STACK_GUARD node, and let it carry the target specific global
3032/// variable if there exists one.
3034 SDValue &Chain) {
3035 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3036 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
3037 EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());
3041 DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD, DL, PtrTy, Chain);
3042 if (Global) {
3043 MachinePointerInfo MPInfo(Global);
3047 MPInfo, Flags, LocationSize::precise(PtrTy.getSizeInBits() / 8),
3048 DAG.getEVTAlign(PtrTy));
3049 DAG.setNodeMemRefs(Node, {MemRef});
3050 }
3051 if (PtrTy != PtrMemTy)
3052 return DAG.getPtrExtOrTrunc(SDValue(Node, 0), DL, PtrMemTy);
3053 return SDValue(Node, 0);
3054}
3055
3056/// Codegen a new tail for a stack protector check ParentMBB which has had its
3057/// tail spliced into a stack protector check success bb.
3058///
3059/// For a high level explanation of how this fits into the stack protector
3060/// generation see the comment on the declaration of class
3061/// StackProtectorDescriptor.
3063 MachineBasicBlock *ParentBB) {
3064
3065 // First create the loads to the guard/stack slot for the comparison.
3067 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
3068 EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());
3069
3070 MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
3071 int FI = MFI.getStackProtectorIndex();
3072
3073 SDValue Guard;
3074 SDLoc dl = getCurSDLoc();
3075 SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
3076 const Module &M = *ParentBB->getParent()->getFunction().getParent();
3077 Align Align =
3078 DAG.getDataLayout().getPrefTypeAlign(PointerType::get(M.getContext(), 0));
3079
3080 // Generate code to load the content of the guard slot.
3081 SDValue GuardVal = DAG.getLoad(
3082 PtrMemTy, dl, DAG.getEntryNode(), StackSlotPtr,
3085
3086 if (TLI.useStackGuardXorFP())
3087 GuardVal = TLI.emitStackGuardXorFP(DAG, GuardVal, dl);
3088
3089 // Retrieve guard check function, nullptr if instrumentation is inlined.
3090 if (const Function *GuardCheckFn = TLI.getSSPStackGuardCheck(M)) {
3091 // The target provides a guard check function to validate the guard value.
3092 // Generate a call to that function with the content of the guard slot as
3093 // argument.
3094 FunctionType *FnTy = GuardCheckFn->getFunctionType();
3095 assert(FnTy->getNumParams() == 1 && "Invalid function signature");
3096
3099 Entry.Node = GuardVal;
3100 Entry.Ty = FnTy->getParamType(0);
3101 if (GuardCheckFn->hasParamAttribute(0, Attribute::AttrKind::InReg))
3102 Entry.IsInReg = true;
3103 Args.push_back(Entry);
3104
3108 .setCallee(GuardCheckFn->getCallingConv(), FnTy->getReturnType(),
3109 getValue(GuardCheckFn), std::move(Args));
3110
3111 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
3112 DAG.setRoot(Result.second);
3113 return;
3114 }
3115
3116 // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.
3117 // Otherwise, emit a volatile load to retrieve the stack guard value.
3118 SDValue Chain = DAG.getEntryNode();
3119 if (TLI.useLoadStackGuardNode()) {
3120 Guard = getLoadStackGuard(DAG, dl, Chain);
3121 } else {
3122 const Value *IRGuard = TLI.getSDagStackGuard(M);
3123 SDValue GuardPtr = getValue(IRGuard);
3124
3125 Guard = DAG.getLoad(PtrMemTy, dl, Chain, GuardPtr,
3126 MachinePointerInfo(IRGuard, 0), Align,
3128 }
3129
3130 // Perform the comparison via a getsetcc.
3132 *DAG.getContext(),
3133 Guard.getValueType()),
3134 Guard, GuardVal, ISD::SETNE);
3135
3136 // If the guard/stackslot do not equal, branch to failure MBB.
3137 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
3138 MVT::Other, GuardVal.getOperand(0),
3139 Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));
3140 // Otherwise branch to success MBB.
3141 SDValue Br = DAG.getNode(ISD::BR, dl,
3142 MVT::Other, BrCond,
3144
3145 DAG.setRoot(Br);
3146}
3147
3148/// Codegen the failure basic block for a stack protector check.
3149///
3150/// A failure stack protector machine basic block consists simply of a call to
3151/// __stack_chk_fail().
3152///
3153/// For a high level explanation of how this fits into the stack protector
3154/// generation see the comment on the declaration of class
3155/// StackProtectorDescriptor.
3156void
3160 CallOptions.setDiscardResult(true);
3161 SDValue Chain =
3162 TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid,
3163 std::nullopt, CallOptions, getCurSDLoc())
3164 .second;
3165 // On PS4/PS5, the "return address" must still be within the calling
3166 // function, even if it's at the very end, so emit an explicit TRAP here.
3167 // Passing 'true' for doesNotReturn above won't generate the trap for us.
3168 if (TM.getTargetTriple().isPS())
3169 Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);
3170 // WebAssembly needs an unreachable instruction after a non-returning call,
3171 // because the function return type can be different from __stack_chk_fail's
3172 // return type (void).
3173 if (TM.getTargetTriple().isWasm())
3174 Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);
3175
3176 DAG.setRoot(Chain);
3177}
3178
3179/// visitBitTestHeader - This function emits necessary code to produce value
3180/// suitable for "bit tests"
3182 MachineBasicBlock *SwitchBB) {
3183 SDLoc dl = getCurSDLoc();
3184
3185 // Subtract the minimum value.
3186 SDValue SwitchOp = getValue(B.SValue);
3187 EVT VT = SwitchOp.getValueType();
3188 SDValue RangeSub =
3189 DAG.getNode(ISD::SUB, dl, VT, SwitchOp, DAG.getConstant(B.First, dl, VT));
3190
3191 // Determine the type of the test operands.
3193 bool UsePtrType = false;
3194 if (!TLI.isTypeLegal(VT)) {
3195 UsePtrType = true;
3196 } else {
3197 for (unsigned i = 0, e = B.Cases.size(); i != e; ++i)
3198 if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) {
3199 // Switch table case range are encoded into series of masks.
3200 // Just use pointer type, it's guaranteed to fit.
3201 UsePtrType = true;
3202 break;
3203 }
3204 }
3205 SDValue Sub = RangeSub;
3206 if (UsePtrType) {
3207 VT = TLI.getPointerTy(DAG.getDataLayout());
3208 Sub = DAG.getZExtOrTrunc(Sub, dl, VT);
3209 }
3210
3211 B.RegVT = VT.getSimpleVT();
3212 B.Reg = FuncInfo.CreateReg(B.RegVT);
3213 SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub);
3214
3215 MachineBasicBlock* MBB = B.Cases[0].ThisBB;
3216
3217 if (!B.FallthroughUnreachable)
3218 addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);
3219 addSuccessorWithProb(SwitchBB, MBB, B.Prob);
3220 SwitchBB->normalizeSuccProbs();
3221
3222 SDValue Root = CopyTo;
3223 if (!B.FallthroughUnreachable) {
3224 // Conditional branch to the default block.
3225 SDValue RangeCmp = DAG.getSetCC(dl,
3227 RangeSub.getValueType()),
3228 RangeSub, DAG.getConstant(B.Range, dl, RangeSub.getValueType()),
3229 ISD::SETUGT);
3230
3231 Root = DAG.getNode(ISD::BRCOND, dl, MVT::Other, Root, RangeCmp,
3232 DAG.getBasicBlock(B.Default));
3233 }
3234
3235 // Avoid emitting unnecessary branches to the next block.
3236 if (MBB != NextBlock(SwitchBB))
3237 Root = DAG.getNode(ISD::BR, dl, MVT::Other, Root, DAG.getBasicBlock(MBB));
3238
3239 DAG.setRoot(Root);
3240}
3241
3242/// visitBitTestCase - this function produces one "bit test"
3244 MachineBasicBlock* NextMBB,
3245 BranchProbability BranchProbToNext,
3246 unsigned Reg,
3247 BitTestCase &B,
3248 MachineBasicBlock *SwitchBB) {
3249 SDLoc dl = getCurSDLoc();
3250 MVT VT = BB.RegVT;
3251 SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT);
3252 SDValue Cmp;
3253 unsigned PopCount = llvm::popcount(B.Mask);
3255 if (PopCount == 1) {
3256 // Testing for a single bit; just compare the shift count with what it
3257 // would need to be to shift a 1 bit in that position.
3258 Cmp = DAG.getSetCC(
3260 ShiftOp, DAG.getConstant(llvm::countr_zero(B.Mask), dl, VT),
3261 ISD::SETEQ);
3262 } else if (PopCount == BB.Range) {
3263 // There is only one zero bit in the range, test for it directly.
3264 Cmp = DAG.getSetCC(
3266 ShiftOp, DAG.getConstant(llvm::countr_one(B.Mask), dl, VT), ISD::SETNE);
3267 } else {
3268 // Make desired shift
3269 SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT,
3270 DAG.getConstant(1, dl, VT), ShiftOp);
3271
3272 // Emit bit tests and jumps
3273 SDValue AndOp = DAG.getNode(ISD::AND, dl,
3274 VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT));
3275 Cmp = DAG.getSetCC(
3277 AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE);
3278 }
3279
3280 // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
3281 addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);
3282 // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
3283 addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
3284 // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
3285 // one as they are relative probabilities (and thus work more like weights),
3286 // and hence we need to normalize them to let the sum of them become one.
3287 SwitchBB->normalizeSuccProbs();
3288
3289 SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl,
3290 MVT::Other, getControlRoot(),
3291 Cmp, DAG.getBasicBlock(B.TargetBB));
3292
3293 // Avoid emitting unnecessary branches to the next block.
3294 if (NextMBB != NextBlock(SwitchBB))
3295 BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd,
3296 DAG.getBasicBlock(NextMBB));
3297
3298 DAG.setRoot(BrAnd);
3299}
3300
3301void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
3302 MachineBasicBlock *InvokeMBB = FuncInfo.MBB;
3303
3304 // Retrieve successors. Look through artificial IR level blocks like
3305 // catchswitch for successors.
3306 MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)];
3307 const BasicBlock *EHPadBB = I.getSuccessor(1);
3308 MachineBasicBlock *EHPadMBB = FuncInfo.MBBMap[EHPadBB];
3309
3310 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
3311 // have to do anything here to lower funclet bundles.
3312 assert(!I.hasOperandBundlesOtherThan(
3313 {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition,
3314 LLVMContext::OB_gc_live, LLVMContext::OB_funclet,
3315 LLVMContext::OB_cfguardtarget,
3316 LLVMContext::OB_clang_arc_attachedcall}) &&
3317 "Cannot lower invokes with arbitrary operand bundles yet!");
3318
3319 const Value *Callee(I.getCalledOperand());
3320 const Function *Fn = dyn_cast<Function>(Callee);
3321 if (isa<InlineAsm>(Callee))
3322 visitInlineAsm(I, EHPadBB);
3323 else if (Fn && Fn->isIntrinsic()) {
3324 switch (Fn->getIntrinsicID()) {
3325 default:
3326 llvm_unreachable("Cannot invoke this intrinsic");
3327 case Intrinsic::donothing:
3328 // Ignore invokes to @llvm.donothing: jump directly to the next BB.
3329 case Intrinsic::seh_try_begin:
3330 case Intrinsic::seh_scope_begin:
3331 case Intrinsic::seh_try_end:
3332 case Intrinsic::seh_scope_end:
3333 if (EHPadMBB)
3334 // a block referenced by EH table
3335 // so dtor-funclet not removed by opts
3336 EHPadMBB->setMachineBlockAddressTaken();
3337 break;
3338 case Intrinsic::experimental_patchpoint_void:
3339 case Intrinsic::experimental_patchpoint:
3340 visitPatchpoint(I, EHPadBB);
3341 break;
3342 case Intrinsic::experimental_gc_statepoint:
3343 LowerStatepoint(cast<GCStatepointInst>(I), EHPadBB);
3344 break;
3345 case Intrinsic::wasm_rethrow: {
3346 // This is usually done in visitTargetIntrinsic, but this intrinsic is
3347 // special because it can be invoked, so we manually lower it to a DAG
3348 // node here.
3350 Ops.push_back(getRoot()); // inchain
3352 Ops.push_back(
3353 DAG.getTargetConstant(Intrinsic::wasm_rethrow, getCurSDLoc(),
3355 SDVTList VTs = DAG.getVTList(ArrayRef<EVT>({MVT::Other})); // outchain
3357 break;
3358 }
3359 }
3360 } else if (I.hasDeoptState()) {
3361 // Currently we do not lower any intrinsic calls with deopt operand bundles.
3362 // Eventually we will support lowering the @llvm.experimental.deoptimize
3363 // intrinsic, and right now there are no plans to support other intrinsics
3364 // with deopt state.
3365 LowerCallSiteWithDeoptBundle(&I, getValue(Callee), EHPadBB);
3366 } else {
3367 LowerCallTo(I, getValue(Callee), false, false, EHPadBB);
3368 }
3369
3370 // If the value of the invoke is used outside of its defining block, make it
3371 // available as a virtual register.
3372 // We already took care of the exported value for the statepoint instruction
3373 // during call to the LowerStatepoint.
3374 if (!isa<GCStatepointInst>(I)) {
3376 }
3377
3380 BranchProbability EHPadBBProb =
3381 BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)
3383 findUnwindDestinations(FuncInfo, EHPadBB, EHPadBBProb, UnwindDests);
3384
3385 // Update successor info.
3386 addSuccessorWithProb(InvokeMBB, Return);
3387 for (auto &UnwindDest : UnwindDests) {
3388 UnwindDest.first->setIsEHPad();
3389 addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
3390 }
3391 InvokeMBB->normalizeSuccProbs();
3392
3393 // Drop into normal successor.
3395 DAG.getBasicBlock(Return)));
3396}
3397
3398void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {
3399 MachineBasicBlock *CallBrMBB = FuncInfo.MBB;
3400
3401 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
3402 // have to do anything here to lower funclet bundles.
3403 assert(!I.hasOperandBundlesOtherThan(
3404 {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&
3405 "Cannot lower callbrs with arbitrary operand bundles yet!");
3406
3407 assert(I.isInlineAsm() && "Only know how to handle inlineasm callbr");
3408 visitInlineAsm(I);
3410
3411 // Retrieve successors.
3413 Dests.insert(I.getDefaultDest());
3414 MachineBasicBlock *Return = FuncInfo.MBBMap[I.getDefaultDest()];
3415
3416 // Update successor info.
3417 addSuccessorWithProb(CallBrMBB, Return, BranchProbability::getOne());
3418 for (unsigned i = 0, e = I.getNumIndirectDests(); i < e; ++i) {
3419 BasicBlock *Dest = I.getIndirectDest(i);
3421 Target->setIsInlineAsmBrIndirectTarget();
3422 Target->setMachineBlockAddressTaken();
3423 Target->setLabelMustBeEmitted();
3424 // Don't add duplicate machine successors.
3425 if (Dests.insert(Dest).second)
3426 addSuccessorWithProb(CallBrMBB, Target, BranchProbability::getZero());
3427 }
3428 CallBrMBB->normalizeSuccProbs();
3429
3430 // Drop into default successor.
3432 MVT::Other, getControlRoot(),
3433 DAG.getBasicBlock(Return)));
3434}
3435
3436void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
3437 llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!");
3438}
3439
3440void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
3442 "Call to landingpad not in landing pad!");
3443
3444 // If there aren't registers to copy the values into (e.g., during SjLj
3445 // exceptions), then don't bother to create these DAG nodes.
3447 const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn();
3448 if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&
3449 TLI.getExceptionSelectorRegister(PersonalityFn) == 0)
3450 return;
3451
3452 // If landingpad's return type is token type, we don't create DAG nodes
3453 // for its exception pointer and selector value. The extraction of exception
3454 // pointer or selector value from token type landingpads is not currently
3455 // supported.
3456 if (LP.getType()->isTokenTy())
3457 return;
3458
3459 SmallVector<EVT, 2> ValueVTs;
3460 SDLoc dl = getCurSDLoc();
3461 ComputeValueVTs(TLI, DAG.getDataLayout(), LP.getType(), ValueVTs);
3462 assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported");
3463
3464 // Get the two live-in registers as SDValues. The physregs have already been
3465 // copied into virtual registers.
3466 SDValue Ops[2];
3468 Ops[0] = DAG.getZExtOrTrunc(
3472 dl, ValueVTs[0]);
3473 } else {
3474 Ops[0] = DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()));
3475 }
3476 Ops[1] = DAG.getZExtOrTrunc(
3480 dl, ValueVTs[1]);
3481
3482 // Merge into one.
3484 DAG.getVTList(ValueVTs), Ops);
3485 setValue(&LP, Res);
3486}
3487
3490 // Update JTCases.
3491 for (JumpTableBlock &JTB : SL->JTCases)
3492 if (JTB.first.HeaderBB == First)
3493 JTB.first.HeaderBB = Last;
3494
3495 // Update BitTestCases.
3496 for (BitTestBlock &BTB : SL->BitTestCases)
3497 if (BTB.Parent == First)
3498 BTB.Parent = Last;
3499}
3500
3501void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
3502 MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
3503
3504 // Update machine-CFG edges with unique successors.
3506 for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
3507 BasicBlock *BB = I.getSuccessor(i);
3508 bool Inserted = Done.insert(BB).second;
3509 if (!Inserted)
3510 continue;
3511
3512 MachineBasicBlock *Succ = FuncInfo.MBBMap[BB];
3513 addSuccessorWithProb(IndirectBrMBB, Succ);
3514 }
3515 IndirectBrMBB->normalizeSuccProbs();
3516
3518 MVT::Other, getControlRoot(),
3519 getValue(I.getAddress())));
3520}
3521
3522void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
3524 return;
3525
3526 // We may be able to ignore unreachable behind a noreturn call.
3528 if (const CallInst *Call = dyn_cast_or_null<CallInst>(I.getPrevNode())) {
3529 if (Call->doesNotReturn())
3530 return;
3531 }
3532 }
3533
3534 DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
3535}
3536
3537void SelectionDAGBuilder::visitUnary(const User &I, unsigned Opcode) {
3539 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3540 Flags.copyFMF(*FPOp);
3541
3542 SDValue Op = getValue(I.getOperand(0));
3543 SDValue UnNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op.getValueType(),
3544 Op, Flags);
3545 setValue(&I, UnNodeValue);
3546}
3547
3548void SelectionDAGBuilder::visitBinary(const User &I, unsigned Opcode) {
3550 if (auto *OFBinOp = dyn_cast<OverflowingBinaryOperator>(&I)) {
3551 Flags.setNoSignedWrap(OFBinOp->hasNoSignedWrap());
3552 Flags.setNoUnsignedWrap(OFBinOp->hasNoUnsignedWrap());
3553 }
3554 if (auto *ExactOp = dyn_cast<PossiblyExactOperator>(&I))
3555 Flags.setExact(ExactOp->isExact());
3556 if (auto *DisjointOp = dyn_cast<PossiblyDisjointInst>(&I))
3557 Flags.setDisjoint(DisjointOp->isDisjoint());
3558 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3559 Flags.copyFMF(*FPOp);
3560
3561 SDValue Op1 = getValue(I.getOperand(0));
3562 SDValue Op2 = getValue(I.getOperand(1));
3563 SDValue BinNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(),
3564 Op1, Op2, Flags);
3565 setValue(&I, BinNodeValue);
3566}
3567
3568void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
3569 SDValue Op1 = getValue(I.getOperand(0));
3570 SDValue Op2 = getValue(I.getOperand(1));
3571
3573 Op1.getValueType(), DAG.getDataLayout());
3574
3575 // Coerce the shift amount to the right type if we can. This exposes the
3576 // truncate or zext to optimization early.
3577 if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
3579 "Unexpected shift type");
3580 Op2 = DAG.getZExtOrTrunc(Op2, getCurSDLoc(), ShiftTy);
3581 }
3582
3583 bool nuw = false;
3584 bool nsw = false;
3585 bool exact = false;
3586
3587 if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {
3588
3589 if (const OverflowingBinaryOperator *OFBinOp =
3590 dyn_cast<const OverflowingBinaryOperator>(&I)) {
3591 nuw = OFBinOp->hasNoUnsignedWrap();
3592 nsw = OFBinOp->hasNoSignedWrap();
3593 }
3594 if (const PossiblyExactOperator *ExactOp =
3595 dyn_cast<const PossiblyExactOperator>(&I))
3596 exact = ExactOp->isExact();
3597 }
3599 Flags.setExact(exact);
3600 Flags.setNoSignedWrap(nsw);
3601 Flags.setNoUnsignedWrap(nuw);
3602 SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,
3603 Flags);
3604 setValue(&I, Res);
3605}
3606
3607void SelectionDAGBuilder::visitSDiv(const User &I) {
3608 SDValue Op1 = getValue(I.getOperand(0));
3609 SDValue Op2 = getValue(I.getOperand(1));
3610
3612 Flags.setExact(isa<PossiblyExactOperator>(&I) &&
3613 cast<PossiblyExactOperator>(&I)->isExact());
3615 Op2, Flags));
3616}
3617
3618void SelectionDAGBuilder::visitICmp(const User &I) {
3620 if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I))
3621 predicate = IC->getPredicate();
3622 else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I))
3623 predicate = ICmpInst::Predicate(IC->getPredicate());
3624 SDValue Op1 = getValue(I.getOperand(0));
3625 SDValue Op2 = getValue(I.getOperand(1));
3626 ISD::CondCode Opcode = getICmpCondCode(predicate);
3627
3628 auto &TLI = DAG.getTargetLoweringInfo();
3629 EVT MemVT =
3630 TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
3631
3632 // If a pointer's DAG type is larger than its memory type then the DAG values
3633 // are zero-extended. This breaks signed comparisons so truncate back to the
3634 // underlying type before doing the compare.
3635 if (Op1.getValueType() != MemVT) {
3636 Op1 = DAG.getPtrExtOrTrunc(Op1, getCurSDLoc(), MemVT);
3637 Op2 = DAG.getPtrExtOrTrunc(Op2, getCurSDLoc(), MemVT);
3638 }
3639
3641 I.getType());
3642 setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));
3643}
3644
3645void SelectionDAGBuilder::visitFCmp(const User &I) {
3647 if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I))
3648 predicate = FC->getPredicate();
3649 else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I))
3650 predicate = FCmpInst::Predicate(FC->getPredicate());
3651 SDValue Op1 = getValue(I.getOperand(0));
3652 SDValue Op2 = getValue(I.getOperand(1));
3653
3654 ISD::CondCode Condition = getFCmpCondCode(predicate);
3655 auto *FPMO = cast<FPMathOperator>(&I);
3656 if (FPMO->hasNoNaNs() || TM.Options.NoNaNsFPMath)
3657 Condition = getFCmpCodeWithoutNaN(Condition);
3658
3660 Flags.copyFMF(*FPMO);
3661 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
3662
3664 I.getType());
3665 setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));
3666}
3667
3668// Check if the condition of the select has one use or two users that are both
3669// selects with the same condition.
3670static bool hasOnlySelectUsers(const Value *Cond) {
3671 return llvm::all_of(Cond->users(), [](const Value *V) {
3672 return isa<SelectInst>(V);
3673 });
3674}
3675
3676void SelectionDAGBuilder::visitSelect(const User &I) {
3677 SmallVector<EVT, 4> ValueVTs;
3679 ValueVTs);
3680 unsigned NumValues = ValueVTs.size();
3681 if (NumValues == 0) return;
3682
3683 SmallVector<SDValue, 4> Values(NumValues);
3684 SDValue Cond = getValue(I.getOperand(0));
3685 SDValue LHSVal = getValue(I.getOperand(1));
3686 SDValue RHSVal = getValue(I.getOperand(2));
3687 SmallVector<SDValue, 1> BaseOps(1, Cond);
3688 ISD::NodeType OpCode =
3689 Cond.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT;
3690
3691 bool IsUnaryAbs = false;
3692 bool Negate = false;
3693
3695 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3696 Flags.copyFMF(*FPOp);
3697
3698 Flags.setUnpredictable(
3699 cast<SelectInst>(I).getMetadata(LLVMContext::MD_unpredictable));
3700
3701 // Min/max matching is only viable if all output VTs are the same.
3702 if (all_equal(ValueVTs)) {
3703 EVT VT = ValueVTs[0];
3704 LLVMContext &Ctx = *DAG.getContext();
3705 auto &TLI = DAG.getTargetLoweringInfo();
3706
3707 // We care about the legality of the operation after it has been type
3708 // legalized.
3709 while (TLI.getTypeAction(Ctx, VT) != TargetLoweringBase::TypeLegal)
3710 VT = TLI.getTypeToTransformTo(Ctx, VT);
3711
3712 // If the vselect is legal, assume we want to leave this as a vector setcc +
3713 // vselect. Otherwise, if this is going to be scalarized, we want to see if
3714 // min/max is legal on the scalar type.
3715 bool UseScalarMinMax = VT.isVector() &&
3717
3718 // ValueTracking's select pattern matching does not account for -0.0,
3719 // so we can't lower to FMINIMUM/FMAXIMUM because those nodes specify that
3720 // -0.0 is less than +0.0.
3721 Value *LHS, *RHS;
3722 auto SPR = matchSelectPattern(const_cast<User*>(&I), LHS, RHS);
3724 switch (SPR.Flavor) {
3725 case SPF_UMAX: Opc = ISD::UMAX; break;
3726 case SPF_UMIN: Opc = ISD::UMIN; break;
3727 case SPF_SMAX: Opc = ISD::SMAX; break;
3728 case SPF_SMIN: Opc = ISD::SMIN; break;
3729 case SPF_FMINNUM:
3730 switch (SPR.NaNBehavior) {
3731 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
3732 case SPNB_RETURNS_NAN: break;
3733 case SPNB_RETURNS_OTHER: Opc = ISD::FMINNUM; break;
3734 case SPNB_RETURNS_ANY:
3736 (UseScalarMinMax &&
3738 Opc = ISD::FMINNUM;
3739 break;
3740 }
3741 break;
3742 case SPF_FMAXNUM:
3743 switch (SPR.NaNBehavior) {
3744 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
3745 case SPNB_RETURNS_NAN: break;
3746 case SPNB_RETURNS_OTHER: Opc = ISD::FMAXNUM; break;
3747 case SPNB_RETURNS_ANY:
3749 (UseScalarMinMax &&
3751 Opc = ISD::FMAXNUM;
3752 break;
3753 }
3754 break;
3755 case SPF_NABS:
3756 Negate = true;
3757 [[fallthrough]];
3758 case SPF_ABS:
3759 IsUnaryAbs = true;
3760 Opc = ISD::ABS;
3761 break;
3762 default: break;
3763 }
3764
3765 if (!IsUnaryAbs && Opc != ISD::DELETED_NODE &&
3766 (TLI.isOperationLegalOrCustomOrPromote(Opc, VT) ||
3767 (UseScalarMinMax &&
3768 TLI.isOperationLegalOrCustom(Opc, VT.getScalarType()))) &&
3769 // If the underlying comparison instruction is used by any other
3770 // instruction, the consumed instructions won't be destroyed, so it is
3771 // not profitable to convert to a min/max.
3772 hasOnlySelectUsers(cast<SelectInst>(I).getCondition())) {
3773 OpCode = Opc;
3774 LHSVal = getValue(LHS);
3775 RHSVal = getValue(RHS);
3776 BaseOps.clear();
3777 }
3778
3779 if (IsUnaryAbs) {
3780 OpCode = Opc;
3781 LHSVal = getValue(LHS);
3782 BaseOps.clear();
3783 }
3784 }
3785
3786 if (IsUnaryAbs) {
3787 for (unsigned i = 0; i != NumValues; ++i) {
3788 SDLoc dl = getCurSDLoc();
3789 EVT VT = LHSVal.getNode()->getValueType(LHSVal.getResNo() + i);
3790 Values[i] =
3791 DAG.getNode(OpCode, dl, VT, LHSVal.getValue(LHSVal.getResNo() + i));
3792 if (Negate)
3793 Values[i] = DAG.getNegative(Values[i], dl, VT);
3794 }
3795 } else {
3796 for (unsigned i = 0; i != NumValues; ++i) {
3797 SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());
3798 Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));
3799 Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));
3800 Values[i] = DAG.getNode(
3801 OpCode, getCurSDLoc(),
3802 LHSVal.getNode()->getValueType(LHSVal.getResNo() + i), Ops, Flags);
3803 }
3804 }
3805
3807 DAG.getVTList(ValueVTs), Values));
3808}
3809
3810void SelectionDAGBuilder::visitTrunc(const User &I) {
3811 // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
3812 SDValue N = getValue(I.getOperand(0));
3814 I.getType());
3816}
3817
3818void SelectionDAGBuilder::visitZExt(const User &I) {
3819 // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3820 // ZExt also can't be a cast to bool for same reason. So, nothing much to do
3821 SDValue N = getValue(I.getOperand(0));
3822 auto &TLI = DAG.getTargetLoweringInfo();
3823 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3824
3826 if (auto *PNI = dyn_cast<PossiblyNonNegInst>(&I))
3827 Flags.setNonNeg(PNI->hasNonNeg());
3828
3829 // Eagerly use nonneg information to canonicalize towards sign_extend if
3830 // that is the target's preference.
3831 // TODO: Let the target do this later.
3832 if (Flags.hasNonNeg() &&
3833 TLI.isSExtCheaperThanZExt(N.getValueType(), DestVT)) {
3835 return;
3836 }
3837
3838 setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N, Flags));
3839}
3840
3841void SelectionDAGBuilder::visitSExt(const User &I) {
3842 // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3843 // SExt also can't be a cast to bool for same reason. So, nothing much to do
3844 SDValue N = getValue(I.getOperand(0));
3846 I.getType());
3848}
3849
3850void SelectionDAGBuilder::visitFPTrunc(const User &I) {
3851 // FPTrunc is never a no-op cast, no need to check
3852 SDValue N = getValue(I.getOperand(0));
3853 SDLoc dl = getCurSDLoc();
3855 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3856 setValue(&I, DAG.getNode(ISD::FP_ROUND, dl, DestVT, N,
3858 0, dl, TLI.getPointerTy(DAG.getDataLayout()))));
3859}
3860
3861void SelectionDAGBuilder::visitFPExt(const User &I) {
3862 // FPExt is never a no-op cast, no need to check
3863 SDValue N = getValue(I.getOperand(0));
3865 I.getType());
3867}
3868
3869void SelectionDAGBuilder::visitFPToUI(const User &I) {
3870 // FPToUI is never a no-op cast, no need to check
3871 SDValue N = getValue(I.getOperand(0));
3873 I.getType());
3875}
3876
3877void SelectionDAGBuilder::visitFPToSI(const User &I) {
3878 // FPToSI is never a no-op cast, no need to check
3879 SDValue N = getValue(I.getOperand(0));
3881 I.getType());
3883}
3884
3885void SelectionDAGBuilder::visitUIToFP(const User &I) {
3886 // UIToFP is never a no-op cast, no need to check
3887 SDValue N = getValue(I.getOperand(0));
3889 I.getType());
3891 if (auto *PNI = dyn_cast<PossiblyNonNegInst>(&I))
3892 Flags.setNonNeg(PNI->hasNonNeg());
3893
3894 setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N, Flags));
3895}
3896
3897void SelectionDAGBuilder::visitSIToFP(const User &I) {
3898 // SIToFP is never a no-op cast, no need to check
3899 SDValue N = getValue(I.getOperand(0));
3901 I.getType());
3903}
3904
3905void SelectionDAGBuilder::visitPtrToInt(const User &I) {
3906 // What to do depends on the size of the integer and the size of the pointer.
3907 // We can either truncate, zero extend, or no-op, accordingly.
3908 SDValue N = getValue(I.getOperand(0));
3909 auto &TLI = DAG.getTargetLoweringInfo();
3911 I.getType());
3912 EVT PtrMemVT =
3913 TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
3914 N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), PtrMemVT);
3915 N = DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT);
3916 setValue(&I, N);
3917}
3918
3919void SelectionDAGBuilder::visitIntToPtr(const User &I) {
3920 // What to do depends on the size of the integer and the size of the pointer.
3921 // We can either truncate, zero extend, or no-op, accordingly.
3922 SDValue N = getValue(I.getOperand(0));
3923 auto &TLI = DAG.getTargetLoweringInfo();
3924 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3925 EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());
3926 N = DAG.getZExtOrTrunc(N, getCurSDLoc(), PtrMemVT);
3927 N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), DestVT);
3928 setValue(&I, N);
3929}
3930
3931void SelectionDAGBuilder::visitBitCast(const User &I) {
3932 SDValue N = getValue(I.getOperand(0));
3933 SDLoc dl = getCurSDLoc();
3935 I.getType());
3936
3937 // BitCast assures us that source and destination are the same size so this is
3938 // either a BITCAST or a no-op.
3939 if (DestVT != N.getValueType())
3941 DestVT, N)); // convert types.
3942 // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
3943 // might fold any kind of constant expression to an integer constant and that
3944 // is not what we are looking for. Only recognize a bitcast of a genuine
3945 // constant integer as an opaque constant.
3946 else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))
3947 setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false,
3948 /*isOpaque*/true));
3949 else
3950 setValue(&I, N); // noop cast.
3951}
3952
3953void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
3955 const Value *SV = I.getOperand(0);
3956 SDValue N = getValue(SV);
3957 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3958
3959 unsigned SrcAS = SV->getType()->getPointerAddressSpace();
3960 unsigned DestAS = I.getType()->getPointerAddressSpace();
3961
3962 if (!TM.isNoopAddrSpaceCast(SrcAS, DestAS))
3963 N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS);
3964
3965 setValue(&I, N);
3966}
3967
3968void SelectionDAGBuilder::visitInsertElement(const User &I) {
3970 SDValue InVec = getValue(I.getOperand(0));
3971 SDValue InVal = getValue(I.getOperand(1));
3972 SDValue InIdx = DAG.getZExtOrTrunc(getValue(I.getOperand(2)), getCurSDLoc(),
3975 TLI.getValueType(DAG.getDataLayout(), I.getType()),
3976 InVec, InVal, InIdx));
3977}
3978
3979void SelectionDAGBuilder::visitExtractElement(const User &I) {
3981 SDValue InVec = getValue(I.getOperand(0));
3982 SDValue InIdx = DAG.getZExtOrTrunc(getValue(I.getOperand(1)), getCurSDLoc(),
3985 TLI.getValueType(DAG.getDataLayout(), I.getType()),
3986 InVec, InIdx));
3987}
3988
3989void SelectionDAGBuilder::visitShuffleVector(const User &I) {
3990 SDValue Src1 = getValue(I.getOperand(0));
3991 SDValue Src2 = getValue(I.getOperand(1));
3993 if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I))
3994 Mask = SVI->getShuffleMask();
3995 else
3996 Mask = cast<ConstantExpr>(I).getShuffleMask();
3997 SDLoc DL = getCurSDLoc();
3999 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4000 EVT SrcVT = Src1.getValueType();
4001
4002 if (all_of(Mask, [](int Elem) { return Elem == 0; }) &&
4003 VT.isScalableVector()) {
4004 // Canonical splat form of first element of first input vector.
4005 SDValue FirstElt =
4008 setValue(&I, DAG.getNode(ISD::SPLAT_VECTOR, DL, VT, FirstElt));
4009 return;
4010 }
4011
4012 // For now, we only handle splats for scalable vectors.
4013 // The DAGCombiner will perform a BUILD_VECTOR -> SPLAT_VECTOR transformation
4014 // for targets that support a SPLAT_VECTOR for non-scalable vector types.
4015 assert(!VT.isScalableVector() && "Unsupported scalable vector shuffle");
4016
4017 unsigned SrcNumElts = SrcVT.getVectorNumElements();
4018 unsigned MaskNumElts = Mask.size();
4019
4020 if (SrcNumElts == MaskNumElts) {
4021 setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, Mask));
4022 return;
4023 }
4024
4025 // Normalize the shuffle vector since mask and vector length don't match.
4026 if (SrcNumElts < MaskNumElts) {
4027 // Mask is longer than the source vectors. We can use concatenate vector to
4028 // make the mask and vectors lengths match.
4029
4030 if (MaskNumElts % SrcNumElts == 0) {
4031 // Mask length is a multiple of the source vector length.
4032 // Check if the shuffle is some kind of concatenation of the input
4033 // vectors.
4034 unsigned NumConcat = MaskNumElts / SrcNumElts;
4035 bool IsConcat = true;
4036 SmallVector<int, 8> ConcatSrcs(NumConcat, -1);
4037 for (unsigned i = 0; i != MaskNumElts; ++i) {
4038 int Idx = Mask[i];
4039 if (Idx < 0)
4040 continue;
4041 // Ensure the indices in each SrcVT sized piece are sequential and that
4042 // the same source is used for the whole piece.
4043 if ((Idx % SrcNumElts != (i % SrcNumElts)) ||
4044 (ConcatSrcs[i / SrcNumElts] >= 0 &&
4045 ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts))) {
4046 IsConcat = false;
4047 break;
4048 }
4049 // Remember which source this index came from.
4050 ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;
4051 }
4052
4053 // The shuffle is concatenating multiple vectors together. Just emit
4054 // a CONCAT_VECTORS operation.
4055 if (IsConcat) {
4056 SmallVector<SDValue, 8> ConcatOps;
4057 for (auto Src : ConcatSrcs) {
4058 if (Src < 0)
4059 ConcatOps.push_back(DAG.getUNDEF(SrcVT));
4060 else if (Src == 0)
4061 ConcatOps.push_back(Src1);
4062 else
4063 ConcatOps.push_back(Src2);
4064 }
4065 setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps));
4066 return;
4067 }
4068 }
4069
4070 unsigned PaddedMaskNumElts = alignTo(MaskNumElts, SrcNumElts);
4071 unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
4072 EVT PaddedVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(),
4073 PaddedMaskNumElts);
4074
4075 // Pad both vectors with undefs to make them the same length as the mask.
4076 SDValue UndefVal = DAG.getUNDEF(SrcVT);
4077
4078 SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
4079 SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
4080 MOps1[0] = Src1;
4081 MOps2[0] = Src2;
4082
4083 Src1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps1);
4084 Src2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps2);
4085
4086 // Readjust mask for new input vector length.
4087 SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);
4088 for (unsigned i = 0; i != MaskNumElts; ++i) {
4089 int Idx = Mask[i];
4090 if (Idx >= (int)SrcNumElts)
4091 Idx -= SrcNumElts - PaddedMaskNumElts;
4092 MappedOps[i] = Idx;
4093 }
4094
4095 SDValue Result = DAG.getVectorShuffle(PaddedVT, DL, Src1, Src2, MappedOps);
4096
4097 // If the concatenated vector was padded, extract a subvector with the
4098 // correct number of elements.
4099 if (MaskNumElts != PaddedMaskNumElts)
4102
4103 setValue(&I, Result);
4104 return;
4105 }
4106
4107 if (SrcNumElts > MaskNumElts) {
4108 // Analyze the access pattern of the vector to see if we can extract
4109 // two subvectors and do the shuffle.
4110 int StartIdx[2] = { -1, -1 }; // StartIdx to extract from
4111 bool CanExtract = true;
4112 for (int Idx : Mask) {
4113 unsigned Input = 0;
4114 if (Idx < 0)
4115 continue;
4116
4117 if (Idx >= (int)SrcNumElts) {
4118 Input = 1;
4119 Idx -= SrcNumElts;
4120 }
4121
4122 // If all the indices come from the same MaskNumElts sized portion of
4123 // the sources we can use extract. Also make sure the extract wouldn't
4124 // extract past the end of the source.
4125 int NewStartIdx = alignDown(Idx, MaskNumElts);
4126 if (NewStartIdx + MaskNumElts > SrcNumElts ||
4127 (StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx))
4128 CanExtract = false;
4129 // Make sure we always update StartIdx as we use it to track if all
4130 // elements are undef.
4131 StartIdx[Input] = NewStartIdx;
4132 }
4133
4134 if (StartIdx[0] < 0 && StartIdx[1] < 0) {
4135 setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.
4136 return;
4137 }
4138 if (CanExtract) {
4139 // Extract appropriate subvector and generate a vector shuffle
4140 for (unsigned Input = 0; Input < 2; ++Input) {
4141 SDValue &Src = Input == 0 ? Src1 : Src2;
4142 if (StartIdx[Input] < 0)
4143 Src = DAG.getUNDEF(VT);
4144 else {
4145 Src = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Src,
4146 DAG.getVectorIdxConstant(StartIdx[Input], DL));
4147 }
4148 }
4149
4150 // Calculate new mask.
4151 SmallVector<int, 8> MappedOps(Mask);
4152 for (int &Idx : MappedOps) {
4153 if (Idx >= (int)SrcNumElts)
4154 Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
4155 else if (Idx >= 0)
4156 Idx -= StartIdx[0];
4157 }
4158
4159 setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, MappedOps));
4160 return;
4161 }
4162 }
4163
4164 // We can't use either concat vectors or extract subvectors so fall back to
4165 // replacing the shuffle with extract and build vector.
4166 // to insert and build vector.
4167 EVT EltVT = VT.getVectorElementType();
4169 for (int Idx : Mask) {
4170 SDValue Res;
4171
4172 if (Idx < 0) {
4173 Res = DAG.getUNDEF(EltVT);
4174 } else {
4175 SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
4176 if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
4177
4178 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Src,
4180 }
4181
4182 Ops.push_back(Res);
4183 }
4184
4185 setValue(&I, DAG.getBuildVector(VT, DL, Ops));
4186}
4187
4188void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) {
4189 ArrayRef<unsigned> Indices = I.getIndices();
4190 const Value *Op0 = I.getOperand(0);
4191 const Value *Op1 = I.getOperand(1);
4192 Type *AggTy = I.getType();
4193 Type *ValTy = Op1->getType();
4194 bool IntoUndef = isa<UndefValue>(Op0);
4195 bool FromUndef = isa<UndefValue>(Op1);
4196
4197 unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
4198
4200 SmallVector<EVT, 4> AggValueVTs;
4201 ComputeValueVTs(TLI, DAG.getDataLayout(), AggTy, AggValueVTs);
4202 SmallVector<EVT, 4> ValValueVTs;
4203 ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
4204
4205 unsigned NumAggValues = AggValueVTs.size();
4206 unsigned NumValValues = ValValueVTs.size();
4207 SmallVector<SDValue, 4> Values(NumAggValues);
4208
4209 // Ignore an insertvalue that produces an empty object
4210 if (!NumAggValues) {
4211 setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
4212 return;
4213 }
4214
4215 SDValue Agg = getValue(Op0);
4216 unsigned i = 0;
4217 // Copy the beginning value(s) from the original aggregate.
4218 for (; i != LinearIndex; ++i)
4219 Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
4220 SDValue(Agg.getNode(), Agg.getResNo() + i);
4221 // Copy values from the inserted value(s).
4222 if (NumValValues) {
4223 SDValue Val = getValue(Op1);
4224 for (; i != LinearIndex + NumValValues; ++i)
4225 Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :
4226 SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
4227 }
4228 // Copy remaining value(s) from the original aggregate.
4229 for (; i != NumAggValues; ++i)
4230 Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
4231 SDValue(Agg.getNode(), Agg.getResNo() + i);
4232
4234 DAG.getVTList(AggValueVTs), Values));
4235}
4236
4237void SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) {
4238 ArrayRef<unsigned> Indices = I.getIndices();
4239 const Value *Op0 = I.getOperand(0);
4240 Type *AggTy = Op0->getType();
4241 Type *ValTy = I.getType();
4242 bool OutOfUndef = isa<UndefValue>(Op0);
4243
4244 unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
4245
4247 SmallVector<EVT, 4> ValValueVTs;
4248 ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
4249
4250 unsigned NumValValues = ValValueVTs.size();
4251
4252 // Ignore a extractvalue that produces an empty object
4253 if (!NumValValues) {
4254 setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
4255 return;
4256 }
4257
4258 SmallVector<SDValue, 4> Values(NumValValues);
4259
4260 SDValue Agg = getValue(Op0);
4261 // Copy out the selected value(s).
4262 for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
4263 Values[i - LinearIndex] =
4264 OutOfUndef ?
4265 DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :
4266 SDValue(Agg.getNode(), Agg.getResNo() + i);
4267
4269 DAG.getVTList(ValValueVTs), Values));
4270}
4271
4272void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
4273 Value *Op0 = I.getOperand(0);
4274 // Note that the pointer operand may be a vector of pointers. Take the scalar
4275 // element which holds a pointer.
4276 unsigned AS = Op0->getType()->getScalarType()->getPointerAddressSpace();
4277 SDValue N = getValue(Op0);
4278 SDLoc dl = getCurSDLoc();
4279 auto &TLI = DAG.getTargetLoweringInfo();
4280
4281 // Normalize Vector GEP - all scalar operands should be converted to the
4282 // splat vector.
4283 bool IsVectorGEP = I.getType()->isVectorTy();
4284 ElementCount VectorElementCount =
4285 IsVectorGEP ? cast<VectorType>(I.getType())->getElementCount()
4287
4288 if (IsVectorGEP && !N.getValueType().isVector()) {
4290 EVT VT = EVT::getVectorVT(Context, N.getValueType(), VectorElementCount);
4291 N = DAG.getSplat(VT, dl, N);
4292 }
4293
4294 for (gep_type_iterator GTI = gep_type_begin(&I), E = gep_type_end(&I);
4295 GTI != E; ++GTI) {
4296 const Value *Idx = GTI.getOperand();
4297 if (StructType *StTy = GTI.getStructTypeOrNull()) {
4298 unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
4299 if (Field) {
4300 // N = N + Offset
4303
4304 // In an inbounds GEP with an offset that is nonnegative even when
4305 // interpreted as signed, assume there is no unsigned overflow.
4307 if (int64_t(Offset) >= 0 && cast<GEPOperator>(I).isInBounds())
4308 Flags.setNoUnsignedWrap(true);
4309
4310 N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N,
4311 DAG.getConstant(Offset, dl, N.getValueType()), Flags);
4312 }
4313 } else {
4314 // IdxSize is the width of the arithmetic according to IR semantics.
4315 // In SelectionDAG, we may prefer to do arithmetic in a wider bitwidth
4316 // (and fix up the result later).
4317 unsigned IdxSize = DAG.getDataLayout().getIndexSizeInBits(AS);
4318 MVT IdxTy = MVT::getIntegerVT(IdxSize);
4319 TypeSize ElementSize =
4320 GTI.getSequentialElementStride(DAG.getDataLayout());
4321 // We intentionally mask away the high bits here; ElementSize may not
4322 // fit in IdxTy.
4323 APInt ElementMul(IdxSize, ElementSize.getKnownMinValue());
4324 bool ElementScalable = ElementSize.isScalable();
4325
4326 // If this is a scalar constant or a splat vector of constants,
4327 // handle it quickly.
4328 const auto *C = dyn_cast<Constant>(Idx);
4329 if (C && isa<VectorType>(C->getType()))
4330 C = C->getSplatValue();
4331
4332 const auto *CI = dyn_cast_or_null<ConstantInt>(C);
4333 if (CI && CI->isZero())
4334 continue;
4335 if (CI && !ElementScalable) {
4336 APInt Offs = ElementMul * CI->getValue().sextOrTrunc(IdxSize);
4338 SDValue OffsVal;
4339 if (IsVectorGEP)
4340 OffsVal = DAG.getConstant(
4341 Offs, dl, EVT::getVectorVT(Context, IdxTy, VectorElementCount));
4342 else
4343 OffsVal = DAG.getConstant(Offs, dl, IdxTy);
4344
4345 // In an inbounds GEP with an offset that is nonnegative even when
4346 // interpreted as signed, assume there is no unsigned overflow.
4348 if (Offs.isNonNegative() && cast<GEPOperator>(I).isInBounds())
4349 Flags.setNoUnsignedWrap(true);
4350
4351 OffsVal = DAG.getSExtOrTrunc(OffsVal, dl, N.getValueType());
4352
4353 N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, OffsVal, Flags);
4354 continue;
4355 }
4356
4357 // N = N + Idx * ElementMul;
4358 SDValue IdxN = getValue(Idx);
4359
4360 if (!IdxN.getValueType().isVector() && IsVectorGEP) {
4362 VectorElementCount);
4363 IdxN = DAG.getSplat(VT, dl, IdxN);
4364 }
4365
4366 // If the index is smaller or larger than intptr_t, truncate or extend
4367 // it.
4368 IdxN = DAG.getSExtOrTrunc(IdxN, dl, N.getValueType());
4369
4370 if (ElementScalable) {
4371 EVT VScaleTy = N.getValueType().getScalarType();
4372 SDValue VScale = DAG.getNode(
4373 ISD::VSCALE, dl, VScaleTy,
4374 DAG.getConstant(ElementMul.getZExtValue(), dl, VScaleTy));
4375 if (IsVectorGEP)
4376 VScale = DAG.getSplatVector(N.getValueType(), dl, VScale);
4377 IdxN = DAG.getNode(ISD::MUL, dl, N.getValueType(), IdxN, VScale);
4378 } else {
4379 // If this is a multiply by a power of two, turn it into a shl
4380 // immediately. This is a very common case.
4381 if (ElementMul != 1) {
4382 if (ElementMul.isPowerOf2()) {
4383 unsigned Amt = ElementMul.logBase2();
4384 IdxN = DAG.getNode(ISD::SHL, dl,
4385 N.getValueType(), IdxN,
4386 DAG.getConstant(Amt, dl, IdxN.getValueType()));
4387 } else {
4388 SDValue Scale = DAG.getConstant(ElementMul.getZExtValue(), dl,
4389 IdxN.getValueType());
4390 IdxN = DAG.getNode(ISD::MUL, dl,
4391 N.getValueType(), IdxN, Scale);
4392 }
4393 }
4394 }
4395
4396 N = DAG.getNode(ISD::ADD, dl,
4397 N.getValueType(), N, IdxN);
4398 }
4399 }
4400
4401 MVT PtrTy = TLI.getPointerTy(DAG.getDataLayout(), AS);
4402 MVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout(), AS);
4403 if (IsVectorGEP) {
4404 PtrTy = MVT::getVectorVT(PtrTy, VectorElementCount);
4405 PtrMemTy = MVT::getVectorVT(PtrMemTy, VectorElementCount);
4406 }
4407
4408 if (PtrMemTy != PtrTy && !cast<GEPOperator>(I).isInBounds())
4409 N = DAG.getPtrExtendInReg(N, dl, PtrMemTy);
4410
4411 setValue(&I, N);
4412}
4413
4414void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
4415 // If this is a fixed sized alloca in the entry block of the function,
4416 // allocate it statically on the stack.
4417 if (FuncInfo.StaticAllocaMap.count(&I))
4418 return; // getValue will auto-populate this.
4419
4420 SDLoc dl = getCurSDLoc();
4421 Type *Ty = I.getAllocatedType();
4423 auto &DL = DAG.getDataLayout();
4424 TypeSize TySize = DL.getTypeAllocSize(Ty);
4425 MaybeAlign Alignment = std::max(DL.getPrefTypeAlign(Ty), I.getAlign());
4426
4427 SDValue AllocSize = getValue(I.getArraySize());
4428
4429 EVT IntPtr = TLI.getPointerTy(DL, I.getAddressSpace());
4430 if (AllocSize.getValueType() != IntPtr)
4431 AllocSize = DAG.getZExtOrTrunc(AllocSize, dl, IntPtr);
4432
4433 if (TySize.isScalable())
4434 AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr, AllocSize,
4435 DAG.getVScale(dl, IntPtr,
4436 APInt(IntPtr.getScalarSizeInBits(),
4437 TySize.getKnownMinValue())));
4438 else {
4439 SDValue TySizeValue =
4441 AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr, AllocSize,
4442 DAG.getZExtOrTrunc(TySizeValue, dl, IntPtr));
4443 }
4444
4445 // Handle alignment. If the requested alignment is less than or equal to
4446 // the stack alignment, ignore it. If the size is greater than or equal to
4447 // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
4449 if (*Alignment <= StackAlign)
4450 Alignment = std::nullopt;
4451
4452 const uint64_t StackAlignMask = StackAlign.value() - 1U;
4453 // Round the size of the allocation up to the stack alignment size
4454 // by add SA-1 to the size. This doesn't overflow because we're computing
4455 // an address inside an alloca.
4457 Flags.setNoUnsignedWrap(true);
4458 AllocSize = DAG.getNode(ISD::ADD, dl, AllocSize.getValueType(), AllocSize,
4459 DAG.getConstant(StackAlignMask, dl, IntPtr), Flags);
4460
4461 // Mask out the low bits for alignment purposes.
4462 AllocSize = DAG.getNode(ISD::AND, dl, AllocSize.getValueType(), AllocSize,
4463 DAG.getConstant(~StackAlignMask, dl, IntPtr));
4464
4465 SDValue Ops[] = {
4466 getRoot(), AllocSize,
4467 DAG.getConstant(Alignment ? Alignment->value() : 0, dl, IntPtr)};
4468 SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);
4469 SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, dl, VTs, Ops);
4470 setValue(&I, DSA);
4471 DAG.setRoot(DSA.getValue(1));
4472
4474}
4475
4476static const MDNode *getRangeMetadata(const Instruction &I) {
4477 // If !noundef is not present, then !range violation results in a poison
4478 // value rather than immediate undefined behavior. In theory, transferring
4479 // these annotations to SDAG is fine, but in practice there are key SDAG
4480 // transforms that are known not to be poison-safe, such as folding logical
4481 // and/or to bitwise and/or. For now, only transfer !range if !noundef is
4482 // also present.
4483 if (!I.hasMetadata(LLVMContext::MD_noundef))
4484 return nullptr;
4485 return I.getMetadata(LLVMContext::MD_range);
4486}
4487
4488void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
4489 if (I.isAtomic())
4490 return visitAtomicLoad(I);
4491
4493 const Value *SV = I.getOperand(0);
4494 if (TLI.supportSwiftError()) {
4495 // Swifterror values can come from either a function parameter with
4496 // swifterror attribute or an alloca with swifterror attribute.
4497 if (const Argument *Arg = dyn_cast<Argument>(SV)) {
4498 if (Arg->hasSwiftErrorAttr())
4499 return visitLoadFromSwiftError(I);
4500 }
4501
4502 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) {
4503 if (Alloca->isSwiftError())
4504 return visitLoadFromSwiftError(I);
4505 }
4506 }
4507
4508 SDValue Ptr = getValue(SV);
4509
4510 Type *Ty = I.getType();
4511 SmallVector<EVT, 4> ValueVTs, MemVTs;
4513 ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &MemVTs, &Offsets);
4514 unsigned NumValues = ValueVTs.size();
4515 if (NumValues == 0)
4516 return;
4517
4518 Align Alignment = I.getAlign();
4519 AAMDNodes AAInfo = I.getAAMetadata();
4520 const MDNode *Ranges = getRangeMetadata(I);
4521 bool isVolatile = I.isVolatile();
4522 MachineMemOperand::Flags MMOFlags =
4524
4525 SDValue Root;
4526 bool ConstantMemory = false;
4527 if (isVolatile)
4528 // Serialize volatile loads with other side effects.
4529 Root = getRoot();
4530 else if (NumValues > MaxParallelChains)
4531 Root = getMemoryRoot();
4532 else if (AA &&
4534 SV,
4536 AAInfo))) {
4537 // Do not serialize (non-volatile) loads of constant memory with anything.
4538 Root = DAG.getEntryNode();
4539 ConstantMemory = true;
4541 } else {
4542 // Do not serialize non-volatile loads against each other.
4543 Root = DAG.getRoot();
4544 }
4545
4546 SDLoc dl = getCurSDLoc();
4547
4548 if (isVolatile)
4549 Root = TLI.prepareVolatileOrAtomicLoad(Root, dl, DAG);
4550
4551 SmallVector<SDValue, 4> Values(NumValues);
4552 SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
4553
4554 unsigned ChainI = 0;
4555 for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
4556 // Serializing loads here may result in excessive register pressure, and
4557 // TokenFactor places arbitrary choke points on the scheduler. SD scheduling
4558 // could recover a bit by hoisting nodes upward in the chain by recognizing
4559 // they are side-effect free or do not alias. The optimizer should really
4560 // avoid this case by converting large object/array copies to llvm.memcpy
4561 // (MaxParallelChains should always remain as failsafe).
4562 if (ChainI == MaxParallelChains) {
4563 assert(PendingLoads.empty() && "PendingLoads must be serialized first");
4564 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4565 ArrayRef(Chains.data(), ChainI));
4566 Root = Chain;
4567 ChainI = 0;
4568 }
4569
4570 // TODO: MachinePointerInfo only supports a fixed length offset.
4571 MachinePointerInfo PtrInfo =
4572 !Offsets[i].isScalable() || Offsets[i].isZero()
4573 ? MachinePointerInfo(SV, Offsets[i].getKnownMinValue())
4575
4576 SDValue A = DAG.getObjectPtrOffset(dl, Ptr, Offsets[i]);
4577 SDValue L = DAG.getLoad(MemVTs[i], dl, Root, A, PtrInfo, Alignment,
4578 MMOFlags, AAInfo, Ranges);
4579 Chains[ChainI] = L.getValue(1);
4580
4581 if (MemVTs[i] != ValueVTs[i])
4582 L = DAG.getPtrExtOrTrunc(L, dl, ValueVTs[i]);
4583
4584 Values[i] = L;
4585 }
4586
4587 if (!ConstantMemory) {
4588 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4589 ArrayRef(Chains.data(), ChainI));
4590 if (isVolatile)
4591 DAG.setRoot(Chain);
4592 else
4593 PendingLoads.push_back(Chain);
4594 }
4595
4597 DAG.getVTList(ValueVTs), Values));
4598}
4599
4600void SelectionDAGBuilder::visitStoreToSwiftError(const StoreInst &I) {
4602 "call visitStoreToSwiftError when backend supports swifterror");
4603
4604 SmallVector<EVT, 4> ValueVTs;
4606 const Value *SrcV = I.getOperand(0);
4608 SrcV->getType(), ValueVTs, &Offsets, 0);
4609 assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
4610 "expect a single EVT for swifterror");
4611
4612 SDValue Src = getValue(SrcV);
4613 // Create a virtual register, then update the virtual register.
4614 Register VReg =
4615 SwiftError.getOrCreateVRegDefAt(&I, FuncInfo.MBB, I.getPointerOperand());
4616 // Chain, DL, Reg, N or Chain, DL, Reg, N, Glue
4617 // Chain can be getRoot or getControlRoot.
4618 SDValue CopyNode = DAG.getCopyToReg(getRoot(), getCurSDLoc(), VReg,
4619 SDValue(Src.getNode(), Src.getResNo()));
4620 DAG.setRoot(CopyNode);
4621}
4622
4623void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) {
4625 "call visitLoadFromSwiftError when backend supports swifterror");
4626
4627 assert(!I.isVolatile() &&
4628 !I.hasMetadata(LLVMContext::MD_nontemporal) &&
4629 !I.hasMetadata(LLVMContext::MD_invariant_load) &&
4630 "Support volatile, non temporal, invariant for load_from_swift_error");
4631
4632 const Value *SV = I.getOperand(0);
4633 Type *Ty = I.getType();
4634 assert(
4635 (!AA ||
4638 I.getAAMetadata()))) &&
4639 "load_from_swift_error should not be constant memory");
4640
4641 SmallVector<EVT, 4> ValueVTs;
4644 ValueVTs, &Offsets, 0);
4645 assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
4646 "expect a single EVT for swifterror");
4647
4648 // Chain, DL, Reg, VT, Glue or Chain, DL, Reg, VT
4650 getRoot(), getCurSDLoc(),
4651 SwiftError.getOrCreateVRegUseAt(&I, FuncInfo.MBB, SV), ValueVTs[0]);
4652
4653 setValue(&I, L);
4654}
4655
4656void SelectionDAGBuilder::visitStore(const StoreInst &I) {
4657 if (I.isAtomic())
4658 return visitAtomicStore(I);
4659
4660 const Value *SrcV = I.getOperand(0);
4661 const Value *PtrV = I.getOperand(1);
4662
4664 if (TLI.supportSwiftError()) {
4665 // Swifterror values can come from either a function parameter with
4666 // swifterror attribute or an alloca with swifterror attribute.
4667 if (const Argument *Arg = dyn_cast<Argument>(PtrV)) {
4668 if (Arg->hasSwiftErrorAttr())
4669 return visitStoreToSwiftError(I);
4670 }
4671
4672 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) {
4673 if (Alloca->isSwiftError())
4674 return visitStoreToSwiftError(I);
4675 }
4676 }
4677
4678 SmallVector<EVT, 4> ValueVTs, MemVTs;
4681 SrcV->getType(), ValueVTs, &MemVTs, &Offsets);
4682 unsigned NumValues = ValueVTs.size();
4683 if (NumValues == 0)
4684 return;
4685
4686 // Get the lowered operands. Note that we do this after
4687 // checking if NumResults is zero, because with zero results
4688 // the operands won't have values in the map.
4689 SDValue Src = getValue(SrcV);
4690 SDValue Ptr = getValue(PtrV);
4691
4692 SDValue Root = I.isVolatile() ? getRoot() : getMemoryRoot();
4693 SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
4694 SDLoc dl = getCurSDLoc();
4695 Align Alignment = I.getAlign();
4696 AAMDNodes AAInfo = I.getAAMetadata();
4697
4698 auto MMOFlags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout());
4699
4700 unsigned ChainI = 0;
4701 for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
4702 // See visitLoad comments.
4703 if (ChainI == MaxParallelChains) {
4704 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4705 ArrayRef(Chains.data(), ChainI));
4706 Root = Chain;
4707 ChainI = 0;
4708 }
4709
4710 // TODO: MachinePointerInfo only supports a fixed length offset.
4711 MachinePointerInfo PtrInfo =
4712 !Offsets[i].isScalable() || Offsets[i].isZero()
4713 ? MachinePointerInfo(PtrV, Offsets[i].getKnownMinValue())
4715
4716 SDValue Add = DAG.getObjectPtrOffset(dl, Ptr, Offsets[i]);
4717 SDValue Val = SDValue(Src.getNode(), Src.getResNo() + i);
4718 if (MemVTs[i] != ValueVTs[i])
4719 Val = DAG.getPtrExtOrTrunc(Val, dl, MemVTs[i]);
4720 SDValue St =
4721 DAG.getStore(Root, dl, Val, Add, PtrInfo, Alignment, MMOFlags, AAInfo);
4722 Chains[ChainI] = St;
4723 }
4724
4725 SDValue StoreNode = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4726 ArrayRef(Chains.data(), ChainI));
4727 setValue(&I, StoreNode);
4728 DAG.setRoot(StoreNode);
4729}
4730
4731void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
4732 bool IsCompressing) {
4733 SDLoc sdl = getCurSDLoc();
4734
4735 auto getMaskedStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
4736 Align &Alignment) {
4737 // llvm.masked.store.*(Src0, Ptr, alignment, Mask)
4738 Src0 = I.getArgOperand(0);
4739 Ptr = I.getArgOperand(1);
4740 Alignment = cast<ConstantInt>(I.getArgOperand(2))->getAlignValue();
4741 Mask = I.getArgOperand(3);
4742 };
4743 auto getCompressingStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
4744 Align &Alignment) {
4745 // llvm.masked.compressstore.*(Src0, Ptr, Mask)
4746 Src0 = I.getArgOperand(0);
4747 Ptr = I.getArgOperand(1);
4748 Mask = I.getArgOperand(2);
4749 Alignment = I.getParamAlign(1).valueOrOne();
4750 };
4751
4752 Value *PtrOperand, *MaskOperand, *Src0Operand;
4753 Align Alignment;
4754 if (IsCompressing)
4755 getCompressingStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4756 else
4757 getMaskedStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4758
4759 SDValue Ptr = getValue(PtrOperand);
4760 SDValue Src0 = getValue(Src0Operand);
4761 SDValue Mask = getValue(MaskOperand);
4762 SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
4763
4764 EVT VT = Src0.getValueType();
4765
4766 auto MMOFlags = MachineMemOperand::MOStore;
4767 if (I.hasMetadata(LLVMContext::MD_nontemporal))
4769
4771 MachinePointerInfo(PtrOperand), MMOFlags,
4772 LocationSize::beforeOrAfterPointer(), Alignment, I.getAAMetadata());
4773 SDValue StoreNode =
4774 DAG.getMaskedStore(getMemoryRoot(), sdl, Src0, Ptr, Offset, Mask, VT, MMO,
4775 ISD::UNINDEXED, false /* Truncating */, IsCompressing);
4776 DAG.setRoot(StoreNode);
4777 setValue(&I, StoreNode);
4778}
4779
4780// Get a uniform base for the Gather/Scatter intrinsic.
4781// The first argument of the Gather/Scatter intrinsic is a vector of pointers.
4782// We try to represent it as a base pointer + vector of indices.
4783// Usually, the vector of pointers comes from a 'getelementptr' instruction.
4784// The first operand of the GEP may be a single pointer or a vector of pointers
4785// Example:
4786// %gep.ptr = getelementptr i32, <8 x i32*> %vptr, <8 x i32> %ind
4787// or
4788// %gep.ptr = getelementptr i32, i32* %ptr, <8 x i32> %ind
4789// %res = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.ptr, ..
4790//
4791// When the first GEP operand is a single pointer - it is the uniform base we
4792// are looking for. If first operand of the GEP is a splat vector - we
4793// extract the splat value and use it as a uniform base.
4794// In all other cases the function returns 'false'.
4796 ISD::MemIndexType &IndexType, SDValue &Scale,
4797 SelectionDAGBuilder *SDB, const BasicBlock *CurBB,
4798 uint64_t ElemSize) {
4799 SelectionDAG& DAG = SDB->DAG;
4800 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4801 const DataLayout &DL = DAG.getDataLayout();
4802
4803 assert(Ptr->getType()->isVectorTy() && "Unexpected pointer type");
4804
4805 // Handle splat constant pointer.
4806 if (auto *C = dyn_cast<Constant>(Ptr)) {
4807 C = C->getSplatValue();
4808 if (!C)
4809 return false;
4810
4811 Base = SDB->getValue(C);
4812
4813 ElementCount NumElts = cast<VectorType>(Ptr->getType())->getElementCount();
4814 EVT VT = EVT::getVectorVT(*DAG.getContext(), TLI.getPointerTy(DL), NumElts);
4815 Index = DAG.getConstant(0, SDB->getCurSDLoc(), VT);
4816 IndexType = ISD::SIGNED_SCALED;
4817 Scale = DAG.getTargetConstant(1, SDB->getCurSDLoc(), TLI.getPointerTy(DL));
4818 return true;
4819 }
4820
4821 const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Ptr);
4822 if (!GEP || GEP->getParent() != CurBB)
4823 return false;
4824
4825 if (GEP->getNumOperands() != 2)
4826 return false;
4827
4828 const Value *BasePtr = GEP->getPointerOperand();
4829 const Value *IndexVal = GEP->getOperand(GEP->getNumOperands() - 1);
4830
4831 // Make sure the base is scalar and the index is a vector.
4832 if (BasePtr->getType()->isVectorTy() || !IndexVal->getType()->isVectorTy())
4833 return false;
4834
4835 TypeSize ScaleVal = DL.getTypeAllocSize(GEP->getResultElementType());
4836 if (ScaleVal.isScalable())
4837 return false;
4838
4839 // Target may not support the required addressing mode.
4840 if (ScaleVal != 1 &&
4841 !TLI.isLegalScaleForGatherScatter(ScaleVal.getFixedValue(), ElemSize))
4842 return false;
4843
4844 Base = SDB->getValue(BasePtr);
4845 Index = SDB->getValue(IndexVal);
4846 IndexType = ISD::SIGNED_SCALED;
4847
4848 Scale =
4849 DAG.getTargetConstant(ScaleVal, SDB->getCurSDLoc(), TLI.getPointerTy(DL));
4850 return true;
4851}
4852
4853void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
4854 SDLoc sdl = getCurSDLoc();
4855
4856 // llvm.masked.scatter.*(Src0, Ptrs, alignment, Mask)
4857 const Value *Ptr = I.getArgOperand(1);
4858 SDValue Src0 = getValue(I.getArgOperand(0));
4859 SDValue Mask = getValue(I.getArgOperand(3));
4860 EVT VT = Src0.getValueType();
4861 Align Alignment = cast<ConstantInt>(I.getArgOperand(2))
4862 ->getMaybeAlignValue()
4863 .value_or(DAG.getEVTAlign(VT.getScalarType()));
4865
4866 SDValue Base;
4867 SDValue Index;
4868 ISD::MemIndexType IndexType;
4869 SDValue Scale;
4870 bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, this,
4871 I.getParent(), VT.getScalarStoreSize());
4872
4873 unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
4876 LocationSize::beforeOrAfterPointer(), Alignment, I.getAAMetadata());
4877 if (!UniformBase) {
4879 Index = getValue(Ptr);
4880 IndexType = ISD::SIGNED_SCALED;
4881 Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4882 }
4883
4884 EVT IdxVT = Index.getValueType();
4885 EVT EltTy = IdxVT.getVectorElementType();
4886 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
4887 EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
4888 Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index);
4889 }
4890
4891 SDValue Ops[] = { getMemoryRoot(), Src0, Mask, Base, Index, Scale };
4892 SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), VT, sdl,
4893 Ops, MMO, IndexType, false);
4894 DAG.setRoot(Scatter);
4895 setValue(&I, Scatter);
4896}
4897
4898void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
4899 SDLoc sdl = getCurSDLoc();
4900
4901 auto getMaskedLoadOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
4902 Align &Alignment) {
4903 // @llvm.masked.load.*(Ptr, alignment, Mask, Src0)
4904 Ptr = I.getArgOperand(0);
4905 Alignment = cast<ConstantInt>(I.getArgOperand(1))->getAlignValue();
4906 Mask = I.getArgOperand(2);
4907 Src0 = I.getArgOperand(3);
4908 };
4909 auto getExpandingLoadOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
4910 Align &Alignment) {
4911 // @llvm.masked.expandload.*(Ptr, Mask, Src0)
4912 Ptr = I.getArgOperand(0);
4913 Alignment = I.getParamAlign(0).valueOrOne();
4914 Mask = I.getArgOperand(1);
4915 Src0 = I.getArgOperand(2);
4916 };
4917
4918 Value *PtrOperand, *MaskOperand, *Src0Operand;
4919 Align Alignment;
4920 if (IsExpanding)
4921 getExpandingLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4922 else
4923 getMaskedLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4924
4925 SDValue Ptr = getValue(PtrOperand);
4926 SDValue Src0 = getValue(Src0Operand);
4927 SDValue Mask = getValue(MaskOperand);
4928 SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
4929
4930 EVT VT = Src0.getValueType();
4931 AAMDNodes AAInfo = I.getAAMetadata();
4932 const MDNode *Ranges = getRangeMetadata(I);
4933
4934 // Do not serialize masked loads of constant memory with anything.
4935 MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
4936 bool AddToChain = !AA || !AA->pointsToConstantMemory(ML);
4937
4938 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
4939
4940 auto MMOFlags = MachineMemOperand::MOLoad;
4941 if (I.hasMetadata(LLVMContext::MD_nontemporal))
4943
4945 MachinePointerInfo(PtrOperand), MMOFlags,
4946 LocationSize::beforeOrAfterPointer(), Alignment, AAInfo, Ranges);
4947
4948 SDValue Load =
4949 DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Offset, Mask, Src0, VT, MMO,
4950 ISD::UNINDEXED, ISD::NON_EXTLOAD, IsExpanding);
4951 if (AddToChain)
4952 PendingLoads.push_back(Load.getValue(1));
4953 setValue(&I, Load);
4954}
4955
4956void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
4957 SDLoc sdl = getCurSDLoc();
4958
4959 // @llvm.masked.gather.*(Ptrs, alignment, Mask, Src0)
4960 const Value *Ptr = I.getArgOperand(0);
4961 SDValue Src0 = getValue(I.getArgOperand(3));
4962 SDValue Mask = getValue(I.getArgOperand(2));
4963
4965 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4966 Align Alignment = cast<ConstantInt>(I.getArgOperand(1))
4967 ->getMaybeAlignValue()
4968 .value_or(DAG.getEVTAlign(VT.getScalarType()));
4969
4970 const MDNode *Ranges = getRangeMetadata(I);
4971
4972 SDValue Root = DAG.getRoot();
4973 SDValue Base;
4974 SDValue Index;
4975 ISD::MemIndexType IndexType;
4976 SDValue Scale;
4977 bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, this,
4978 I.getParent(), VT.getScalarStoreSize());
4979 unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
4982 LocationSize::beforeOrAfterPointer(), Alignment, I.getAAMetadata(),
4983 Ranges);
4984
4985 if (!UniformBase) {
4987 Index = getValue(Ptr);
4988 IndexType = ISD::SIGNED_SCALED;
4989 Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4990 }
4991
4992 EVT IdxVT = Index.getValueType();
4993 EVT EltTy = IdxVT.getVectorElementType();
4994 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
4995 EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
4996 Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index);
4997 }
4998
4999 SDValue Ops[] = { Root, Src0, Mask, Base, Index, Scale };
5000 SDValue Gather = DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl,
5001 Ops, MMO, IndexType, ISD::NON_EXTLOAD);
5002
5003 PendingLoads.push_back(Gather.getValue(1));
5004 setValue(&I, Gather);
5005}
5006
5007void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
5008 SDLoc dl = getCurSDLoc();
5009 AtomicOrdering SuccessOrdering = I.getSuccessOrdering();
5010 AtomicOrdering FailureOrdering = I.getFailureOrdering();
5011 SyncScope::ID SSID = I.getSyncScopeID();
5012
5013 SDValue InChain = getRoot();
5014
5015 MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType();
5016 SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other);
5017
5020
5023 MachinePointerInfo(I.getPointerOperand()), Flags,
5025 AAMDNodes(), nullptr, SSID, SuccessOrdering, FailureOrdering);
5026
5028 dl, MemVT, VTs, InChain,
5029 getValue(I.getPointerOperand()),
5030 getValue(I.getCompareOperand()),
5031 getValue(I.getNewValOperand()), MMO);
5032
5033 SDValue OutChain = L.getValue(2);
5034
5035 setValue(&I, L);
5036 DAG.setRoot(OutChain);
5037}
5038
5039void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
5040 SDLoc dl = getCurSDLoc();
5042 switch (I.getOperation()) {
5043 default: llvm_unreachable("Unknown atomicrmw operation");
5061 break;
5064 break;
5065 }
5066 AtomicOrdering Ordering = I.getOrdering();
5067 SyncScope::ID SSID = I.getSyncScopeID();
5068
5069 SDValue InChain = getRoot();
5070
5071 auto MemVT = getValue(I.getValOperand()).getSimpleValueType();
5074
5077 MachinePointerInfo(I.getPointerOperand()), Flags,
5079 AAMDNodes(), nullptr, SSID, Ordering);
5080
5081 SDValue L =
5082 DAG.getAtomic(NT, dl, MemVT, InChain,
5083 getValue(I.getPointerOperand()), getValue(I.getValOperand()),
5084 MMO);
5085
5086 SDValue OutChain = L.getValue(1);
5087
5088 setValue(&I, L);
5089 DAG.setRoot(OutChain);
5090}
5091
5092void SelectionDAGBuilder::visitFence(const FenceInst &I) {
5093 SDLoc dl = getCurSDLoc();
5095 SDValue Ops[3];
5096 Ops[0] = getRoot();
5097 Ops[1] = DAG.getTargetConstant((unsigned)I.getOrdering(), dl,
5099 Ops[2] = DAG.getTargetConstant(I.getSyncScopeID(), dl,
5101 SDValue N = DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops);
5102 setValue(&I, N);
5103 DAG.setRoot(N);
5104}
5105
5106void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
5107 SDLoc dl = getCurSDLoc();
5108 AtomicOrdering Order = I.getOrdering();
5109 SyncScope::ID SSID = I.getSyncScopeID();
5110
5111 SDValue InChain = getRoot();
5112
5114 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
5115 EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());
5116
5117 if (!TLI.supportsUnalignedAtomics() &&
5118 I.getAlign().value() < MemVT.getSizeInBits() / 8)
5119 report_fatal_error("Cannot generate unaligned atomic load");
5120
5122
5124 MachinePointerInfo(I.getPointerOperand()), Flags,
5125 LocationSize::precise(MemVT.getStoreSize()), I.getAlign(), AAMDNodes(),
5126 nullptr, SSID, Order);
5127
5128 InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);
5129
5130 SDValue Ptr = getValue(I.getPointerOperand());
5131 SDValue L = DAG.getAtomic(ISD::ATOMIC_LOAD, dl, MemVT, MemVT, InChain,
5132 Ptr, MMO);
5133
5134 SDValue OutChain = L.getValue(1);
5135 if (MemVT != VT)
5136 L = DAG.getPtrExtOrTrunc(L, dl, VT);
5137
5138 setValue(&I, L);
5139 DAG.setRoot(OutChain);
5140}
5141
5142void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
5143 SDLoc dl = getCurSDLoc();
5144
5145 AtomicOrdering Ordering = I.getOrdering();
5146 SyncScope::ID SSID = I.getSyncScopeID();
5147
5148 SDValue InChain = getRoot();
5149
5151 EVT MemVT =
5152 TLI.getMemValueType(DAG.getDataLayout(), I.getValueOperand()->getType());
5153
5154 if (!TLI.supportsUnalignedAtomics() &&
5155 I.getAlign().value() < MemVT.getSizeInBits() / 8)
5156 report_fatal_error("Cannot generate unaligned atomic store");
5157
5159
5162 MachinePointerInfo(I.getPointerOperand()), Flags,
5163 LocationSize::precise(MemVT.getStoreSize()), I.getAlign(), AAMDNodes(),
5164 nullptr, SSID, Ordering);
5165
5166 SDValue Val = getValue(I.getValueOperand());
5167 if (Val.getValueType() != MemVT)
5168 Val = DAG.getPtrExtOrTrunc(Val, dl, MemVT);
5169 SDValue Ptr = getValue(I.getPointerOperand());
5170
5171 SDValue OutChain =
5172 DAG.getAtomic(ISD::ATOMIC_STORE, dl, MemVT, InChain, Val, Ptr, MMO);
5173
5174 setValue(&I, OutChain);
5175 DAG.setRoot(OutChain);
5176}
5177
5178/// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
5179/// node.
5180void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
5181 unsigned Intrinsic) {
5182 // Ignore the callsite's attributes. A specific call site may be marked with
5183 // readnone, but the lowering code will expect the chain based on the
5184 // definition.
5185 const Function *F = I.getCalledFunction();
5186 bool HasChain = !F->doesNotAccessMemory();
5187 bool OnlyLoad = HasChain && F->onlyReadsMemory();
5188
5189 // Build the operand list.
5191 if (HasChain) { // If this intrinsic has side-effects, chainify it.
5192 if (OnlyLoad) {
5193 // We don't need to serialize loads against other loads.
5194 Ops.push_back(DAG.getRoot());
5195 } else {
5196 Ops.push_back(getRoot());
5197 }
5198 }
5199
5200 // Info is set by getTgtMemIntrinsic
5203 bool IsTgtIntrinsic = TLI.getTgtMemIntrinsic(Info, I,
5205 Intrinsic);
5206
5207 // Add the intrinsic ID as an integer operand if it's not a target intrinsic.
5208 if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID ||
5210 Ops.push_back(DAG.getTargetConstant(Intrinsic, getCurSDLoc(),
5212
5213 // Add all operands of the call to the operand list.
5214 for (unsigned i = 0, e = I.arg_size(); i != e; ++i) {
5215 const Value *Arg = I.getArgOperand(i);
5216 if (!I.paramHasAttr(i, Attribute::ImmArg)) {
5217 Ops.push_back(getValue(Arg));
5218 continue;
5219 }
5220
5221 // Use TargetConstant instead of a regular constant for immarg.
5222 EVT VT = TLI.getValueType(DAG.getDataLayout(), Arg->getType(), true);
5223 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Arg)) {
5224 assert(CI->getBitWidth() <= 64 &&
5225 "large intrinsic immediates not handled");
5226 Ops.push_back(DAG.getTargetConstant(*CI, SDLoc(), VT));
5227 } else {
5228 Ops.push_back(
5229 DAG.getTargetConstantFP(*cast<ConstantFP>(Arg), SDLoc(), VT));
5230 }
5231 }
5232
5233 SmallVector<EVT, 4> ValueVTs;
5234 ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);
5235
5236 if (HasChain)
5237 ValueVTs.push_back(MVT::Other);
5238
5239 SDVTList VTs = DAG.getVTList(ValueVTs);
5240
5241 // Propagate fast-math-flags from IR to node(s).
5243 if (auto *FPMO = dyn_cast<FPMathOperator>(&I))
5244 Flags.copyFMF(*FPMO);
5245 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
5246
5247 // Create the node.
5249
5250 if (auto Bundle = I.getOperandBundle(LLVMContext::OB_convergencectrl)) {
5251 auto *Token = Bundle->Inputs[0].get();
5252 SDValue ConvControlToken = getValue(Token);
5253 assert(Ops.back().getValueType() != MVT::Glue &&
5254 "Did not expected another glue node here.");
5255 ConvControlToken =
5256 DAG.getNode(ISD::CONVERGENCECTRL_GLUE, {}, MVT::Glue, ConvControlToken);
5257 Ops.push_back(ConvControlToken);
5258 }
5259
5260 // In some cases, custom collection of operands from CallInst I may be needed.
5262 if (IsTgtIntrinsic) {
5263 // This is target intrinsic that touches memory
5264 //
5265 // TODO: We currently just fallback to address space 0 if getTgtMemIntrinsic
5266 // didn't yield anything useful.
5268 if (Info.ptrVal)
5269 MPI = MachinePointerInfo(Info.ptrVal, Info.offset);
5270 else if (Info.fallbackAddressSpace)
5271 MPI = MachinePointerInfo(*Info.fallbackAddressSpace);
5272 Result = DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(), VTs, Ops,
5273 Info.memVT, MPI, Info.align, Info.flags,
5274 Info.size, I.getAAMetadata());
5275 } else if (!HasChain) {
5277 } else if (!I.getType()->isVoidTy()) {
5279 } else {
5281 }
5282
5283 if (HasChain) {
5284 SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1);
5285 if (OnlyLoad)
5286 PendingLoads.push_back(Chain);
5287 else
5288 DAG.setRoot(Chain);
5289 }
5290
5291 if (!I.getType()->isVoidTy()) {
5292 if (!isa<VectorType>(I.getType()))
5293 Result = lowerRangeToAssertZExt(DAG, I, Result);
5294
5295 MaybeAlign Alignment = I.getRetAlign();
5296
5297 // Insert `assertalign` node if there's an alignment.
5298 if (InsertAssertAlign && Alignment) {
5299 Result =
5300 DAG.getAssertAlign(getCurSDLoc(), Result, Alignment.valueOrOne());
5301 }
5302 }
5303
5304 setValue(&I, Result);
5305}
5306
5307/// GetSignificand - Get the significand and build it into a floating-point
5308/// number with exponent of 1:
5309///
5310/// Op = (Op & 0x007fffff) | 0x3f800000;
5311///
5312/// where Op is the hexadecimal representation of floating point value.
5314 SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
5315 DAG.getConstant(0x007fffff, dl, MVT::i32));
5316 SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,
5317 DAG.getConstant(0x3f800000, dl, MVT::i32));
5318 return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2);
5319}
5320
5321/// GetExponent - Get the exponent:
5322///
5323/// (float)(int)(((Op & 0x7f800000) >> 23) - 127);
5324///
5325/// where Op is the hexadecimal representation of floating point value.
5327 const TargetLowering &TLI, const SDLoc &dl) {
5328 SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
5329 DAG.getConstant(0x7f800000, dl, MVT::i32));
5330 SDValue t1 = DAG.getNode(
5331 ISD::SRL, dl, MVT::i32, t0,
5332 DAG.getConstant(23, dl,
5333 TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout())));
5334 SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,
5335 DAG.getConstant(127, dl, MVT::i32));
5336 return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);
5337}
5338
5339/// getF32Constant - Get 32-bit floating point constant.
5341 const SDLoc &dl) {
5342 return DAG.getConstantFP(APFloat(APFloat::IEEEsingle(), APInt(32, Flt)), dl,
5343 MVT::f32);
5344}
5345
5347 SelectionDAG &DAG) {
5348 // TODO: What fast-math-flags should be set on the floating-point nodes?
5349
5350 // IntegerPartOfX = ((int32_t)(t0);
5351 SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);
5352
5353 // FractionalPartOfX = t0 - (float)IntegerPartOfX;
5354 SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
5355 SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
5356
5357 // IntegerPartOfX <<= 23;
5358 IntegerPartOfX =
5359 DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX,
5360 DAG.getConstant(23, dl,
5362 MVT::i32, DAG.getDataLayout())));
5363
5364 SDValue TwoToFractionalPartOfX;
5365 if (LimitFloatPrecision <= 6) {
5366 // For floating-point precision of 6:
5367 //
5368 // TwoToFractionalPartOfX =
5369 // 0.997535578f +
5370 // (0.735607626f + 0.252464424f * x) * x;
5371 //
5372 // error 0.0144103317, which is 6 bits
5373 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5374 getF32Constant(DAG, 0x3e814304, dl));
5375 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5376 getF32Constant(DAG, 0x3f3c50c8, dl));
5377 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5378 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5379 getF32Constant(DAG, 0x3f7f5e7e, dl));
5380 } else if (LimitFloatPrecision <= 12) {
5381 // For floating-point precision of 12:
5382 //
5383 // TwoToFractionalPartOfX =
5384 // 0.999892986f +
5385 // (0.696457318f +
5386 // (0.224338339f + 0.792043434e-1f * x) * x) * x;
5387 //
5388 // error 0.000107046256, which is 13 to 14 bits
5389 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5390 getF32Constant(DAG, 0x3da235e3, dl));
5391 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5392 getF32Constant(DAG, 0x3e65b8f3, dl));
5393 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5394 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5395 getF32Constant(DAG, 0x3f324b07, dl));
5396 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5397 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
5398 getF32Constant(DAG, 0x3f7ff8fd, dl));
5399 } else { // LimitFloatPrecision <= 18
5400 // For floating-point precision of 18:
5401 //
5402 // TwoToFractionalPartOfX =
5403 // 0.999999982f +
5404 // (0.693148872f +
5405 // (0.240227044f +
5406 // (0.554906021e-1f +
5407 // (0.961591928e-2f +
5408 // (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
5409 // error 2.47208000*10^(-7), which is better than 18 bits
5410 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5411 getF32Constant(DAG, 0x3924b03e, dl));
5412 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5413 getF32Constant(DAG, 0x3ab24b87, dl));
5414 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5415 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5416 getF32Constant(DAG, 0x3c1d8c17, dl));
5417 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5418 SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
5419 getF32Constant(DAG, 0x3d634a1d, dl));
5420 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5421 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
5422 getF32Constant(DAG, 0x3e75fe14, dl));
5423 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
5424 SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
5425 getF32Constant(DAG, 0x3f317234, dl));
5426 SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
5427 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
5428 getF32Constant(DAG, 0x3f800000, dl));
5429 }
5430
5431 // Add the exponent into the result in integer domain.
5432 SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFractionalPartOfX);
5433 return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
5434 DAG.getNode(ISD::ADD, dl, MVT::i32, t13, IntegerPartOfX));
5435}
5436
5437/// expandExp - Lower an exp intrinsic. Handles the special sequences for
5438/// limited-precision mode.
5440 const TargetLowering &TLI, SDNodeFlags Flags) {
5441 if (Op.getValueType() == MVT::f32 &&
5443
5444 // Put the exponent in the right bit position for later addition to the
5445 // final result:
5446 //
5447 // t0 = Op * log2(e)
5448
5449 // TODO: What fast-math-flags should be set here?
5450 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op,
5451 DAG.getConstantFP(numbers::log2ef, dl, MVT::f32));
5452 return getLimitedPrecisionExp2(t0, dl, DAG);
5453 }
5454
5455 // No special expansion.
5456 return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op, Flags);
5457}
5458
5459/// expandLog - Lower a log intrinsic. Handles the special sequences for
5460/// limited-precision mode.
5462 const TargetLowering &TLI, SDNodeFlags Flags) {
5463 // TODO: What fast-math-flags should be set on the floating-point nodes?
5464
5465 if (Op.getValueType() == MVT::f32 &&
5467 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
5468
5469 // Scale the exponent by log(2).
5470 SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
5471 SDValue LogOfExponent =
5472 DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
5473 DAG.getConstantFP(numbers::ln2f, dl, MVT::f32));
5474
5475 // Get the significand and build it into a floating-point number with
5476 // exponent of 1.
5477 SDValue X = GetSignificand(DAG, Op1, dl);
5478
5479 SDValue LogOfMantissa;
5480 if (LimitFloatPrecision <= 6) {
5481 // For floating-point precision of 6:
5482 //
5483 // LogofMantissa =
5484 // -1.1609546f +
5485 // (1.4034025f - 0.23903021f * x) * x;
5486 //
5487 // error 0.0034276066, which is better than 8 bits
5488 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5489 getF32Constant(DAG, 0xbe74c456, dl));
5490 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5491 getF32Constant(DAG, 0x3fb3a2b1, dl));
5492 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5493 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5494 getF32Constant(DAG, 0x3f949a29, dl));
5495 } else if (LimitFloatPrecision <= 12) {
5496 // For floating-point precision of 12:
5497 //
5498 // LogOfMantissa =
5499 // -1.7417939f +
5500 // (2.8212026f +
5501 // (-1.4699568f +
5502 // (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;
5503 //
5504 // error 0.000061011436, which is 14 bits
5505 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5506 getF32Constant(DAG, 0xbd67b6d6, dl));
5507 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5508 getF32Constant(DAG, 0x3ee4f4b8, dl));
5509 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5510 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5511 getF32Constant(DAG, 0x3fbc278b, dl));
5512 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5513 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5514 getF32Constant(DAG, 0x40348e95, dl));
5515 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5516 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5517 getF32Constant(DAG, 0x3fdef31a, dl));
5518 } else { // LimitFloatPrecision <= 18
5519 // For floating-point precision of 18:
5520 //
5521 // LogOfMantissa =
5522 // -2.1072184f +
5523 // (4.2372794f +
5524 // (-3.7029485f +
5525 // (2.2781945f +
5526 // (-0.87823314f +
5527 // (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;
5528 //
5529 // error 0.0000023660568, which is better than 18 bits
5530 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5531 getF32Constant(DAG, 0xbc91e5ac, dl));
5532 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5533 getF32Constant(DAG, 0x3e4350aa, dl));
5534 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5535 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5536 getF32Constant(DAG, 0x3f60d3e3, dl));
5537 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5538 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5539 getF32Constant(DAG, 0x4011cdf0, dl));
5540 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5541 SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5542 getF32Constant(DAG, 0x406cfd1c, dl));
5543 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5544 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
5545 getF32Constant(DAG, 0x408797cb, dl));
5546 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
5547 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
5548 getF32Constant(DAG, 0x4006dcab, dl));
5549 }
5550
5551 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa);
5552 }
5553
5554 // No special expansion.
5555 return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op, Flags);
5556}
5557
5558/// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for
5559/// limited-precision mode.
5561 const TargetLowering &TLI, SDNodeFlags Flags) {
5562 // TODO: What fast-math-flags should be set on the floating-point nodes?
5563
5564 if (Op.getValueType() == MVT::f32 &&
5566 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
5567
5568 // Get the exponent.
5569 SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl);
5570
5571 // Get the significand and build it into a floating-point number with
5572 // exponent of 1.
5573 SDValue X = GetSignificand(DAG, Op1, dl);
5574
5575 // Different possible minimax approximations of significand in
5576 // floating-point for various degrees of accuracy over [1,2].
5577 SDValue Log2ofMantissa;
5578 if (LimitFloatPrecision <= 6) {
5579 // For floating-point precision of 6:
5580 //
5581 // Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x;
5582 //
5583 // error 0.0049451742, which is more than 7 bits
5584 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5585 getF32Constant(DAG, 0xbeb08fe0, dl));
5586 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5587 getF32Constant(DAG, 0x40019463, dl));
5588 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5589 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5590 getF32Constant(DAG, 0x3fd6633d, dl));
5591 } else if (LimitFloatPrecision <= 12) {
5592 // For floating-point precision of 12:
5593 //
5594 // Log2ofMantissa =
5595 // -2.51285454f +
5596 // (4.07009056f +
5597 // (-2.12067489f +
5598 // (.645142248f - 0.816157886e-1f * x) * x) * x) * x;
5599 //
5600 // error 0.0000876136000, which is better than 13 bits
5601 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5602 getF32Constant(DAG, 0xbda7262e, dl));
5603 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5604 getF32Constant(DAG, 0x3f25280b, dl));
5605 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5606 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5607 getF32Constant(DAG, 0x4007b923, dl));
5608 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5609 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5610 getF32Constant(DAG, 0x40823e2f, dl));
5611 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5612 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5613 getF32Constant(DAG, 0x4020d29c, dl));
5614 } else { // LimitFloatPrecision <= 18
5615 // For floating-point precision of 18:
5616 //
5617 // Log2ofMantissa =
5618 // -3.0400495f +
5619 // (6.1129976f +
5620 // (-5.3420409f +
5621 // (3.2865683f +
5622 // (-1.2669343f +
5623 // (0.27515199f -
5624 // 0.25691327e-1f * x) * x) * x) * x) * x) * x;
5625 //
5626 // error 0.0000018516, which is better than 18 bits
5627 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5628 getF32Constant(DAG, 0xbcd2769e, dl));
5629 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5630 getF32Constant(DAG, 0x3e8ce0b9, dl));
5631 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5632 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5633 getF32Constant(DAG, 0x3fa22ae7, dl));
5634 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5635 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5636 getF32Constant(DAG, 0x40525723, dl));
5637 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5638 SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5639 getF32Constant(DAG, 0x40aaf200, dl));
5640 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5641 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
5642 getF32Constant(DAG, 0x40c39dad, dl));
5643 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
5644 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
5645 getF32Constant(DAG, 0x4042902c, dl));
5646 }
5647
5648 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa);
5649 }
5650
5651 // No special expansion.
5652 return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op, Flags);
5653}
5654
5655/// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for
5656/// limited-precision mode.
5658 const TargetLowering &TLI, SDNodeFlags Flags) {
5659 // TODO: What fast-math-flags should be set on the floating-point nodes?
5660
5661 if (Op.getValueType() == MVT::f32 &&
5663 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
5664
5665 // Scale the exponent by log10(2) [0.30102999f].
5666 SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
5667 SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
5668 getF32Constant(DAG, 0x3e9a209a, dl));
5669
5670 // Get the significand and build it into a floating-point number with
5671 // exponent of 1.
5672 SDValue X = GetSignificand(DAG, Op1, dl);
5673
5674 SDValue Log10ofMantissa;
5675 if (LimitFloatPrecision <= 6) {
5676 // For floating-point precision of 6:
5677 //
5678 // Log10ofMantissa =
5679 // -0.50419619f +
5680 // (0.60948995f - 0.10380950f * x) * x;
5681 //
5682 // error 0.0014886165, which is 6 bits
5683 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5684 getF32Constant(DAG, 0xbdd49a13, dl));
5685 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5686 getF32Constant(DAG, 0x3f1c0789, dl));
5687 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5688 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5689 getF32Constant(DAG, 0x3f011300, dl));
5690 } else if (LimitFloatPrecision <= 12) {
5691 // For floating-point precision of 12:
5692 //
5693 // Log10ofMantissa =
5694 // -0.64831180f +
5695 // (0.91751397f +
5696 // (-0.31664806f + 0.47637168e-1f * x) * x) * x;
5697 //
5698 // error 0.00019228036, which is better than 12 bits
5699 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5700 getF32Constant(DAG, 0x3d431f31, dl));
5701 SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
5702 getF32Constant(DAG, 0x3ea21fb2, dl));
5703 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5704 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5705 getF32Constant(DAG, 0x3f6ae232, dl));
5706 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5707 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
5708 getF32Constant(DAG, 0x3f25f7c3, dl));
5709 } else { // LimitFloatPrecision <= 18
5710 // For floating-point precision of 18:
5711 //
5712 // Log10ofMantissa =
5713 // -0.84299375f +
5714 // (1.5327582f +
5715 // (-1.0688956f +
5716 // (0.49102474f +
5717 // (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x;
5718 //
5719 // error 0.0000037995730, which is better than 18 bits
5720 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5721 getF32Constant(DAG, 0x3c5d51ce, dl));
5722 SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
5723 getF32Constant(DAG, 0x3e00685a, dl));
5724 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5725 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5726 getF32Constant(DAG, 0x3efb6798, dl));
5727 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5728 SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
5729 getF32Constant(DAG, 0x3f88d192, dl));
5730 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5731 SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
5732 getF32Constant(DAG, 0x3fc4316c, dl));
5733 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5734 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8,
5735 getF32Constant(DAG, 0x3f57ce70, dl));
5736 }
5737
5738 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa);
5739 }
5740
5741 // No special expansion.
5742 return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op, Flags);
5743}
5744
5745/// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for
5746/// limited-precision mode.
5748 const TargetLowering &TLI, SDNodeFlags Flags) {
5749 if (Op.getValueType() == MVT::f32 &&
5751 return getLimitedPrecisionExp2(Op, dl, DAG);
5752
5753 // No special expansion.
5754 return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op, Flags);
5755}
5756
5757/// visitPow - Lower a pow intrinsic. Handles the special sequences for
5758/// limited-precision mode with x == 10.0f.
5759static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS,
5760 SelectionDAG &DAG, const TargetLowering &TLI,
5761 SDNodeFlags Flags) {
5762 bool IsExp10 = false;
5763 if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 &&
5765 if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(LHS)) {
5766 APFloat Ten(10.0f);
5767 IsExp10 = LHSC->isExactlyValue(Ten);
5768 }
5769 }
5770
5771 // TODO: What fast-math-flags should be set on the FMUL node?
5772 if (IsExp10) {
5773 // Put the exponent in the right bit position for later addition to the
5774 // final result:
5775 //
5776 // #define LOG2OF10 3.3219281f
5777 // t0 = Op * LOG2OF10;
5778 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS,
5779 getF32Constant(DAG, 0x40549a78, dl));
5780 return getLimitedPrecisionExp2(t0, dl, DAG);
5781 }
5782
5783 // No special expansion.
5784 return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS, Flags);
5785}
5786
5787/// ExpandPowI - Expand a llvm.powi intrinsic.
5788static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS,
5789 SelectionDAG &DAG) {
5790 // If RHS is a constant, we can expand this out to a multiplication tree if
5791 // it's beneficial on the target, otherwise we end up lowering to a call to
5792 // __powidf2 (for example).
5793 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
5794 unsigned Val = RHSC->getSExtValue();
5795
5796 // powi(x, 0) -> 1.0
5797 if (Val == 0)
5798 return DAG.getConstantFP(1.0, DL, LHS.getValueType());
5799
5801 Val, DAG.shouldOptForSize())) {
5802 // Get the exponent as a positive value.
5803 if ((int)Val < 0)
5804 Val = -Val;
5805 // We use the simple binary decomposition method to generate the multiply
5806 // sequence. There are more optimal ways to do this (for example,
5807 // powi(x,15) generates one more multiply than it should), but this has
5808 // the benefit of being both really simple and much better than a libcall.
5809 SDValue Res; // Logically starts equal to 1.0
5810 SDValue CurSquare = LHS;
5811 // TODO: Intrinsics should have fast-math-flags that propagate to these
5812 // nodes.
5813 while (Val) {
5814 if (Val & 1) {
5815 if (Res.getNode())
5816 Res =
5817 DAG.getNode(ISD::FMUL, DL, Res.getValueType(), Res, CurSquare);
5818 else
5819 Res = CurSquare; // 1.0*CurSquare.
5820 }
5821
5822 CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(),
5823 CurSquare, CurSquare);
5824 Val >>= 1;
5825 }
5826
5827 // If the original was negative, invert the result, producing 1/(x*x*x).
5828 if (RHSC->getSExtValue() < 0)
5829 Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(),
5830 DAG.getConstantFP(1.0, DL, LHS.getValueType()), Res);
5831 return Res;
5832 }
5833 }
5834
5835 // Otherwise, expand to a libcall.
5836 return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS);
5837}
5838
5839static SDValue expandDivFix(unsigned Opcode, const SDLoc &DL,
5840 SDValue LHS, SDValue RHS, SDValue Scale,
5841 SelectionDAG &DAG, const TargetLowering &TLI) {
5842 EVT VT = LHS.getValueType();
5843 bool Signed = Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT;
5844 bool Saturating = Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIXSAT;
5845 LLVMContext &Ctx = *DAG.getContext();
5846
5847 // If the type is legal but the operation isn't, this node might survive all
5848 // the way to operation legalization. If we end up there and we do not have
5849 // the ability to widen the type (if VT*2 is not legal), we cannot expand the
5850 // node.
5851
5852 // Coax the legalizer into expanding the node during type legalization instead
5853 // by bumping the size by one bit. This will force it to Promote, enabling the
5854 // early expansion and avoiding the need to expand later.
5855
5856 // We don't have to do this if Scale is 0; that can always be expanded, unless
5857 // it's a saturating signed operation. Those can experience true integer
5858 // division overflow, a case which we must avoid.
5859
5860 // FIXME: We wouldn't have to do this (or any of the early
5861 // expansion/promotion) if it was possible to expand a libcall of an
5862 // illegal type during operation legalization. But it's not, so things
5863 // get a bit hacky.
5864 unsigned ScaleInt = Scale->getAsZExtVal();
5865 if ((ScaleInt > 0 || (Saturating && Signed)) &&
5866 (TLI.isTypeLegal(VT) ||
5867 (VT.isVector() && TLI.isTypeLegal(VT.getVectorElementType())))) {
5869 Opcode, VT, ScaleInt);
5870 if (Action != TargetLowering::Legal && Action != TargetLowering::Custom) {
5871 EVT PromVT;
5872 if (VT.isScalarInteger())
5873 PromVT = EVT::getIntegerVT(Ctx, VT.getSizeInBits() + 1);
5874 else if (VT.isVector()) {
5875 PromVT = VT.getVectorElementType();
5876 PromVT = EVT::getIntegerVT(Ctx, PromVT.getSizeInBits() + 1);
5877 PromVT = EVT::getVectorVT(Ctx, PromVT, VT.getVectorElementCount());
5878 } else
5879 llvm_unreachable("Wrong VT for DIVFIX?");
5880 LHS = DAG.getExtOrTrunc(Signed, LHS, DL, PromVT);
5881 RHS = DAG.getExtOrTrunc(Signed, RHS, DL, PromVT);
5882 EVT ShiftTy = TLI.getShiftAmountTy(PromVT, DAG.getDataLayout());
5883 // For saturating operations, we need to shift up the LHS to get the
5884 // proper saturation width, and then shift down again afterwards.
5885 if (Saturating)
5886 LHS = DAG.getNode(ISD::SHL, DL, PromVT, LHS,
5887 DAG.getConstant(1, DL, ShiftTy));
5888 SDValue Res = DAG.getNode(Opcode, DL, PromVT, LHS, RHS, Scale);
5889 if (Saturating)
5890 Res = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, DL, PromVT, Res,
5891 DAG.getConstant(1, DL, ShiftTy));
5892 return DAG.getZExtOrTrunc(Res, DL, VT);
5893 }
5894 }
5895
5896 return DAG.getNode(Opcode, DL, VT, LHS, RHS, Scale);
5897}
5898
5899// getUnderlyingArgRegs - Find underlying registers used for a truncated,
5900// bitcasted, or split argument. Returns a list of <Register, size in bits>
5901static void
5902getUnderlyingArgRegs(SmallVectorImpl<std::pair<unsigned, TypeSize>> &Regs,
5903 const SDValue &N) {
5904 switch (N.getOpcode()) {
5905 case ISD::CopyFromReg: {
5906 SDValue Op = N.getOperand(1);
5907 Regs.emplace_back(cast<RegisterSDNode>(Op)->getReg(),
5908 Op.getValueType().getSizeInBits());
5909 return;
5910 }
5911 case ISD::BITCAST:
5912 case ISD::AssertZext:
5913 case ISD::AssertSext:
5914 case ISD::TRUNCATE:
5915 getUnderlyingArgRegs(Regs, N.getOperand(0));
5916 return;
5917 case ISD::BUILD_PAIR:
5918 case ISD::BUILD_VECTOR:
5920 for (SDValue Op : N->op_values())
5921 getUnderlyingArgRegs(Regs, Op);
5922 return;
5923 default:
5924 return;
5925 }
5926}
5927
5928/// If the DbgValueInst is a dbg_value of a function argument, create the
5929/// corresponding DBG_VALUE machine instruction for it now. At the end of
5930/// instruction selection, they will be inserted to the entry BB.
5931/// We don't currently support this for variadic dbg_values, as they shouldn't
5932/// appear for function arguments or in the prologue.
5933bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(
5934 const Value *V, DILocalVariable *Variable, DIExpression *Expr,
5935 DILocation *DL, FuncArgumentDbgValueKind Kind, const SDValue &N) {
5936 const Argument *Arg = dyn_cast<Argument>(V);
5937 if (!Arg)
5938 return false;
5939
5942
5943 // Helper to create DBG_INSTR_REFs or DBG_VALUEs, depending on what kind
5944 // we've been asked to pursue.
5945 auto MakeVRegDbgValue = [&](Register Reg, DIExpression *FragExpr,
5946 bool Indirect) {
5947 if (Reg.isVirtual() && MF.useDebugInstrRef()) {
5948 // For VRegs, in instruction referencing mode, create a DBG_INSTR_REF
5949 // pointing at the VReg, which will be patched up later.
5950 auto &Inst = TII->get(TargetOpcode::DBG_INSTR_REF);
5952 /* Reg */ Reg, /* isDef */ false, /* isImp */ false,
5953 /* isKill */ false, /* isDead */ false,
5954 /* isUndef */ false, /* isEarlyClobber */ false,
5955 /* SubReg */ 0, /* isDebug */ true)});
5956
5957 auto *NewDIExpr = FragExpr;
5958 // We don't have an "Indirect" field in DBG_INSTR_REF, fold that into
5959 // the DIExpression.
5960 if (Indirect)
5961 NewDIExpr = DIExpression::prepend(FragExpr, DIExpression::DerefBefore);
5963 NewDIExpr = DIExpression::prependOpcodes(NewDIExpr, Ops);
5964 return BuildMI(MF, DL, Inst, false, MOs, Variable, NewDIExpr);
5965 } else {
5966 // Create a completely standard DBG_VALUE.
5967 auto &Inst = TII->get(TargetOpcode::DBG_VALUE);
5968 return BuildMI(MF, DL, Inst, Indirect, Reg, Variable, FragExpr);
5969 }
5970 };
5971
5972 if (Kind == FuncArgumentDbgValueKind::Value) {
5973 // ArgDbgValues are hoisted to the beginning of the entry block. So we
5974 // should only emit as ArgDbgValue if the dbg.value intrinsic is found in
5975 // the entry block.
5976 bool IsInEntryBlock = FuncInfo.MBB == &FuncInfo.MF->front();
5977 if (!IsInEntryBlock)
5978 return false;
5979
5980 // ArgDbgValues are hoisted to the beginning of the entry block. So we
5981 // should only emit as ArgDbgValue if the dbg.value intrinsic describes a
5982 // variable that also is a param.
5983 //
5984 // Although, if we are at the top of the entry block already, we can still
5985 // emit using ArgDbgValue. This might catch some situations when the
5986 // dbg.value refers to an argument that isn't used in the entry block, so
5987 // any CopyToReg node would be optimized out and the only way to express
5988 // this DBG_VALUE is by using the physical reg (or FI) as done in this
5989 // method. ArgDbgValues are hoisted to the beginning of the entry block. So
5990 // we should only emit as ArgDbgValue if the Variable is an argument to the
5991 // current function, and the dbg.value intrinsic is found in the entry
5992 // block.
5993 bool VariableIsFunctionInputArg = Variable->isParameter() &&
5994 !DL->getInlinedAt();
5995 bool IsInPrologue = SDNodeOrder == LowestSDNodeOrder;
5996 if (!IsInPrologue && !VariableIsFunctionInputArg)
5997 return false;
5998
5999 // Here we assume that a function argument on IR level only can be used to
6000 // describe one input parameter on source level. If we for example have
6001 // source code like this
6002 //
6003 // struct A { long x, y; };
6004 // void foo(struct A a, long b) {
6005 // ...
6006 // b = a.x;
6007 // ...
6008 // }
6009 //
6010 // and IR like this
6011 //
6012 // define void @foo(i32 %a1, i32 %a2, i32 %b) {
6013 // entry:
6014 // call void @llvm.dbg.value(metadata i32 %a1, "a", DW_OP_LLVM_fragment
6015 // call void @llvm.dbg.value(metadata i32 %a2, "a", DW_OP_LLVM_fragment
6016 // call void @llvm.dbg.value(metadata i32 %b, "b",
6017 // ...
6018 // call void @llvm.dbg.value(metadata i32 %a1, "b"
6019 // ...
6020 //
6021 // then the last dbg.value is describing a parameter "b" using a value that
6022 // is an argument. But since we already has used %a1 to describe a parameter
6023 // we should not handle that last dbg.value here (that would result in an
6024 // incorrect hoisting of the DBG_VALUE to the function entry).
6025 // Notice that we allow one dbg.value per IR level argument, to accommodate
6026 // for the situation with fragments above.
6027 // If there is no node for the value being handled, we return true to skip
6028 // the normal generation of debug info, as it would kill existing debug
6029 // info for the parameter in case of duplicates.
6030 if (VariableIsFunctionInputArg) {
6031 unsigned ArgNo = Arg->getArgNo();
6032 if (ArgNo >= FuncInfo.DescribedArgs.size())
6033 FuncInfo.DescribedArgs.resize(ArgNo + 1, false);
6034 else if (!IsInPrologue && FuncInfo.DescribedArgs.test(ArgNo))
6035 return !NodeMap[V].getNode();
6036 FuncInfo.DescribedArgs.set(ArgNo);
6037 }
6038 }
6039
6040 bool IsIndirect = false;
6041 std::optional<MachineOperand> Op;
6042 // Some arguments' frame index is recorded during argument lowering.
6043 int FI = FuncInfo.getArgumentFrameIndex(Arg);
6044 if (FI != std::numeric_limits<int>::max())
6046
6048 if (!Op && N.getNode()) {
6049 getUnderlyingArgRegs(ArgRegsAndSizes, N);
6050 Register Reg;
6051 if (ArgRegsAndSizes.size() == 1)
6052 Reg = ArgRegsAndSizes.front().first;
6053
6054 if (Reg && Reg.isVirtual()) {
6056 Register PR = RegInfo.getLiveInPhysReg(Reg);
6057 if (PR)
6058 Reg = PR;
6059 }
6060 if (Reg) {
6061 Op = MachineOperand::CreateReg(Reg, false);
6062 IsIndirect = Kind != FuncArgumentDbgValueKind::Value;
6063 }
6064 }
6065
6066 if (!Op && N.getNode()) {
6067 // Check if frame index is available.
6068 SDValue LCandidate = peekThroughBitcasts(N);
6069 if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(LCandidate.getNode()))
6070 if (FrameIndexSDNode *FINode =
6071 dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
6072 Op = MachineOperand::CreateFI(FINode->getIndex());
6073 }
6074
6075 if (!Op) {
6076 // Create a DBG_VALUE for each decomposed value in ArgRegs to cover Reg
6077 auto splitMultiRegDbgValue = [&](ArrayRef<std::pair<unsigned, TypeSize>>
6078 SplitRegs) {
6079 unsigned Offset = 0;
6080 for (const auto &RegAndSize : SplitRegs) {
6081 // If the expression is already a fragment, the current register
6082 // offset+size might extend beyond the fragment. In this case, only
6083 // the register bits that are inside the fragment are relevant.
6084 int RegFragmentSizeInBits = RegAndSize.second;
6085 if (auto ExprFragmentInfo = Expr->getFragmentInfo()) {
6086 uint64_t ExprFragmentSizeInBits = ExprFragmentInfo->SizeInBits;
6087 // The register is entirely outside the expression fragment,
6088 // so is irrelevant for debug info.
6089 if (Offset >= ExprFragmentSizeInBits)
6090 break;
6091 // The register is partially outside the expression fragment, only
6092 // the low bits within the fragment are relevant for debug info.
6093 if (Offset + RegFragmentSizeInBits > ExprFragmentSizeInBits) {
6094 RegFragmentSizeInBits = ExprFragmentSizeInBits - Offset;
6095 }
6096 }
6097
6098 auto FragmentExpr = DIExpression::createFragmentExpression(
6099 Expr, Offset, RegFragmentSizeInBits);
6100 Offset += RegAndSize.second;
6101 // If a valid fragment expression cannot be created, the variable's
6102 // correct value cannot be determined and so it is set as Undef.
6103 if (!FragmentExpr) {
6105 Variable, Expr, UndefValue::get(V->getType()), DL, SDNodeOrder);
6106 DAG.AddDbgValue(SDV, false);
6107 continue;
6108 }
6109 MachineInstr *NewMI =
6110 MakeVRegDbgValue(RegAndSize.first, *FragmentExpr,
6111 Kind != FuncArgumentDbgValueKind::Value);
6112 FuncInfo.ArgDbgValues.push_back(NewMI);
6113 }
6114 };
6115
6116 // Check if ValueMap has reg number.
6118 VMI = FuncInfo.ValueMap.find(V);
6119 if (VMI != FuncInfo.ValueMap.end()) {
6120 const auto &TLI = DAG.getTargetLoweringInfo();
6121 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), VMI->second,
6122 V->getType(), std::nullopt);
6123 if (RFV.occupiesMultipleRegs()) {
6124 splitMultiRegDbgValue(RFV.getRegsAndSizes());
6125 return true;
6126 }
6127
6128 Op = MachineOperand::CreateReg(VMI->second, false);
6129 IsIndirect = Kind != FuncArgumentDbgValueKind::Value;
6130 } else if (ArgRegsAndSizes.size() > 1) {
6131 // This was split due to the calling convention, and no virtual register
6132 // mapping exists for the value.
6133 splitMultiRegDbgValue(ArgRegsAndSizes);
6134 return true;
6135 }
6136 }
6137
6138 if (!Op)
6139 return false;
6140
6142 "Expected inlined-at fields to agree");
6143 MachineInstr *NewMI = nullptr;
6144
6145 if (Op->isReg())
6146 NewMI = MakeVRegDbgValue(Op->getReg(), Expr, IsIndirect);
6147 else
6148 NewMI = BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE), true, *Op,
6149 Variable, Expr);
6150
6151 // Otherwise, use ArgDbgValues.
6152 FuncInfo.ArgDbgValues.push_back(NewMI);
6153 return true;
6154}
6155
6156/// Return the appropriate SDDbgValue based on N.
6157SDDbgValue *SelectionDAGBuilder::getDbgValue(SDValue N,
6158 DILocalVariable *Variable,
6159 DIExpression *Expr,
6160 const DebugLoc &dl,
6161 unsigned DbgSDNodeOrder) {
6162 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) {
6163 // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can describe
6164 // stack slot locations.
6165 //
6166 // Consider "int x = 0; int *px = &x;". There are two kinds of interesting
6167 // debug values here after optimization:
6168 //
6169 // dbg.value(i32* %px, !"int *px", !DIExpression()), and
6170 // dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))
6171 //
6172 // Both describe the direct values of their associated variables.
6173 return DAG.getFrameIndexDbgValue(Variable, Expr, FISDN->getIndex(),
6174 /*IsIndirect*/ false, dl, DbgSDNodeOrder);
6175 }
6176 return DAG.getDbgValue(Variable, Expr, N.getNode(), N.getResNo(),
6177 /*IsIndirect*/ false, dl, DbgSDNodeOrder);
6178}
6179
6180static unsigned FixedPointIntrinsicToOpcode(unsigned Intrinsic) {
6181 switch (Intrinsic) {
6182 case Intrinsic::smul_fix:
6183 return ISD::SMULFIX;
6184 case Intrinsic::umul_fix:
6185 return ISD::UMULFIX;
6186 case Intrinsic::smul_fix_sat:
6187 return ISD::SMULFIXSAT;
6188 case Intrinsic::umul_fix_sat:
6189 return ISD::UMULFIXSAT;
6190 case Intrinsic::sdiv_fix:
6191 return ISD::SDIVFIX;
6192 case Intrinsic::udiv_fix:
6193 return ISD::UDIVFIX;
6194 case Intrinsic::sdiv_fix_sat:
6195 return ISD::SDIVFIXSAT;
6196 case Intrinsic::udiv_fix_sat:
6197 return ISD::UDIVFIXSAT;
6198 default:
6199 llvm_unreachable("Unhandled fixed point intrinsic");
6200 }
6201}
6202
6203void SelectionDAGBuilder::lowerCallToExternalSymbol(const CallInst &I,
6204 const char *FunctionName) {
6205 assert(FunctionName && "FunctionName must not be nullptr");
6207 FunctionName,
6209 LowerCallTo(I, Callee, I.isTailCall(), I.isMustTailCall());
6210}
6211
6212/// Given a @llvm.call.preallocated.setup, return the corresponding
6213/// preallocated call.
6214static const CallBase *FindPreallocatedCall(const Value *PreallocatedSetup) {
6215 assert(cast<CallBase>(PreallocatedSetup)
6217 ->getIntrinsicID() == Intrinsic::call_preallocated_setup &&
6218 "expected call_preallocated_setup Value");
6219 for (const auto *U : PreallocatedSetup->users()) {
6220 auto *UseCall = cast<CallBase>(U);
6221 const Function *Fn = UseCall->getCalledFunction();
6222 if (!Fn || Fn->getIntrinsicID() != Intrinsic::call_preallocated_arg) {
6223 return UseCall;
6224 }
6225 }
6226 llvm_unreachable("expected corresponding call to preallocated setup/arg");
6227}
6228
6229/// If DI is a debug value with an EntryValue expression, lower it using the
6230/// corresponding physical register of the associated Argument value
6231/// (guaranteed to exist by the verifier).
6232bool SelectionDAGBuilder::visitEntryValueDbgValue(
6233 ArrayRef<const Value *> Values, DILocalVariable *Variable,
6234 DIExpression *Expr, DebugLoc DbgLoc) {
6235 if (!Expr->isEntryValue() || !hasSingleElement(Values))
6236 return false;
6237
6238 // These properties are guaranteed by the verifier.
6239 const Argument *Arg = cast<Argument>(Values[0]);
6240 assert(Arg->hasAttribute(Attribute::AttrKind::SwiftAsync));
6241
6242 auto ArgIt = FuncInfo.ValueMap.find(Arg);
6243 if (ArgIt == FuncInfo.ValueMap.end()) {
6244 LLVM_DEBUG(
6245 dbgs() << "Dropping dbg.value: expression is entry_value but "
6246 "couldn't find an associated register for the Argument\n");
6247 return true;
6248 }
6249 Register ArgVReg = ArgIt->getSecond();
6250
6251 for (auto [PhysReg, VirtReg] : FuncInfo.RegInfo->liveins())
6252 if (ArgVReg == VirtReg || ArgVReg == PhysReg) {
6254 Variable, Expr, PhysReg, false /*IsIndidrect*/, DbgLoc, SDNodeOrder);
6255 DAG.AddDbgValue(SDV, false /*treat as dbg.declare byval parameter*/);
6256 return true;
6257 }
6258 LLVM_DEBUG(dbgs() << "Dropping dbg.value: expression is entry_value but "
6259 "couldn't find a physical register\n");
6260 return true;
6261}
6262
6263/// Lower the call to the specified intrinsic function.
6264void SelectionDAGBuilder::visitConvergenceControl(const CallInst &I,
6265 unsigned Intrinsic) {
6266 SDLoc sdl = getCurSDLoc();
6267 switch (Intrinsic) {
6268 case Intrinsic::experimental_convergence_anchor:
6269 setValue(&I, DAG.getNode(ISD::CONVERGENCECTRL_ANCHOR, sdl, MVT::Untyped));
6270 break;
6271 case Intrinsic::experimental_convergence_entry:
6272 setValue(&I, DAG.getNode(ISD::CONVERGENCECTRL_ENTRY, sdl, MVT::Untyped));
6273 break;
6274 case Intrinsic::experimental_convergence_loop: {
6275 auto Bundle = I.getOperandBundle(LLVMContext::OB_convergencectrl);
6276 auto *Token = Bundle->Inputs[0].get();
6277 setValue(&I, DAG.getNode(ISD::CONVERGENCECTRL_LOOP, sdl, MVT::Untyped,
6278 getValue(Token)));
6279 break;
6280 }
6281 }
6282}
6283
6284/// Lower the call to the specified intrinsic function.
6285void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
6286 unsigned Intrinsic) {
6288 SDLoc sdl = getCurSDLoc();
6289 DebugLoc dl = getCurDebugLoc();
6290 SDValue Res;
6291
6293 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
6294 Flags.copyFMF(*FPOp);
6295
6296 switch (Intrinsic) {
6297 default:
6298 // By default, turn this into a target intrinsic node.
6299 visitTargetIntrinsic(I, Intrinsic);
6300 return;
6301 case Intrinsic::vscale: {
6302 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6303 setValue(&I, DAG.getVScale(sdl, VT, APInt(VT.getSizeInBits(), 1)));
6304 return;
6305 }
6306 case Intrinsic::vastart: visitVAStart(I); return;
6307 case Intrinsic::vaend: visitVAEnd(I); return;
6308 case Intrinsic::vacopy: visitVACopy(I); return;
6309 case Intrinsic::returnaddress:
6311 TLI.getValueType(DAG.getDataLayout(), I.getType()),
6312 getValue(I.getArgOperand(0))));
6313 return;
6314 case Intrinsic::addressofreturnaddress:
6315 setValue(&I,
6317 TLI.getValueType(DAG.getDataLayout(), I.getType())));
6318 return;
6319 case Intrinsic::sponentry:
6320 setValue(&I,
6322 TLI.getValueType(DAG.getDataLayout(), I.getType())));
6323 return;
6324 case Intrinsic::frameaddress:
6327 getValue(I.getArgOperand(0))));
6328 return;
6329 case Intrinsic::read_volatile_register:
6330 case Intrinsic::read_register: {
6331 Value *Reg = I.getArgOperand(0);
6332 SDValue Chain = getRoot();
6334 DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
6335 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6336 Res = DAG.getNode(ISD::READ_REGISTER, sdl,
6337 DAG.getVTList(VT, MVT::Other), Chain, RegName);
6338 setValue(&I, Res);
6339 DAG.setRoot(Res.getValue(1));
6340 return;
6341 }
6342 case Intrinsic::write_register: {
6343 Value *Reg = I.getArgOperand(0);
6344 Value *RegValue = I.getArgOperand(1);
6345 SDValue Chain = getRoot();
6347 DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
6348 DAG.setRoot(DAG.getNode(ISD::WRITE_REGISTER, sdl, MVT::Other, Chain,
6349 RegName, getValue(RegValue)));
6350 return;
6351 }
6352 case Intrinsic::memcpy: {
6353 const auto &MCI = cast<MemCpyInst>(I);
6354 SDValue Op1 = getValue(I.getArgOperand(0));
6355 SDValue Op2 = getValue(I.getArgOperand(1));
6356 SDValue Op3 = getValue(I.getArgOperand(2));
6357 // @llvm.memcpy defines 0 and 1 to both mean no alignment.
6358 Align DstAlign = MCI.getDestAlign().valueOrOne();
6359 Align SrcAlign = MCI.getSourceAlign().valueOrOne();
6360 Align Alignment = std::min(DstAlign, SrcAlign);
6361 bool isVol = MCI.isVolatile();
6362 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
6363 // FIXME: Support passing different dest/src alignments to the memcpy DAG
6364 // node.
6365 SDValue Root = isVol ? getRoot() : getMemoryRoot();
6366 SDValue MC = DAG.getMemcpy(
6367 Root, sdl, Op1, Op2, Op3, Alignment, isVol,
6368 /* AlwaysInline */ false, isTC, MachinePointerInfo(I.getArgOperand(0)),
6369 MachinePointerInfo(I.getArgOperand(1)), I.getAAMetadata(), AA);
6370 updateDAGForMaybeTailCall(MC);
6371 return;
6372 }
6373 case Intrinsic::memcpy_inline: {
6374 const auto &MCI = cast<MemCpyInlineInst>(I);
6375 SDValue Dst = getValue(I.getArgOperand(0));
6376 SDValue Src = getValue(I.getArgOperand(1));
6377 SDValue Size = getValue(I.getArgOperand(2));
6378 assert(isa<ConstantSDNode>(Size) && "memcpy_inline needs constant size");
6379 // @llvm.memcpy.inline defines 0 and 1 to both mean no alignment.
6380 Align DstAlign = MCI.getDestAlign().valueOrOne();
6381 Align SrcAlign = MCI.getSourceAlign().valueOrOne();
6382 Align Alignment = std::min(DstAlign, SrcAlign);
6383 bool isVol = MCI.isVolatile();
6384 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
6385 // FIXME: Support passing different dest/src alignments to the memcpy DAG
6386 // node.
6387 SDValue MC = DAG.getMemcpy(
6388 getRoot(), sdl, Dst, Src, Size, Alignment, isVol,
6389 /* AlwaysInline */ true, isTC, MachinePointerInfo(I.getArgOperand(0)),
6390 MachinePointerInfo(I.getArgOperand(1)), I.getAAMetadata(), AA);
6391 updateDAGForMaybeTailCall(MC);
6392 return;
6393 }
6394 case Intrinsic::memset: {
6395 const auto &MSI = cast<MemSetInst>(I);
6396 SDValue Op1 = getValue(I.getArgOperand(0));
6397 SDValue Op2 = getValue(I.getArgOperand(1));
6398 SDValue Op3 = getValue(I.getArgOperand(2));
6399 // @llvm.memset defines 0 and 1 to both mean no alignment.
6400 Align Alignment = MSI.getDestAlign().valueOrOne();
6401 bool isVol = MSI.isVolatile();
6402 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
6403 SDValue Root = isVol ? getRoot() : getMemoryRoot();
6404 SDValue MS = DAG.getMemset(
6405 Root, sdl, Op1, Op2, Op3, Alignment, isVol, /* AlwaysInline */ false,
6406 isTC, MachinePointerInfo(I.getArgOperand(0)), I.getAAMetadata());
6407 updateDAGForMaybeTailCall(MS);
6408 return;
6409 }
6410 case Intrinsic::memset_inline: {
6411 const auto &MSII = cast<MemSetInlineInst>(I);
6412 SDValue Dst = getValue(I.getArgOperand(0));
6413 SDValue Value = getValue(I.getArgOperand(1));
6414 SDValue Size = getValue(I.getArgOperand(2));
6415 assert(isa<ConstantSDNode>(Size) && "memset_inline needs constant size");
6416 // @llvm.memset defines 0 and 1 to both mean no alignment.
6417 Align DstAlign = MSII.getDestAlign().valueOrOne();
6418 bool isVol = MSII.isVolatile();
6419 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
6420 SDValue Root = isVol ? getRoot() : getMemoryRoot();
6421 SDValue MC = DAG.getMemset(Root, sdl, Dst, Value, Size, DstAlign, isVol,
6422 /* AlwaysInline */ true, isTC,
6423 MachinePointerInfo(I.getArgOperand(0)),
6424 I.getAAMetadata());
6425 updateDAGForMaybeTailCall(MC);
6426 return;
6427 }
6428 case Intrinsic::memmove: {
6429 const auto &MMI = cast<MemMoveInst>(I);
6430 SDValue Op1 = getValue(I.getArgOperand(0));
6431 SDValue Op2 = getValue(I.getArgOperand(1));
6432 SDValue Op3 = getValue(I.getArgOperand(2));
6433 // @llvm.memmove defines 0 and 1 to both mean no alignment.
6434 Align DstAlign = MMI.getDestAlign().valueOrOne();
6435 Align SrcAlign = MMI.getSourceAlign().valueOrOne();
6436 Align Alignment = std::min(DstAlign, SrcAlign);
6437 bool isVol = MMI.isVolatile();
6438 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
6439 // FIXME: Support passing different dest/src alignments to the memmove DAG
6440 // node.
6441 SDValue Root = isVol ? getRoot() : getMemoryRoot();
6442 SDValue MM = DAG.getMemmove(Root, sdl, Op1, Op2, Op3, Alignment, isVol,
6443 isTC, MachinePointerInfo(I.getArgOperand(0)),
6444 MachinePointerInfo(I.getArgOperand(1)),
6445 I.getAAMetadata(), AA);
6446 updateDAGForMaybeTailCall(MM);
6447 return;
6448 }
6449 case Intrinsic::memcpy_element_unordered_atomic: {
6450 const AtomicMemCpyInst &MI = cast<AtomicMemCpyInst>(I);
6451 SDValue Dst = getValue(MI.getRawDest());
6452 SDValue Src = getValue(MI.getRawSource());
6453 SDValue Length = getValue(MI.getLength());
6454
6455 Type *LengthTy = MI.getLength()->getType();
6456 unsigned ElemSz = MI.getElementSizeInBytes();
6457 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
6458 SDValue MC =
6459 DAG.getAtomicMemcpy(getRoot(), sdl, Dst, Src, Length, LengthTy, ElemSz,
6460 isTC, MachinePointerInfo(MI.getRawDest()),
6461 MachinePointerInfo(MI.getRawSource()));
6462 updateDAGForMaybeTailCall(MC);
6463 return;
6464 }
6465 case Intrinsic::memmove_element_unordered_atomic: {
6466 auto &MI = cast<AtomicMemMoveInst>(I);
6467 SDValue Dst = getValue(MI.getRawDest());
6468 SDValue Src = getValue(MI.getRawSource());
6469 SDValue Length = getValue(MI.getLength());
6470
6471 Type *LengthTy = MI.getLength()->getType();
6472 unsigned ElemSz = MI.getElementSizeInBytes();
6473 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
6474 SDValue MC =
6475 DAG.getAtomicMemmove(getRoot(), sdl, Dst, Src, Length, LengthTy, ElemSz,
6476 isTC, MachinePointerInfo(MI.getRawDest()),
6477 MachinePointerInfo(MI.getRawSource()));
6478 updateDAGForMaybeTailCall(MC);
6479 return;
6480 }
6481 case Intrinsic::memset_element_unordered_atomic: {
6482 auto &MI = cast<AtomicMemSetInst>(I);
6483 SDValue Dst = getValue(MI.getRawDest());
6484 SDValue Val = getValue(MI.getValue());
6485 SDValue Length = getValue(MI.getLength());
6486
6487 Type *LengthTy = MI.getLength()->getType();
6488 unsigned ElemSz = MI.getElementSizeInBytes();
6489 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
6490 SDValue MC =
6491 DAG.getAtomicMemset(getRoot(), sdl, Dst, Val, Length, LengthTy, ElemSz,
6492 isTC, MachinePointerInfo(MI.getRawDest()));
6493 updateDAGForMaybeTailCall(MC);
6494 return;
6495 }
6496 case Intrinsic::call_preallocated_setup: {
6497 const CallBase *PreallocatedCall = FindPreallocatedCall(&I);
6498 SDValue SrcValue = DAG.getSrcValue(PreallocatedCall);
6499 SDValue Res = DAG.getNode(ISD::PREALLOCATED_SETUP, sdl, MVT::Other,
6500 getRoot(), SrcValue);
6501 setValue(&I, Res);
6502 DAG.setRoot(Res);
6503 return;
6504 }
6505 case Intrinsic::call_preallocated_arg: {
6506 const CallBase *PreallocatedCall = FindPreallocatedCall(I.getOperand(0));
6507 SDValue SrcValue = DAG.getSrcValue(PreallocatedCall);
6508 SDValue Ops[3];
6509 Ops[0] = getRoot();
6510 Ops[1] = SrcValue;
6511 Ops[2] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(1)), sdl,
6512 MVT::i32); // arg index
6513 SDValue Res = DAG.getNode(
6515 DAG.getVTList(TLI.getPointerTy(DAG.getDataLayout()), MVT::Other), Ops);
6516 setValue(&I, Res);
6517 DAG.setRoot(Res.getValue(1));
6518 return;
6519 }
6520 case Intrinsic::dbg_declare: {
6521 const auto &DI = cast<DbgDeclareInst>(I);
6522 // Debug intrinsics are handled separately in assignment tracking mode.
6523 // Some intrinsics are handled right after Argument lowering.
6524 if (AssignmentTrackingEnabled ||
6526 return;
6527 LLVM_DEBUG(dbgs() << "SelectionDAG visiting dbg_declare: " << DI << "\n");
6528 DILocalVariable *Variable = DI.getVariable();
6529 DIExpression *Expression = DI.getExpression();
6531 // Assume dbg.declare can not currently use DIArgList, i.e.
6532 // it is non-variadic.
6533 assert(!DI.hasArgList() && "Only dbg.value should currently use DIArgList");
6534 handleDebugDeclare(DI.getVariableLocationOp(0), Variable, Expression,
6535 DI.getDebugLoc());
6536 return;
6537 }
6538 case Intrinsic::dbg_label: {
6539 const DbgLabelInst &DI = cast<DbgLabelInst>(I);
6540 DILabel *Label = DI.getLabel();
6541 assert(Label && "Missing label");
6542
6543 SDDbgLabel *SDV;
6544 SDV = DAG.getDbgLabel(Label, dl, SDNodeOrder);
6545 DAG.AddDbgLabel(SDV);
6546 return;
6547 }
6548 case Intrinsic::dbg_assign: {
6549 // Debug intrinsics are handled seperately in assignment tracking mode.
6550 if (AssignmentTrackingEnabled)
6551 return;
6552 // If assignment tracking hasn't been enabled then fall through and treat
6553 // the dbg.assign as a dbg.value.
6554 [[fallthrough]];
6555 }
6556 case Intrinsic::dbg_value: {
6557 // Debug intrinsics are handled seperately in assignment tracking mode.
6558 if (AssignmentTrackingEnabled)
6559 return;
6560 const DbgValueInst &DI = cast<DbgValueInst>(I);
6561 assert(DI.getVariable() && "Missing variable");
6562
6563 DILocalVariable *Variable = DI.getVariable();
6566
6567 if (DI.isKillLocation()) {
6568 handleKillDebugValue(Variable, Expression, DI.getDebugLoc(), SDNodeOrder);
6569 return;
6570 }
6571
6573 if (Values.empty())
6574 return;
6575
6576 bool IsVariadic = DI.hasArgList();
6577 if (!handleDebugValue(Values, Variable, Expression, DI.getDebugLoc(),
6578 SDNodeOrder, IsVariadic))
6579 addDanglingDebugInfo(Values, Variable, Expression, IsVariadic,
6580 DI.getDebugLoc(), SDNodeOrder);
6581 return;
6582 }
6583
6584 case Intrinsic::eh_typeid_for: {
6585 // Find the type id for the given typeinfo.
6586 GlobalValue *GV = ExtractTypeInfo(I.getArgOperand(0));
6587 unsigned TypeID = DAG.getMachineFunction().getTypeIDFor(GV);
6588 Res = DAG.getConstant(TypeID, sdl, MVT::i32);
6589 setValue(&I, Res);
6590 return;
6591 }
6592
6593 case Intrinsic::eh_return_i32:
6594 case Intrinsic::eh_return_i64:
6597 MVT::Other,
6599 getValue(I.getArgOperand(0)),
6600 getValue(I.getArgOperand(1))));
6601 return;
6602 case Intrinsic::eh_unwind_init:
6604 return;
6605 case Intrinsic::eh_dwarf_cfa:
6608 getValue(I.getArgOperand(0))));
6609 return;
6610 case Intrinsic::eh_sjlj_callsite: {
6612 ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(0));
6613 assert(MMI.getCurrentCallSite() == 0 && "Overlapping call sites!");
6614
6616 return;
6617 }
6618 case Intrinsic::eh_sjlj_functioncontext: {
6619 // Get and store the index of the function context.
6621 AllocaInst *FnCtx =
6622 cast<AllocaInst>(I.getArgOperand(0)->stripPointerCasts());
6623 int FI = FuncInfo.StaticAllocaMap[FnCtx];
6625 return;
6626 }
6627 case Intrinsic::eh_sjlj_setjmp: {
6628 SDValue Ops[2];
6629 Ops[0] = getRoot();
6630 Ops[1] = getValue(I.getArgOperand(0));
6632 DAG.getVTList(MVT::i32, MVT::Other), Ops);
6633 setValue(&I, Op.getValue(0));
6634 DAG.setRoot(Op.getValue(1));
6635 return;
6636 }
6637 case Intrinsic::eh_sjlj_longjmp:
6638 DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_LONGJMP, sdl, MVT::Other,
6639 getRoot(), getValue(I.getArgOperand(0))));
6640 return;
6641 case Intrinsic::eh_sjlj_setup_dispatch:
6643 getRoot()));
6644 return;
6645 case Intrinsic::masked_gather:
6646 visitMaskedGather(I);
6647 return;
6648 case Intrinsic::masked_load:
6649 visitMaskedLoad(I);
6650 return;
6651 case Intrinsic::masked_scatter:
6652 visitMaskedScatter(I);
6653 return;
6654 case Intrinsic::masked_store:
6655 visitMaskedStore(I);
6656 return;
6657 case Intrinsic::masked_expandload:
6658 visitMaskedLoad(I, true /* IsExpanding */);
6659 return;
6660 case Intrinsic::masked_compressstore:
6661 visitMaskedStore(I, true /* IsCompressing */);
6662 return;
6663 case Intrinsic::powi:
6664 setValue(&I, ExpandPowI(sdl, getValue(I.getArgOperand(0)),
6665 getValue(I.getArgOperand(1)), DAG));
6666 return;
6667 case Intrinsic::log:
6668 setValue(&I, expandLog(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6669 return;
6670 case Intrinsic::log2:
6671 setValue(&I,
6672 expandLog2(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6673 return;
6674 case Intrinsic::log10:
6675 setValue(&I,
6676 expandLog10(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6677 return;
6678 case Intrinsic::exp:
6679 setValue(&I, expandExp(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6680 return;
6681 case Intrinsic::exp2:
6682 setValue(&I,
6683 expandExp2(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6684 return;
6685 case Intrinsic::pow:
6686 setValue(&I, expandPow(sdl, getValue(I.getArgOperand(0)),
6687 getValue(I.getArgOperand(1)), DAG, TLI, Flags));
6688 return;
6689 case Intrinsic::sqrt:
6690 case Intrinsic::fabs:
6691 case Intrinsic::sin:
6692 case Intrinsic::cos:
6693 case Intrinsic::exp10:
6694 case Intrinsic::floor:
6695 case Intrinsic::ceil:
6696 case Intrinsic::trunc:
6697 case Intrinsic::rint:
6698 case Intrinsic::nearbyint:
6699 case Intrinsic::round:
6700 case Intrinsic::roundeven:
6701 case Intrinsic::canonicalize: {
6702 unsigned Opcode;
6703 // clang-format off
6704 switch (Intrinsic) {
6705 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
6706 case Intrinsic::sqrt: Opcode = ISD::FSQRT; break;
6707 case Intrinsic::fabs: Opcode = ISD::FABS; break;
6708 case Intrinsic::sin: Opcode = ISD::FSIN; break;
6709 case Intrinsic::cos: Opcode = ISD::FCOS; break;
6710 case Intrinsic::exp10: Opcode = ISD::FEXP10; break;
6711 case Intrinsic::floor: Opcode = ISD::FFLOOR; break;
6712 case Intrinsic::ceil: Opcode = ISD::FCEIL; break;
6713 case Intrinsic::trunc: Opcode = ISD::FTRUNC; break;
6714 case Intrinsic::rint: Opcode = ISD::FRINT; break;
6715 case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break;
6716 case Intrinsic::round: Opcode = ISD::FROUND; break;
6717 case Intrinsic::roundeven: Opcode = ISD::FROUNDEVEN; break;
6718 case Intrinsic::canonicalize: Opcode = ISD::FCANONICALIZE; break;
6719 }
6720 // clang-format on
6721
6722 setValue(&I, DAG.getNode(Opcode, sdl,
6723 getValue(I.getArgOperand(0)).getValueType(),
6724 getValue(I.getArgOperand(0)), Flags));
6725 return;
6726 }
6727 case Intrinsic::lround:
6728 case Intrinsic::llround:
6729 case Intrinsic::lrint:
6730 case Intrinsic::llrint: {
6731 unsigned Opcode;
6732 // clang-format off
6733 switch (Intrinsic) {
6734 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
6735 case Intrinsic::lround: Opcode = ISD::LROUND; break;
6736 case Intrinsic::llround: Opcode = ISD::LLROUND; break;
6737 case Intrinsic::lrint: Opcode = ISD::LRINT; break;
6738 case Intrinsic::llrint: Opcode = ISD::LLRINT; break;
6739 }
6740 // clang-format on
6741
6742 EVT RetVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6743 setValue(&I, DAG.getNode(Opcode, sdl, RetVT,
6744 getValue(I.getArgOperand(0))));
6745 return;
6746 }
6747 case Intrinsic::minnum:
6749 getValue(I.getArgOperand(0)).getValueType(),
6750 getValue(I.getArgOperand(0)),
6751 getValue(I.getArgOperand(1)), Flags));
6752 return;
6753 case Intrinsic::maxnum:
6755 getValue(I.getArgOperand(0)).getValueType(),
6756 getValue(I.getArgOperand(0)),
6757 getValue(I.getArgOperand(1)), Flags));
6758 return;
6759 case Intrinsic::minimum:
6761 getValue(I.getArgOperand(0)).getValueType(),
6762 getValue(I.getArgOperand(0)),
6763 getValue(I.getArgOperand(1)), Flags));
6764 return;
6765 case Intrinsic::maximum:
6767 getValue(I.getArgOperand(0)).getValueType(),
6768 getValue(I.getArgOperand(0)),
6769 getValue(I.getArgOperand(1)), Flags));
6770 return;
6771 case Intrinsic::copysign:
6773 getValue(I.getArgOperand(0)).getValueType(),
6774 getValue(I.getArgOperand(0)),
6775 getValue(I.getArgOperand(1)), Flags));
6776 return;
6777 case Intrinsic::ldexp:
6779 getValue(I.getArgOperand(0)).getValueType(),
6780 getValue(I.getArgOperand(0)),
6781 getValue(I.getArgOperand(1)), Flags));
6782 return;
6783 case Intrinsic::frexp: {
6784 SmallVector<EVT, 2> ValueVTs;
6785 ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);
6786 SDVTList VTs = DAG.getVTList(ValueVTs);
6787 setValue(&I,
6788 DAG.getNode(ISD::FFREXP, sdl, VTs, getValue(I.getArgOperand(0))));
6789 return;
6790 }
6791 case Intrinsic::arithmetic_fence: {
6793 getValue(I.getArgOperand(0)).getValueType(),
6794 getValue(I.getArgOperand(0)), Flags));
6795 return;
6796 }
6797 case Intrinsic::fma:
6799 ISD::FMA, sdl, getValue(I.getArgOperand(0)).getValueType(),
6800 getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)),
6801 getValue(I.getArgOperand(2)), Flags));
6802 return;
6803#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
6804 case Intrinsic::INTRINSIC:
6805#include "llvm/IR/ConstrainedOps.def"
6806 visitConstrainedFPIntrinsic(cast<ConstrainedFPIntrinsic>(I));
6807 return;
6808#define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) case Intrinsic::VPID:
6809#include "llvm/IR/VPIntrinsics.def"
6810 visitVectorPredicationIntrinsic(cast<VPIntrinsic>(I));
6811 return;
6812 case Intrinsic::fptrunc_round: {
6813 // Get the last argument, the metadata and convert it to an integer in the
6814 // call
6815 Metadata *MD = cast<MetadataAsValue>(I.getArgOperand(1))->getMetadata();
6816 std::optional<RoundingMode> RoundMode =
6817 convertStrToRoundingMode(cast<MDString>(MD)->getString());
6818
6819 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6820
6821 // Propagate fast-math-flags from IR to node(s).
6823 Flags.copyFMF(*cast<FPMathOperator>(&I));
6824 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
6825
6827 Result = DAG.getNode(
6828 ISD::FPTRUNC_ROUND, sdl, VT, getValue(I.getArgOperand(0)),
6829 DAG.getTargetConstant((int)*RoundMode, sdl,
6831 setValue(&I, Result);
6832
6833 return;
6834 }
6835 case Intrinsic::fmuladd: {
6836 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6839 setValue(&I, DAG.getNode(ISD::FMA, sdl,
6840 getValue(I.getArgOperand(0)).getValueType(),
6841 getValue(I.getArgOperand(0)),
6842 getValue(I.getArgOperand(1)),
6843 getValue(I.getArgOperand(2)), Flags));
6844 } else {
6845 // TODO: Intrinsic calls should have fast-math-flags.
6847 ISD::FMUL, sdl, getValue(I.getArgOperand(0)).getValueType(),
6848 getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)), Flags);
6850 getValue(I.getArgOperand(0)).getValueType(),
6851 Mul, getValue(I.getArgOperand(2)), Flags);
6852 setValue(&I, Add);
6853 }
6854 return;
6855 }
6856 case Intrinsic::convert_to_fp16:
6857 setValue(&I, DAG.getNode(ISD::BITCAST, sdl, MVT::i16,
6858 DAG.getNode(ISD::FP_ROUND, sdl, MVT::f16,
6859 getValue(I.getArgOperand(0)),
6860 DAG.getTargetConstant(0, sdl,
6861 MVT::i32))));
6862 return;
6863 case Intrinsic::convert_from_fp16:
6865 TLI.getValueType(DAG.getDataLayout(), I.getType()),
6866 DAG.getNode(ISD::BITCAST, sdl, MVT::f16,
6867 getValue(I.getArgOperand(0)))));
6868 return;
6869 case Intrinsic::fptosi_sat: {
6870 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6872 getValue(I.getArgOperand(0)),
6874 return;
6875 }
6876 case Intrinsic::fptoui_sat: {
6877 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6879 getValue(I.getArgOperand(0)),
6881 return;
6882 }
6883 case Intrinsic::set_rounding:
6884 Res = DAG.getNode(ISD::SET_ROUNDING, sdl, MVT::Other,
6885 {getRoot(), getValue(I.getArgOperand(0))});
6886 setValue(&I, Res);
6887 DAG.setRoot(Res.getValue(0));
6888 return;
6889 case Intrinsic::is_fpclass: {
6890 const DataLayout DLayout = DAG.getDataLayout();
6891 EVT DestVT = TLI.getValueType(DLayout, I.getType());
6892 EVT ArgVT = TLI.getValueType(DLayout, I.getArgOperand(0)->getType());
6893 FPClassTest Test = static_cast<FPClassTest>(
6894 cast<ConstantInt>(I.getArgOperand(1))->getZExtValue());
6896 const Function &F = MF.getFunction();
6897 SDValue Op = getValue(I.getArgOperand(0));
6899 Flags.setNoFPExcept(
6900 !F.getAttributes().hasFnAttr(llvm::Attribute::StrictFP));
6901 // If ISD::IS_FPCLASS should be expanded, do it right now, because the
6902 // expansion can use illegal types. Making expansion early allows
6903 // legalizing these types prior to selection.
6904 if (!TLI.isOperationLegalOrCustom(ISD::IS_FPCLASS, ArgVT)) {
6905 SDValue Result = TLI.expandIS_FPCLASS(DestVT, Op, Test, Flags, sdl, DAG);
6906 setValue(&I, Result);
6907 return;
6908 }
6909
6910 SDValue Check = DAG.getTargetConstant(Test, sdl, MVT::i32);
6911 SDValue V = DAG.getNode(ISD::IS_FPCLASS, sdl, DestVT, {Op, Check}, Flags);
6912 setValue(&I, V);
6913 return;
6914 }
6915 case Intrinsic::get_fpenv: {
6916 const DataLayout DLayout = DAG.getDataLayout();
6917 EVT EnvVT = TLI.getValueType(DLayout, I.getType());
6918 Align TempAlign = DAG.getEVTAlign(EnvVT);
6919 SDValue Chain = getRoot();
6920 // Use GET_FPENV if it is legal or custom. Otherwise use memory-based node
6921 // and temporary storage in stack.
6922 if (TLI.isOperationLegalOrCustom(ISD::GET_FPENV, EnvVT)) {
6923 Res = DAG.getNode(
6924 ISD::GET_FPENV, sdl,
6925 DAG.getVTList(TLI.getValueType(DAG.getDataLayout(), I.getType()),
6926 MVT::Other),
6927 Chain);
6928 } else {
6929 SDValue Temp = DAG.CreateStackTemporary(EnvVT, TempAlign.value());
6930 int SPFI = cast<FrameIndexSDNode>(Temp.getNode())->getIndex();
6931 auto MPI =
6935 TempAlign);
6936 Chain = DAG.getGetFPEnv(Chain, sdl, Temp, EnvVT, MMO);
6937 Res = DAG.getLoad(EnvVT, sdl, Chain, Temp, MPI);
6938 }
6939 setValue(&I, Res);
6940 DAG.setRoot(Res.getValue(1));
6941 return;
6942 }
6943 case Intrinsic::set_fpenv: {
6944 const DataLayout DLayout = DAG.getDataLayout();
6945 SDValue Env = getValue(I.getArgOperand(0));
6946 EVT EnvVT = Env.getValueType();
6947 Align TempAlign = DAG.getEVTAlign(EnvVT);
6948 SDValue Chain = getRoot();
6949 // If SET_FPENV is custom or legal, use it. Otherwise use loading
6950 // environment from memory.
6951 if (TLI.isOperationLegalOrCustom(ISD::SET_FPENV, EnvVT)) {
6952 Chain = DAG.getNode(ISD::SET_FPENV, sdl, MVT::Other, Chain, Env);
6953 } else {
6954 // Allocate space in stack, copy environment bits into it and use this
6955 // memory in SET_FPENV_MEM.
6956 SDValue Temp = DAG.CreateStackTemporary(EnvVT, TempAlign.value());
6957 int SPFI = cast<FrameIndexSDNode>(Temp.getNode())->getIndex();
6958 auto MPI =
6960 Chain = DAG.getStore(Chain, sdl, Env, Temp, MPI, TempAlign,
6964 TempAlign);
6965 Chain = DAG.getSetFPEnv(Chain, sdl, Temp, EnvVT, MMO);
6966 }
6967 DAG.setRoot(Chain);
6968 return;
6969 }
6970 case Intrinsic::reset_fpenv:
6971 DAG.setRoot(DAG.getNode(ISD::RESET_FPENV, sdl, MVT::Other, getRoot()));
6972 return;
6973 case Intrinsic::get_fpmode:
6974 Res = DAG.getNode(
6975 ISD::GET_FPMODE, sdl,
6976 DAG.getVTList(TLI.getValueType(DAG.getDataLayout(), I.getType()),
6977 MVT::Other),
6978 DAG.getRoot());
6979 setValue(&I, Res);
6980 DAG.setRoot(Res.getValue(1));
6981 return;
6982 case Intrinsic::set_fpmode:
6983 Res = DAG.getNode(ISD::SET_FPMODE, sdl, MVT::Other, {DAG.getRoot()},
6984 getValue(I.getArgOperand(0)));
6985 DAG.setRoot(Res);
6986 return;
6987 case Intrinsic::reset_fpmode: {
6988 Res = DAG.getNode(ISD::RESET_FPMODE, sdl, MVT::Other, getRoot());
6989 DAG.setRoot(Res);
6990 return;
6991 }
6992 case Intrinsic::pcmarker: {
6993 SDValue Tmp = getValue(I.getArgOperand(0));
6994 DAG.setRoot(DAG.getNode(ISD::PCMARKER, sdl, MVT::Other, getRoot(), Tmp));
6995 return;
6996 }
6997 case Intrinsic::readcyclecounter: {
6998 SDValue Op = getRoot();
7000 DAG.getVTList(MVT::i64, MVT::Other), Op);
7001 setValue(&I, Res);
7002 DAG.setRoot(Res.getValue(1));
7003 return;
7004 }
7005 case Intrinsic::readsteadycounter: {
7006 SDValue Op = getRoot();
7008 DAG.getVTList(MVT::i64, MVT::Other), Op);
7009 setValue(&I, Res);
7010 DAG.setRoot(Res.getValue(1));
7011 return;
7012 }
7013 case Intrinsic::bitreverse:
7015 getValue(I.getArgOperand(0)).getValueType(),
7016 getValue(I.getArgOperand(0))));
7017 return;
7018 case Intrinsic::bswap:
7020 getValue(I.getArgOperand(0)).getValueType(),
7021 getValue(I.getArgOperand(0))));
7022 return;
7023 case Intrinsic::cttz: {
7024 SDValue Arg = getValue(I.getArgOperand(0));
7025 ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
7026 EVT Ty = Arg.getValueType();
7028 sdl, Ty, Arg));
7029 return;
7030 }
7031 case Intrinsic::ctlz: {
7032 SDValue Arg = getValue(I.getArgOperand(0));
7033 ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
7034 EVT Ty = Arg.getValueType();
7036 sdl, Ty, Arg));
7037 return;
7038 }
7039 case Intrinsic::ctpop: {
7040 SDValue Arg = getValue(I.getArgOperand(0));
7041 EVT Ty = Arg.getValueType();
7042 setValue(&I, DAG.getNode(ISD::CTPOP, sdl, Ty, Arg));
7043 return;
7044 }
7045 case Intrinsic::fshl:
7046 case Intrinsic::fshr: {
7047 bool IsFSHL = Intrinsic == Intrinsic::fshl;
7048 SDValue X = getValue(I.getArgOperand(0));
7049 SDValue Y = getValue(I.getArgOperand(1));
7050 SDValue Z = getValue(I.getArgOperand(2));
7051 EVT VT = X.getValueType();
7052
7053 if (X == Y) {
7054 auto RotateOpcode = IsFSHL ? ISD::ROTL : ISD::ROTR;
7055 setValue(&I, DAG.getNode(RotateOpcode, sdl, VT, X, Z));
7056 } else {
7057 auto FunnelOpcode = IsFSHL ? ISD::FSHL : ISD::FSHR;
7058 setValue(&I, DAG.getNode(FunnelOpcode, sdl, VT, X, Y, Z));
7059 }
7060 return;
7061 }
7062 case Intrinsic::sadd_sat: {
7063 SDValue Op1 = getValue(I.getArgOperand(0));
7064 SDValue Op2 = getValue(I.getArgOperand(1));
7065 setValue(&I, DAG.getNode(ISD::SADDSAT, sdl, Op1.getValueType(), Op1, Op2));
7066 return;
7067 }
7068 case Intrinsic::uadd_sat: {
7069 SDValue Op1 = getValue(I.getArgOperand(0));
7070 SDValue Op2 = getValue(I.getArgOperand(1));
7071 setValue(&I, DAG.getNode(ISD::UADDSAT, sdl, Op1.getValueType(), Op1, Op2));
7072 return;
7073 }
7074 case Intrinsic::ssub_sat: {
7075 SDValue Op1 = getValue(I.getArgOperand(0));
7076 SDValue Op2 = getValue(I.getArgOperand(1));
7077 setValue(&I, DAG.getNode(ISD::SSUBSAT, sdl, Op1.getValueType(), Op1, Op2));
7078 return;
7079 }
7080 case Intrinsic::usub_sat: {
7081 SDValue Op1 = getValue(I.getArgOperand(0));
7082 SDValue Op2 = getValue(I.getArgOperand(1));
7083 setValue(&I, DAG.getNode(ISD::USUBSAT, sdl, Op1.getValueType(), Op1, Op2));
7084 return;
7085 }
7086 case Intrinsic::sshl_sat: {
7087 SDValue Op1 = getValue(I.getArgOperand(0));
7088 SDValue Op2 = getValue(I.getArgOperand(1));
7089 setValue(&I, DAG.getNode(ISD::SSHLSAT, sdl, Op1.getValueType(), Op1, Op2));
7090 return;
7091 }
7092 case Intrinsic::ushl_sat: {
7093 SDValue Op1 = getValue(I.getArgOperand(0));
7094 SDValue Op2 = getValue(I.getArgOperand(1));
7095 setValue(&I, DAG.getNode(ISD::USHLSAT, sdl, Op1.getValueType(), Op1, Op2));
7096 return;
7097 }
7098 case Intrinsic::smul_fix:
7099 case Intrinsic::umul_fix:
7100 case Intrinsic::smul_fix_sat:
7101 case Intrinsic::umul_fix_sat: {
7102 SDValue Op1 = getValue(I.getArgOperand(0));
7103 SDValue Op2 = getValue(I.getArgOperand(1));
7104 SDValue Op3 = getValue(I.getArgOperand(2));
7106 Op1.getValueType(), Op1, Op2, Op3));
7107 return;
7108 }
7109 case Intrinsic::sdiv_fix:
7110 case Intrinsic::udiv_fix:
7111 case Intrinsic::sdiv_fix_sat:
7112 case Intrinsic::udiv_fix_sat: {
7113 SDValue Op1 = getValue(I.getArgOperand(0));
7114 SDValue Op2 = getValue(I.getArgOperand(1));
7115 SDValue Op3 = getValue(I.getArgOperand(2));
7117 Op1, Op2, Op3, DAG, TLI));
7118 return;
7119 }
7120 case Intrinsic::smax: {
7121 SDValue Op1 = getValue(I.getArgOperand(0));
7122 SDValue Op2 = getValue(I.getArgOperand(1));
7123 setValue(&I, DAG.getNode(ISD::SMAX, sdl, Op1.getValueType(), Op1, Op2));
7124 return;
7125 }
7126 case Intrinsic::smin: {
7127 SDValue Op1 = getValue(I.getArgOperand(0));
7128 SDValue Op2 = getValue(I.getArgOperand(1));
7129 setValue(&I, DAG.getNode(ISD::SMIN, sdl, Op1.getValueType(), Op1, Op2));
7130 return;
7131 }
7132 case Intrinsic::umax: {
7133 SDValue Op1 = getValue(I.getArgOperand(0));
7134 SDValue Op2 = getValue(I.getArgOperand(1));
7135 setValue(&I, DAG.getNode(ISD::UMAX, sdl, Op1.getValueType(), Op1, Op2));
7136 return;
7137 }
7138 case Intrinsic::umin: {
7139 SDValue Op1 = getValue(I.getArgOperand(0));
7140 SDValue Op2 = getValue(I.getArgOperand(1));
7141 setValue(&I, DAG.getNode(ISD::UMIN, sdl, Op1.getValueType(), Op1, Op2));
7142 return;
7143 }
7144 case Intrinsic::abs: {
7145 // TODO: Preserve "int min is poison" arg in SDAG?
7146 SDValue Op1 = getValue(I.getArgOperand(0));
7147 setValue(&I, DAG.getNode(ISD::ABS, sdl, Op1.getValueType(), Op1));
7148 return;
7149 }
7150 case Intrinsic::stacksave: {
7151 SDValue Op = getRoot();
7152 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7153 Res = DAG.getNode(ISD::STACKSAVE, sdl, DAG.getVTList(VT, MVT::Other), Op);
7154 setValue(&I, Res);
7155 DAG.setRoot(Res.getValue(1));
7156 return;
7157 }
7158 case Intrinsic::stackrestore:
7159 Res = getValue(I.getArgOperand(0));
7160 DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, sdl, MVT::Other, getRoot(), Res));
7161 return;
7162 case Intrinsic::get_dynamic_area_offset: {
7163 SDValue Op = getRoot();
7164 EVT PtrTy = TLI.getFrameIndexTy(DAG.getDataLayout());
7165 EVT ResTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
7166 // Result type for @llvm.get.dynamic.area.offset should match PtrTy for
7167 // target.
7168 if (PtrTy.getFixedSizeInBits() < ResTy.getFixedSizeInBits())
7169 report_fatal_error("Wrong result type for @llvm.get.dynamic.area.offset"
7170 " intrinsic!");
7172 Op);
7173 DAG.setRoot(Op);
7174 setValue(&I, Res);
7175 return;
7176 }
7177 case Intrinsic::stackguard: {
7179 const Module &M = *MF.getFunction().getParent();
7180 EVT PtrTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
7181 SDValue Chain = getRoot();
7182 if (TLI.useLoadStackGuardNode()) {
7183 Res = getLoadStackGuard(DAG, sdl, Chain);
7184 Res = DAG.getPtrExtOrTrunc(Res, sdl, PtrTy);
7185 } else {
7186 const Value *Global = TLI.getSDagStackGuard(M);
7188 Res = DAG.getLoad(PtrTy, sdl, Chain, getValue(Global),
7191 }
7192 if (TLI.useStackGuardXorFP())
7193 Res = TLI.emitStackGuardXorFP(DAG, Res, sdl);
7194 DAG.setRoot(Chain);
7195 setValue(&I, Res);
7196 return;
7197 }
7198 case Intrinsic::stackprotector: {
7199 // Emit code into the DAG to store the stack guard onto the stack.
7201 MachineFrameInfo &MFI = MF.getFrameInfo();
7202 SDValue Src, Chain = getRoot();
7203
7204 if (TLI.useLoadStackGuardNode())
7205 Src = getLoadStackGuard(DAG, sdl, Chain);
7206 else
7207 Src = getValue(I.getArgOperand(0)); // The guard's value.
7208
7209 AllocaInst *Slot = cast<AllocaInst>(I.getArgOperand(1));
7210
7211 int FI = FuncInfo.StaticAllocaMap[Slot];
7212 MFI.setStackProtectorIndex(FI);
7213 EVT PtrTy = TLI.getFrameIndexTy(DAG.getDataLayout());
7214
7215 SDValue FIN = DAG.getFrameIndex(FI, PtrTy);
7216
7217 // Store the stack protector onto the stack.
7218 Res = DAG.getStore(
7219 Chain, sdl, Src, FIN,
7222 setValue(&I, Res);
7223 DAG.setRoot(Res);
7224 return;
7225 }
7226 case Intrinsic::objectsize:
7227 llvm_unreachable("llvm.objectsize.* should have been lowered already");
7228
7229 case Intrinsic::is_constant:
7230 llvm_unreachable("llvm.is.constant.* should have been lowered already");
7231
7232 case Intrinsic::annotation:
7233 case Intrinsic::ptr_annotation:
7234 case Intrinsic::launder_invariant_group:
7235 case Intrinsic::strip_invariant_group:
7236 // Drop the intrinsic, but forward the value
7237 setValue(&I, getValue(I.getOperand(0)));
7238 return;
7239
7240 case Intrinsic::assume:
7241 case Intrinsic::experimental_noalias_scope_decl:
7242 case Intrinsic::var_annotation:
7243 case Intrinsic::sideeffect:
7244 // Discard annotate attributes, noalias scope declarations, assumptions, and
7245 // artificial side-effects.
7246 return;
7247
7248 case Intrinsic::codeview_annotation: {
7249 // Emit a label associated with this metadata.
7251 MCSymbol *Label =
7252 MF.getMMI().getContext().createTempSymbol("annotation", true);
7253 Metadata *MD = cast<MetadataAsValue>(I.getArgOperand(0))->getMetadata();
7254 MF.addCodeViewAnnotation(Label, cast<MDNode>(MD));
7255 Res = DAG.getLabelNode(ISD::ANNOTATION_LABEL, sdl, getRoot(), Label);
7256 DAG.setRoot(Res);
7257 return;
7258 }
7259
7260 case Intrinsic::init_trampoline: {
7261 const Function *F = cast<Function>(I.getArgOperand(1)->stripPointerCasts());
7262
7263 SDValue Ops[6];
7264 Ops[0] = getRoot();
7265 Ops[1] = getValue(I.getArgOperand(0));
7266 Ops[2] = getValue(I.getArgOperand(1));
7267 Ops[3] = getValue(I.getArgOperand(2));
7268 Ops[4] = DAG.getSrcValue(I.getArgOperand(0));
7269 Ops[5] = DAG.getSrcValue(F);
7270
7271 Res = DAG.getNode(ISD::INIT_TRAMPOLINE, sdl, MVT::Other, Ops);
7272
7273 DAG.setRoot(Res);
7274 return;
7275 }
7276 case Intrinsic::adjust_trampoline:
7279 getValue(I.getArgOperand(0))));
7280 return;
7281 case Intrinsic::gcroot: {
7283 "only valid in functions with gc specified, enforced by Verifier");
7284 assert(GFI && "implied by previous");
7285 const Value *Alloca = I.getArgOperand(0)->stripPointerCasts();
7286 const Constant *TypeMap = cast<Constant>(I.getArgOperand(1));
7287
7288 FrameIndexSDNode *FI = cast<FrameIndexSDNode>(getValue(Alloca).getNode());
7289 GFI->addStackRoot(FI->getIndex(), TypeMap);
7290 return;
7291 }
7292 case Intrinsic::gcread:
7293 case Intrinsic::gcwrite:
7294 llvm_unreachable("GC failed to lower gcread/gcwrite intrinsics!");
7295 case Intrinsic::get_rounding:
7296 Res = DAG.getNode(ISD::GET_ROUNDING, sdl, {MVT::i32, MVT::Other}, getRoot());
7297 setValue(&I, Res);
7298 DAG.setRoot(Res.getValue(1));
7299 return;
7300
7301 case Intrinsic::expect:
7302 // Just replace __builtin_expect(exp, c) with EXP.
7303 setValue(&I, getValue(I.getArgOperand(0)));
7304 return;
7305
7306 case Intrinsic::ubsantrap:
7307 case Intrinsic::debugtrap:
7308 case Intrinsic::trap: {
7309 StringRef TrapFuncName =
7310 I.getAttributes().getFnAttr("trap-func-name").getValueAsString();
7311 if (TrapFuncName.empty()) {
7312 switch (Intrinsic) {
7313 case Intrinsic::trap:
7314 DAG.setRoot(DAG.getNode(ISD::TRAP, sdl, MVT::Other, getRoot()));
7315 break;
7316 case Intrinsic::debugtrap:
7317 DAG.setRoot(DAG.getNode(ISD::DEBUGTRAP, sdl, MVT::Other, getRoot()));
7318 break;
7319 case Intrinsic::ubsantrap:
7321 ISD::UBSANTRAP, sdl, MVT::Other, getRoot(),
7323 cast<ConstantInt>(I.getArgOperand(0))->getZExtValue(), sdl,
7324 MVT::i32)));
7325 break;
7326 default: llvm_unreachable("unknown trap intrinsic");
7327 }
7328 return;
7329 }
7331 if (Intrinsic == Intrinsic::ubsantrap) {
7333 Args[0].Val = I.getArgOperand(0);
7334 Args[0].Node = getValue(Args[0].Val);
7335 Args[0].Ty = Args[0].Val->getType();
7336 }
7337
7339 CLI.setDebugLoc(sdl).setChain(getRoot()).setLibCallee(
7340 CallingConv::C, I.getType(),
7341 DAG.getExternalSymbol(TrapFuncName.data(),
7343 std::move(Args));
7344
7345 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
7346 DAG.setRoot(Result.second);
7347 return;
7348 }
7349
7350 case Intrinsic::allow_runtime_check:
7351 case Intrinsic::allow_ubsan_check:
7352 setValue(&I, getValue(ConstantInt::getTrue(I.getType())));
7353 return;
7354
7355 case Intrinsic::uadd_with_overflow:
7356 case Intrinsic::sadd_with_overflow:
7357 case Intrinsic::usub_with_overflow:
7358 case Intrinsic::ssub_with_overflow:
7359 case Intrinsic::umul_with_overflow:
7360 case Intrinsic::smul_with_overflow: {
7362 switch (Intrinsic) {
7363 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
7364 case Intrinsic::uadd_with_overflow: Op = ISD::UADDO; break;
7365 case Intrinsic::sadd_with_overflow: Op = ISD::SADDO; break;
7366 case Intrinsic::usub_with_overflow: Op = ISD::USUBO; break;
7367 case Intrinsic::ssub_with_overflow: Op = ISD::SSUBO; break;
7368 case Intrinsic::umul_with_overflow: Op = ISD::UMULO; break;
7369 case Intrinsic::smul_with_overflow: Op = ISD::SMULO; break;
7370 }
7371 SDValue Op1 = getValue(I.getArgOperand(0));
7372 SDValue Op2 = getValue(I.getArgOperand(1));
7373
7374 EVT ResultVT = Op1.getValueType();
7375 EVT OverflowVT = MVT::i1;
7376 if (ResultVT.isVector())
7377 OverflowVT = EVT::getVectorVT(
7378 *Context, OverflowVT, ResultVT.getVectorElementCount());
7379
7380 SDVTList VTs = DAG.getVTList(ResultVT, OverflowVT);
7381 setValue(&I, DAG.getNode(Op, sdl, VTs, Op1, Op2));
7382 return;
7383 }
7384 case Intrinsic::prefetch: {
7385 SDValue Ops[5];
7386 unsigned rw = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
7388 Ops[0] = DAG.getRoot();
7389 Ops[1] = getValue(I.getArgOperand(0));
7390 Ops[2] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(1)), sdl,
7391 MVT::i32);
7392 Ops[3] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(2)), sdl,
7393 MVT::i32);
7394 Ops[4] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(3)), sdl,
7395 MVT::i32);
7397 ISD::PREFETCH, sdl, DAG.getVTList(MVT::Other), Ops,
7398 EVT::getIntegerVT(*Context, 8), MachinePointerInfo(I.getArgOperand(0)),
7399 /* align */ std::nullopt, Flags);
7400
7401 // Chain the prefetch in parallel with any pending loads, to stay out of
7402 // the way of later optimizations.
7403 PendingLoads.push_back(Result);
7404 Result = getRoot();
7405 DAG.setRoot(Result);
7406 return;
7407 }
7408 case Intrinsic::lifetime_start:
7409 case Intrinsic::lifetime_end: {
7410 bool IsStart = (Intrinsic == Intrinsic::lifetime_start);
7411 // Stack coloring is not enabled in O0, discard region information.
7413 return;
7414
7415 const int64_t ObjectSize =
7416 cast<ConstantInt>(I.getArgOperand(0))->getSExtValue();
7417 Value *const ObjectPtr = I.getArgOperand(1);
7419 getUnderlyingObjects(ObjectPtr, Allocas);
7420
7421 for (const Value *Alloca : Allocas) {
7422 const AllocaInst *LifetimeObject = dyn_cast_or_null<AllocaInst>(Alloca);
7423
7424 // Could not find an Alloca.
7425 if (!LifetimeObject)
7426 continue;
7427
7428 // First check that the Alloca is static, otherwise it won't have a
7429 // valid frame index.
7430 auto SI = FuncInfo.StaticAllocaMap.find(LifetimeObject);
7431 if (SI == FuncInfo.StaticAllocaMap.end())
7432 return;
7433
7434 const int FrameIndex = SI->second;
7435 int64_t Offset;
7437 ObjectPtr, Offset, DAG.getDataLayout()) != LifetimeObject)
7438 Offset = -1; // Cannot determine offset from alloca to lifetime object.
7439 Res = DAG.getLifetimeNode(IsStart, sdl, getRoot(), FrameIndex, ObjectSize,
7440 Offset);
7441 DAG.setRoot(Res);
7442 }
7443 return;
7444 }
7445 case Intrinsic::pseudoprobe: {
7446 auto Guid = cast<ConstantInt>(I.getArgOperand(0))->getZExtValue();
7447 auto Index = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
7448 auto Attr = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue();
7449 Res = DAG.getPseudoProbeNode(sdl, getRoot(), Guid, Index, Attr);
7450 DAG.setRoot(Res);
7451 return;
7452 }
7453 case Intrinsic::invariant_start:
7454 // Discard region information.
7455 setValue(&I,
7456 DAG.getUNDEF(TLI.getValueType(DAG.getDataLayout(), I.getType())));
7457 return;
7458 case Intrinsic::invariant_end:
7459 // Discard region information.
7460 return;
7461 case Intrinsic::clear_cache:
7462 /// FunctionName may be null.
7463 if (const char *FunctionName = TLI.getClearCacheBuiltinName())
7464 lowerCallToExternalSymbol(I, FunctionName);
7465 return;
7466 case Intrinsic::donothing:
7467 case Intrinsic::seh_try_begin:
7468 case Intrinsic::seh_scope_begin:
7469 case Intrinsic::seh_try_end:
7470 case Intrinsic::seh_scope_end:
7471 // ignore
7472 return;
7473 case Intrinsic::experimental_stackmap:
7474 visitStackmap(I);
7475 return;
7476 case Intrinsic::experimental_patchpoint_void:
7477 case Intrinsic::experimental_patchpoint:
7478 visitPatchpoint(I);
7479 return;
7480 case Intrinsic::experimental_gc_statepoint:
7481 LowerStatepoint(cast<GCStatepointInst>(I));
7482 return;
7483 case Intrinsic::experimental_gc_result:
7484 visitGCResult(cast<GCResultInst>(I));
7485 return;
7486 case Intrinsic::experimental_gc_relocate:
7487 visitGCRelocate(cast<GCRelocateInst>(I));
7488 return;
7489 case Intrinsic::instrprof_cover:
7490 llvm_unreachable("instrprof failed to lower a cover");
7491 case Intrinsic::instrprof_increment:
7492 llvm_unreachable("instrprof failed to lower an increment");
7493 case Intrinsic::instrprof_timestamp:
7494 llvm_unreachable("instrprof failed to lower a timestamp");
7495 case Intrinsic::instrprof_value_profile:
7496 llvm_unreachable("instrprof failed to lower a value profiling call");
7497 case Intrinsic::instrprof_mcdc_parameters:
7498 llvm_unreachable("instrprof failed to lower mcdc parameters");
7499 case Intrinsic::instrprof_mcdc_tvbitmap_update:
7500 llvm_unreachable("instrprof failed to lower an mcdc tvbitmap update");
7501 case Intrinsic::instrprof_mcdc_condbitmap_update:
7502 llvm_unreachable("instrprof failed to lower an mcdc condbitmap update");
7503 case Intrinsic::localescape: {
7506
7507 // Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission
7508 // is the same on all targets.
7509 for (unsigned Idx = 0, E = I.arg_size(); Idx < E; ++Idx) {
7510 Value *Arg = I.getArgOperand(Idx)->stripPointerCasts();
7511 if (isa<ConstantPointerNull>(Arg))
7512 continue; // Skip null pointers. They represent a hole in index space.
7513 AllocaInst *Slot = cast<AllocaInst>(Arg);
7514 assert(FuncInfo.StaticAllocaMap.count(Slot) &&
7515 "can only escape static allocas");
7516 int FI = FuncInfo.StaticAllocaMap[Slot];
7517 MCSymbol *FrameAllocSym =
7521 TII->get(TargetOpcode::LOCAL_ESCAPE))
7522 .addSym(FrameAllocSym)
7523 .addFrameIndex(FI);
7524 }
7525
7526 return;
7527 }
7528
7529 case Intrinsic::localrecover: {
7530 // i8* @llvm.localrecover(i8* %fn, i8* %fp, i32 %idx)
7532
7533 // Get the symbol that defines the frame offset.
7534 auto *Fn = cast<Function>(I.getArgOperand(0)->stripPointerCasts());
7535 auto *Idx = cast<ConstantInt>(I.getArgOperand(2));
7536 unsigned IdxVal =
7537 unsigned(Idx->getLimitedValue(std::numeric_limits<int>::max()));
7538 MCSymbol *FrameAllocSym =
7541
7542 Value *FP = I.getArgOperand(1);
7543 SDValue FPVal = getValue(FP);
7544 EVT PtrVT = FPVal.getValueType();
7545
7546 // Create a MCSymbol for the label to avoid any target lowering
7547 // that would make this PC relative.
7548 SDValue OffsetSym = DAG.getMCSymbol(FrameAllocSym, PtrVT);
7549 SDValue OffsetVal =
7550 DAG.getNode(ISD::LOCAL_RECOVER, sdl, PtrVT, OffsetSym);
7551
7552 // Add the offset to the FP.
7553 SDValue Add = DAG.getMemBasePlusOffset(FPVal, OffsetVal, sdl);
7554 setValue(&I, Add);
7555
7556 return;
7557 }
7558
7559 case Intrinsic::eh_exceptionpointer:
7560 case Intrinsic::eh_exceptioncode: {
7561 // Get the exception pointer vreg, copy from it, and resize it to fit.
7562 const auto *CPI = cast<CatchPadInst>(I.getArgOperand(0));
7563 MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
7564 const TargetRegisterClass *PtrRC = TLI.getRegClassFor(PtrVT);
7565 unsigned VReg = FuncInfo.getCatchPadExceptionPointerVReg(CPI, PtrRC);
7566 SDValue N = DAG.getCopyFromReg(DAG.getEntryNode(), sdl, VReg, PtrVT);
7567 if (Intrinsic == Intrinsic::eh_exceptioncode)
7568 N = DAG.getZExtOrTrunc(N, sdl, MVT::i32);
7569 setValue(&I, N);
7570 return;
7571 }
7572 case Intrinsic::xray_customevent: {
7573 // Here we want to make sure that the intrinsic behaves as if it has a
7574 // specific calling convention.
7575 const auto &Triple = DAG.getTarget().getTargetTriple();
7576 if (!Triple.isAArch64(64) && Triple.getArch() != Triple::x86_64)
7577 return;
7578
7580
7581 // We want to say that we always want the arguments in registers.
7582 SDValue LogEntryVal = getValue(I.getArgOperand(0));
7583 SDValue StrSizeVal = getValue(I.getArgOperand(1));
7584 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
7585 SDValue Chain = getRoot();
7586 Ops.push_back(LogEntryVal);
7587 Ops.push_back(StrSizeVal);
7588 Ops.push_back(Chain);
7589
7590 // We need to enforce the calling convention for the callsite, so that
7591 // argument ordering is enforced correctly, and that register allocation can
7592 // see that some registers may be assumed clobbered and have to preserve
7593 // them across calls to the intrinsic.
7594 MachineSDNode *MN = DAG.getMachineNode(TargetOpcode::PATCHABLE_EVENT_CALL,
7595 sdl, NodeTys, Ops);
7596 SDValue patchableNode = SDValue(MN, 0);
7597 DAG.setRoot(patchableNode);
7598 setValue(&I, patchableNode);
7599 return;
7600 }
7601 case Intrinsic::xray_typedevent: {
7602 // Here we want to make sure that the intrinsic behaves as if it has a
7603 // specific calling convention.
7604 const auto &Triple = DAG.getTarget().getTargetTriple();
7605 if (!Triple.isAArch64(64) && Triple.getArch() != Triple::x86_64)
7606 return;
7607
7609
7610 // We want to say that we always want the arguments in registers.
7611 // It's unclear to me how manipulating the selection DAG here forces callers
7612 // to provide arguments in registers instead of on the stack.
7613 SDValue LogTypeId = getValue(I.getArgOperand(0));
7614 SDValue LogEntryVal = getValue(I.getArgOperand(1));
7615 SDValue StrSizeVal = getValue(I.getArgOperand(2));
7616 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
7617 SDValue Chain = getRoot();
7618 Ops.push_back(LogTypeId);
7619 Ops.push_back(LogEntryVal);
7620 Ops.push_back(StrSizeVal);
7621 Ops.push_back(Chain);
7622
7623 // We need to enforce the calling convention for the callsite, so that
7624 // argument ordering is enforced correctly, and that register allocation can
7625 // see that some registers may be assumed clobbered and have to preserve
7626 // them across calls to the intrinsic.
7628 TargetOpcode::PATCHABLE_TYPED_EVENT_CALL, sdl, NodeTys, Ops);
7629 SDValue patchableNode = SDValue(MN, 0);
7630 DAG.setRoot(patchableNode);
7631 setValue(&I, patchableNode);
7632 return;
7633 }
7634 case Intrinsic::experimental_deoptimize:
7636 return;
7637 case Intrinsic::experimental_stepvector:
7638 visitStepVector(I);
7639 return;
7640 case Intrinsic::vector_reduce_fadd:
7641 case Intrinsic::vector_reduce_fmul:
7642 case Intrinsic::vector_reduce_add:
7643 case Intrinsic::vector_reduce_mul:
7644 case Intrinsic::vector_reduce_and:
7645 case Intrinsic::vector_reduce_or:
7646 case Intrinsic::vector_reduce_xor:
7647 case Intrinsic::vector_reduce_smax:
7648 case Intrinsic::vector_reduce_smin:
7649 case Intrinsic::vector_reduce_umax:
7650 case Intrinsic::vector_reduce_umin:
7651 case Intrinsic::vector_reduce_fmax:
7652 case Intrinsic::vector_reduce_fmin:
7653 case Intrinsic::vector_reduce_fmaximum:
7654 case Intrinsic::vector_reduce_fminimum:
7655 visitVectorReduce(I, Intrinsic);
7656 return;
7657
7658 case Intrinsic::icall_branch_funnel: {
7660 Ops.push_back(getValue(I.getArgOperand(0)));
7661
7662 int64_t Offset;
7663 auto *Base = dyn_cast<GlobalObject>(GetPointerBaseWithConstantOffset(
7664 I.getArgOperand(1), Offset, DAG.getDataLayout()));
7665 if (!Base)
7667 "llvm.icall.branch.funnel operand must be a GlobalValue");
7668 Ops.push_back(DAG.getTargetGlobalAddress(Base, sdl, MVT::i64, 0));
7669
7670 struct BranchFunnelTarget {
7671 int64_t Offset;
7673 };
7675
7676 for (unsigned Op = 1, N = I.arg_size(); Op != N; Op += 2) {
7677 auto *ElemBase = dyn_cast<GlobalObject>(GetPointerBaseWithConstantOffset(
7678 I.getArgOperand(Op), Offset, DAG.getDataLayout()));
7679 if (ElemBase != Base)
7680 report_fatal_error("all llvm.icall.branch.funnel operands must refer "
7681 "to the same GlobalValue");
7682
7683 SDValue Val = getValue(I.getArgOperand(Op + 1));
7684 auto *GA = dyn_cast<GlobalAddressSDNode>(Val);
7685 if (!GA)
7687 "llvm.icall.branch.funnel operand must be a GlobalValue");
7689 GA->getGlobal(), sdl, Val.getValueType(),
7690 GA->getOffset())});
7691 }
7692 llvm::sort(Targets,
7693 [](const BranchFunnelTarget &T1, const BranchFunnelTarget &T2) {
7694 return T1.Offset < T2.Offset;
7695 });
7696
7697 for (auto &T : Targets) {
7698 Ops.push_back(DAG.getTargetConstant(T.Offset, sdl, MVT::i32));
7699 Ops.push_back(T.Target);
7700 }
7701
7702 Ops.push_back(DAG.getRoot()); // Chain
7703 SDValue N(DAG.getMachineNode(TargetOpcode::ICALL_BRANCH_FUNNEL, sdl,
7704 MVT::Other, Ops),
7705 0);
7706 DAG.setRoot(N);
7707 setValue(&I, N);
7708 HasTailCall = true;
7709 return;
7710 }
7711
7712 case Intrinsic::wasm_landingpad_index:
7713 // Information this intrinsic contained has been transferred to
7714 // MachineFunction in SelectionDAGISel::PrepareEHLandingPad. We can safely
7715 // delete it now.
7716 return;
7717
7718 case Intrinsic::aarch64_settag:
7719 case Intrinsic::aarch64_settag_zero: {
7721 bool ZeroMemory = Intrinsic == Intrinsic::aarch64_settag_zero;
7723 DAG, sdl, getRoot(), getValue(I.getArgOperand(0)),
7724 getValue(I.getArgOperand(1)), MachinePointerInfo(I.getArgOperand(0)),
7725 ZeroMemory);
7726 DAG.setRoot(Val);
7727 setValue(&I, Val);
7728 return;
7729 }
7730 case Intrinsic::amdgcn_cs_chain: {
7731 assert(I.arg_size() == 5 && "Additional args not supported yet");
7732 assert(cast<ConstantInt>(I.getOperand(4))->isZero() &&
7733 "Non-zero flags not supported yet");
7734
7735 // At this point we don't care if it's amdgpu_cs_chain or
7736 // amdgpu_cs_chain_preserve.
7738
7739 Type *RetTy = I.getType();
7740 assert(RetTy->isVoidTy() && "Should not return");
7741
7742 SDValue Callee = getValue(I.getOperand(0));
7743
7744 // We only have 2 actual args: one for the SGPRs and one for the VGPRs.
7745 // We'll also tack the value of the EXEC mask at the end.
7747 Args.reserve(3);
7748
7749 for (unsigned Idx : {2, 3, 1}) {
7751 Arg.Node = getValue(I.getOperand(Idx));
7752 Arg.Ty = I.getOperand(Idx)->getType();
7753 Arg.setAttributes(&I, Idx);
7754 Args.push_back(Arg);
7755 }
7756
7757 assert(Args[0].IsInReg && "SGPR args should be marked inreg");
7758 assert(!Args[1].IsInReg && "VGPR args should not be marked inreg");
7759 Args[2].IsInReg = true; // EXEC should be inreg
7760
7762 CLI.setDebugLoc(getCurSDLoc())
7763 .setChain(getRoot())
7764 .setCallee(CC, RetTy, Callee, std::move(Args))
7765 .setNoReturn(true)
7766 .setTailCall(true)
7767 .setConvergent(I.isConvergent());
7768 CLI.CB = &I;
7769 std::pair<SDValue, SDValue> Result =
7770 lowerInvokable(CLI, /*EHPadBB*/ nullptr);
7771 (void)Result;
7772 assert(!Result.first.getNode() && !Result.second.getNode() &&
7773 "Should've lowered as tail call");
7774
7775 HasTailCall = true;
7776 return;
7777 }
7778 case Intrinsic::ptrmask: {
7779 SDValue Ptr = getValue(I.getOperand(0));
7780 SDValue Mask = getValue(I.getOperand(1));
7781
7782 EVT PtrVT = Ptr.getValueType();
7783 assert(PtrVT == Mask.getValueType() &&
7784 "Pointers with different index type are not supported by SDAG");
7785 setValue(&I, DAG.getNode(ISD::AND, sdl, PtrVT, Ptr, Mask));
7786 return;
7787 }
7788 case Intrinsic::threadlocal_address: {
7789 setValue(&I, getValue(I.getOperand(0)));
7790 return;
7791 }
7792 case Intrinsic::get_active_lane_mask: {
7793 EVT CCVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7794 SDValue Index = getValue(I.getOperand(0));
7795 EVT ElementVT = Index.getValueType();
7796
7797 if (!TLI.shouldExpandGetActiveLaneMask(CCVT, ElementVT)) {
7798 visitTargetIntrinsic(I, Intrinsic);
7799 return;
7800 }
7801
7802 SDValue TripCount = getValue(I.getOperand(1));
7803 EVT VecTy = EVT::getVectorVT(*DAG.getContext(), ElementVT,
7804 CCVT.getVectorElementCount());
7805
7806 SDValue VectorIndex = DAG.getSplat(VecTy, sdl, Index);
7807 SDValue VectorTripCount = DAG.getSplat(VecTy, sdl, TripCount);
7808 SDValue VectorStep = DAG.getStepVector(sdl, VecTy);
7809 SDValue VectorInduction = DAG.getNode(
7810 ISD::UADDSAT, sdl, VecTy, VectorIndex, VectorStep);
7811 SDValue SetCC = DAG.getSetCC(sdl, CCVT, VectorInduction,
7812 VectorTripCount, ISD::CondCode::SETULT);
7813 setValue(&I, SetCC);
7814 return;
7815 }
7816 case Intrinsic::experimental_get_vector_length: {
7817 assert(cast<ConstantInt>(I.getOperand(1))->getSExtValue() > 0 &&
7818 "Expected positive VF");
7819 unsigned VF = cast<ConstantInt>(I.getOperand(1))->getZExtValue();
7820 bool IsScalable = cast<ConstantInt>(I.getOperand(2))->isOne();
7821
7822 SDValue Count = getValue(I.getOperand(0));
7823 EVT CountVT = Count.getValueType();
7824
7825 if (!TLI.shouldExpandGetVectorLength(CountVT, VF, IsScalable)) {
7826 visitTargetIntrinsic(I, Intrinsic);
7827 return;
7828 }
7829
7830 // Expand to a umin between the trip count and the maximum elements the type
7831 // can hold.
7832 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7833
7834 // Extend the trip count to at least the result VT.
7835 if (CountVT.bitsLT(VT)) {
7836 Count = DAG.getNode(ISD::ZERO_EXTEND, sdl, VT, Count);
7837 CountVT = VT;
7838 }
7839
7840 SDValue MaxEVL = DAG.getElementCount(sdl, CountVT,
7841 ElementCount::get(VF, IsScalable));
7842
7843 SDValue UMin = DAG.getNode(ISD::UMIN, sdl, CountVT, Count, MaxEVL);
7844 // Clip to the result type if needed.
7845 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, sdl, VT, UMin);
7846
7847 setValue(&I, Trunc);
7848 return;
7849 }
7850 case Intrinsic::experimental_cttz_elts: {
7851 auto DL = getCurSDLoc();
7852 SDValue Op = getValue(I.getOperand(0));
7853 EVT OpVT = Op.getValueType();
7854
7855 if (!TLI.shouldExpandCttzElements(OpVT)) {
7856 visitTargetIntrinsic(I, Intrinsic);
7857 return;
7858 }
7859
7860 if (OpVT.getScalarType() != MVT::i1) {
7861 // Compare the input vector elements to zero & use to count trailing zeros
7862 SDValue AllZero = DAG.getConstant(0, DL, OpVT);
7863 OpVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
7864 OpVT.getVectorElementCount());
7865 Op = DAG.getSetCC(DL, OpVT, Op, AllZero, ISD::SETNE);
7866 }
7867
7868 // If the zero-is-poison flag is set, we can assume the upper limit
7869 // of the result is VF-1.
7870 bool ZeroIsPoison =
7871 !cast<ConstantSDNode>(getValue(I.getOperand(1)))->isZero();
7872 ConstantRange VScaleRange(1, true); // Dummy value.
7873 if (isa<ScalableVectorType>(I.getOperand(0)->getType()))
7874 VScaleRange = getVScaleRange(I.getCaller(), 64);
7875 unsigned EltWidth = TLI.getBitWidthForCttzElements(
7876 I.getType(), OpVT.getVectorElementCount(), ZeroIsPoison, &VScaleRange);
7877
7878 MVT NewEltTy = MVT::getIntegerVT(EltWidth);
7879
7880 // Create the new vector type & get the vector length
7881 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), NewEltTy,
7882 OpVT.getVectorElementCount());
7883
7884 SDValue VL =
7885 DAG.getElementCount(DL, NewEltTy, OpVT.getVectorElementCount());
7886
7887 SDValue StepVec = DAG.getStepVector(DL, NewVT);
7888 SDValue SplatVL = DAG.getSplat(NewVT, DL, VL);
7889 SDValue StepVL = DAG.getNode(ISD::SUB, DL, NewVT, SplatVL, StepVec);
7891 SDValue And = DAG.getNode(ISD::AND, DL, NewVT, StepVL, Ext);
7893 SDValue Sub = DAG.getNode(ISD::SUB, DL, NewEltTy, VL, Max);
7894
7895 EVT RetTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
7897
7898 setValue(&I, Ret);
7899 return;
7900 }
7901 case Intrinsic::vector_insert: {
7902 SDValue Vec = getValue(I.getOperand(0));
7903 SDValue SubVec = getValue(I.getOperand(1));
7904 SDValue Index = getValue(I.getOperand(2));
7905
7906 // The intrinsic's index type is i64, but the SDNode requires an index type
7907 // suitable for the target. Convert the index as required.
7908 MVT VectorIdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
7909 if (Index.getValueType() != VectorIdxTy)
7910 Index = DAG.getVectorIdxConstant(Index->getAsZExtVal(), sdl);
7911
7912 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7913 setValue(&I, DAG.getNode(ISD::INSERT_SUBVECTOR, sdl, ResultVT, Vec, SubVec,
7914 Index));
7915 return;
7916 }
7917 case Intrinsic::vector_extract: {
7918 SDValue Vec = getValue(I.getOperand(0));
7919 SDValue Index = getValue(I.getOperand(1));
7920 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7921
7922 // The intrinsic's index type is i64, but the SDNode requires an index type
7923 // suitable for the target. Convert the index as required.
7924 MVT VectorIdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
7925 if (Index.getValueType() != VectorIdxTy)
7926 Index = DAG.getVectorIdxConstant(Index->getAsZExtVal(), sdl);
7927
7928 setValue(&I,
7929 DAG.getNode(ISD::EXTRACT_SUBVECTOR, sdl, ResultVT, Vec, Index));
7930 return;
7931 }
7932 case Intrinsic::vector_reverse:
7933 visitVectorReverse(I);
7934 return;
7935 case Intrinsic::vector_splice:
7936 visitVectorSplice(I);
7937 return;
7938 case Intrinsic::callbr_landingpad:
7939 visitCallBrLandingPad(I);
7940 return;
7941 case Intrinsic::vector_interleave2:
7942 visitVectorInterleave(I);
7943 return;
7944 case Intrinsic::vector_deinterleave2:
7945 visitVectorDeinterleave(I);
7946 return;
7947 case Intrinsic::experimental_convergence_anchor:
7948 case Intrinsic::experimental_convergence_entry:
7949 case Intrinsic::experimental_convergence_loop:
7950 visitConvergenceControl(I, Intrinsic);
7951 }
7952}
7953
7954void SelectionDAGBuilder::visitConstrainedFPIntrinsic(
7955 const ConstrainedFPIntrinsic &FPI) {
7956 SDLoc sdl = getCurSDLoc();
7957
7958 // We do not need to serialize constrained FP intrinsics against
7959 // each other or against (nonvolatile) loads, so they can be
7960 // chained like loads.
7961 SDValue Chain = DAG.getRoot();
7963 Opers.push_back(Chain);
7964 for (unsigned I = 0, E = FPI.getNonMetadataArgCount(); I != E; ++I)
7965 Opers.push_back(getValue(FPI.getArgOperand(I)));
7966
7967 auto pushOutChain = [this](SDValue Result, fp::ExceptionBehavior EB) {
7968 assert(Result.getNode()->getNumValues() == 2);
7969
7970 // Push node to the appropriate list so that future instructions can be
7971 // chained up correctly.
7972 SDValue OutChain = Result.getValue(1);
7973 switch (EB) {
7975 // The only reason why ebIgnore nodes still need to be chained is that
7976 // they might depend on the current rounding mode, and therefore must
7977 // not be moved across instruction that may change that mode.
7978 [[fallthrough]];
7980 // These must not be moved across calls or instructions that may change
7981 // floating-point exception masks.
7982 PendingConstrainedFP.push_back(OutChain);
7983 break;
7985 // These must not be moved across calls or instructions that may change
7986 // floating-point exception masks or read floating-point exception flags.
7987 // In addition, they cannot be optimized out even if unused.
7988 PendingConstrainedFPStrict.push_back(OutChain);
7989 break;
7990 }
7991 };
7992
7994 EVT VT = TLI.getValueType(DAG.getDataLayout(), FPI.getType());
7995 SDVTList VTs = DAG.getVTList(VT, MVT::Other);
7997
8000 Flags.setNoFPExcept(true);
8001
8002 if (auto *FPOp = dyn_cast<FPMathOperator>(&FPI))
8003 Flags.copyFMF(*FPOp);
8004
8005 unsigned Opcode;
8006 switch (FPI.getIntrinsicID()) {
8007 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
8008#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
8009 case Intrinsic::INTRINSIC: \
8010 Opcode = ISD::STRICT_##DAGN; \
8011 break;
8012#include "llvm/IR/ConstrainedOps.def"
8013 case Intrinsic::experimental_constrained_fmuladd: {
8014 Opcode = ISD::STRICT_FMA;
8015 // Break fmuladd into fmul and fadd.
8018 Opers.pop_back();
8019 SDValue Mul = DAG.getNode(ISD::STRICT_FMUL, sdl, VTs, Opers, Flags);
8020 pushOutChain(Mul, EB);
8021 Opcode = ISD::STRICT_FADD;
8022 Opers.clear();
8023 Opers.push_back(Mul.getValue(1));
8024 Opers.push_back(Mul.getValue(0));
8025 Opers.push_back(getValue(FPI.getArgOperand(2)));
8026 }
8027 break;
8028 }
8029 }
8030
8031 // A few strict DAG nodes carry additional operands that are not
8032 // set up by the default code above.
8033 switch (Opcode) {
8034 default: break;
8036 Opers.push_back(
8038 break;
8039 case ISD::STRICT_FSETCC:
8040 case ISD::STRICT_FSETCCS: {
8041 auto *FPCmp = dyn_cast<ConstrainedFPCmpIntrinsic>(&FPI);
8042 ISD::CondCode Condition = getFCmpCondCode(FPCmp->getPredicate());
8043 if (TM.Options.NoNaNsFPMath)
8044 Condition = getFCmpCodeWithoutNaN(Condition);
8045 Opers.push_back(DAG.getCondCode(Condition));
8046 break;
8047 }
8048 }
8049
8050 SDValue Result = DAG.getNode(Opcode, sdl, VTs, Opers, Flags);
8051 pushOutChain(Result, EB);
8052
8053 SDValue FPResult = Result.getValue(0);
8054 setValue(&FPI, FPResult);
8055}
8056
8057static unsigned getISDForVPIntrinsic(const VPIntrinsic &VPIntrin) {
8058 std::optional<unsigned> ResOPC;
8059 switch (VPIntrin.getIntrinsicID()) {
8060 case Intrinsic::vp_ctlz: {
8061 bool IsZeroUndef = cast<ConstantInt>(VPIntrin.getArgOperand(1))->isOne();
8062 ResOPC = IsZeroUndef ? ISD::VP_CTLZ_ZERO_UNDEF : ISD::VP_CTLZ;
8063 break;
8064 }
8065 case Intrinsic::vp_cttz: {
8066 bool IsZeroUndef = cast<ConstantInt>(VPIntrin.getArgOperand(1))->isOne();
8067 ResOPC = IsZeroUndef ? ISD::VP_CTTZ_ZERO_UNDEF : ISD::VP_CTTZ;
8068 break;
8069 }
8070 case Intrinsic::vp_cttz_elts: {
8071 bool IsZeroPoison = cast<ConstantInt>(VPIntrin.getArgOperand(1))->isOne();
8072 ResOPC = IsZeroPoison ? ISD::VP_CTTZ_ELTS_ZERO_UNDEF : ISD::VP_CTTZ_ELTS;
8073 break;
8074 }
8075#define HELPER_MAP_VPID_TO_VPSD(VPID, VPSD) \
8076 case Intrinsic::VPID: \
8077 ResOPC = ISD::VPSD; \
8078 break;
8079#include "llvm/IR/VPIntrinsics.def"
8080 }
8081
8082 if (!ResOPC)
8084 "Inconsistency: no SDNode available for this VPIntrinsic!");
8085
8086 if (*ResOPC == ISD::VP_REDUCE_SEQ_FADD ||
8087 *ResOPC == ISD::VP_REDUCE_SEQ_FMUL) {
8088 if (VPIntrin.getFastMathFlags().allowReassoc())
8089 return *ResOPC == ISD::VP_REDUCE_SEQ_FADD ? ISD::VP_REDUCE_FADD
8090 : ISD::VP_REDUCE_FMUL;
8091 }
8092
8093 return *ResOPC;
8094}
8095
8096void SelectionDAGBuilder::visitVPLoad(
8097 const VPIntrinsic &VPIntrin, EVT VT,
8098 const SmallVectorImpl<SDValue> &OpValues) {
8099 SDLoc DL = getCurSDLoc();
8100 Value *PtrOperand = VPIntrin.getArgOperand(0);
8101 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8102 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8103 const MDNode *Ranges = getRangeMetadata(VPIntrin);
8104 SDValue LD;
8105 // Do not serialize variable-length loads of constant memory with
8106 // anything.
8107 if (!Alignment)
8108 Alignment = DAG.getEVTAlign(VT);
8109 MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
8110 bool AddToChain = !AA || !AA->pointsToConstantMemory(ML);
8111 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
8114 LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo, Ranges);
8115 LD = DAG.getLoadVP(VT, DL, InChain, OpValues[0], OpValues[1], OpValues[2],
8116 MMO, false /*IsExpanding */);
8117 if (AddToChain)
8118 PendingLoads.push_back(LD.getValue(1));
8119 setValue(&VPIntrin, LD);
8120}
8121
8122void SelectionDAGBuilder::visitVPGather(
8123 const VPIntrinsic &VPIntrin, EVT VT,
8124 const SmallVectorImpl<SDValue> &OpValues) {
8125 SDLoc DL = getCurSDLoc();
8127 Value *PtrOperand = VPIntrin.getArgOperand(0);
8128 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8129 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8130 const MDNode *Ranges = getRangeMetadata(VPIntrin);
8131 SDValue LD;
8132 if (!Alignment)
8133 Alignment = DAG.getEVTAlign(VT.getScalarType());
8134 unsigned AS =
8135 PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
8138 LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo, Ranges);
8139 SDValue Base, Index, Scale;
8140 ISD::MemIndexType IndexType;
8141 bool UniformBase = getUniformBase(PtrOperand, Base, Index, IndexType, Scale,
8142 this, VPIntrin.getParent(),
8143 VT.getScalarStoreSize());
8144 if (!UniformBase) {
8146 Index = getValue(PtrOperand);
8147 IndexType = ISD::SIGNED_SCALED;
8149 }
8150 EVT IdxVT = Index.getValueType();
8151 EVT EltTy = IdxVT.getVectorElementType();
8152 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
8153 EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
8154 Index = DAG.getNode(ISD::SIGN_EXTEND, DL, NewIdxVT, Index);
8155 }
8156 LD = DAG.getGatherVP(
8157 DAG.getVTList(VT, MVT::Other), VT, DL,
8158 {DAG.getRoot(), Base, Index, Scale, OpValues[1], OpValues[2]}, MMO,
8159 IndexType);
8160 PendingLoads.push_back(LD.getValue(1));
8161 setValue(&VPIntrin, LD);
8162}
8163
8164void SelectionDAGBuilder::visitVPStore(
8165 const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) {
8166 SDLoc DL = getCurSDLoc();
8167 Value *PtrOperand = VPIntrin.getArgOperand(1);
8168 EVT VT = OpValues[0].getValueType();
8169 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8170 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8171 SDValue ST;
8172 if (!Alignment)
8173 Alignment = DAG.getEVTAlign(VT);
8174 SDValue Ptr = OpValues[1];
8175 SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
8178 LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo);
8179 ST = DAG.getStoreVP(getMemoryRoot(), DL, OpValues[0], Ptr, Offset,
8180 OpValues[2], OpValues[3], VT, MMO, ISD::UNINDEXED,
8181 /* IsTruncating */ false, /*IsCompressing*/ false);
8182 DAG.setRoot(ST);
8183 setValue(&VPIntrin, ST);
8184}
8185
8186void SelectionDAGBuilder::visitVPScatter(
8187 const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) {
8188 SDLoc DL = getCurSDLoc();
8190 Value *PtrOperand = VPIntrin.getArgOperand(1);
8191 EVT VT = OpValues[0].getValueType();
8192 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8193 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8194 SDValue ST;
8195 if (!Alignment)
8196 Alignment = DAG.getEVTAlign(VT.getScalarType());
8197 unsigned AS =
8198 PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
8201 LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo);
8202 SDValue Base, Index, Scale;
8203 ISD::MemIndexType IndexType;
8204 bool UniformBase = getUniformBase(PtrOperand, Base, Index, IndexType, Scale,
8205 this, VPIntrin.getParent(),
8206 VT.getScalarStoreSize());
8207 if (!UniformBase) {
8209 Index = getValue(PtrOperand);
8210 IndexType = ISD::SIGNED_SCALED;
8211 Scale =
8213 }
8214 EVT IdxVT = Index.getValueType();
8215 EVT EltTy = IdxVT.getVectorElementType();
8216 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
8217 EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
8218 Index = DAG.getNode(ISD::SIGN_EXTEND, DL, NewIdxVT, Index);
8219 }
8220 ST = DAG.getScatterVP(DAG.getVTList(MVT::Other), VT, DL,
8221 {getMemoryRoot(), OpValues[0], Base, Index, Scale,
8222 OpValues[2], OpValues[3]},
8223 MMO, IndexType);
8224 DAG.setRoot(ST);
8225 setValue(&VPIntrin, ST);
8226}
8227
8228void SelectionDAGBuilder::visitVPStridedLoad(
8229 const VPIntrinsic &VPIntrin, EVT VT,
8230 const SmallVectorImpl<SDValue> &OpValues) {
8231 SDLoc DL = getCurSDLoc();
8232 Value *PtrOperand = VPIntrin.getArgOperand(0);
8233 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8234 if (!Alignment)
8235 Alignment = DAG.getEVTAlign(VT.getScalarType());
8236 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8237 const MDNode *Ranges = getRangeMetadata(VPIntrin);
8238 MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
8239 bool AddToChain = !AA || !AA->pointsToConstantMemory(ML);
8240 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
8241 unsigned AS = PtrOperand->getType()->getPointerAddressSpace();
8244 LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo, Ranges);
8245
8246 SDValue LD = DAG.getStridedLoadVP(VT, DL, InChain, OpValues[0], OpValues[1],
8247 OpValues[2], OpValues[3], MMO,
8248 false /*IsExpanding*/);
8249
8250 if (AddToChain)
8251 PendingLoads.push_back(LD.getValue(1));
8252 setValue(&VPIntrin, LD);
8253}
8254
8255void SelectionDAGBuilder::visitVPStridedStore(
8256 const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) {
8257 SDLoc DL = getCurSDLoc();
8258 Value *PtrOperand = VPIntrin.getArgOperand(1);
8259 EVT VT = OpValues[0].getValueType();
8260 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8261 if (!Alignment)
8262 Alignment = DAG.getEVTAlign(VT.getScalarType());
8263 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8264 unsigned AS = PtrOperand->getType()->getPointerAddressSpace();
8267 LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo);
8268
8270 getMemoryRoot(), DL, OpValues[0], OpValues[1],
8271 DAG.getUNDEF(OpValues[1].getValueType()), OpValues[2], OpValues[3],
8272 OpValues[4], VT, MMO, ISD::UNINDEXED, /*IsTruncating*/ false,
8273 /*IsCompressing*/ false);
8274
8275 DAG.setRoot(ST);
8276 setValue(&VPIntrin, ST);
8277}
8278
8279void SelectionDAGBuilder::visitVPCmp(const VPCmpIntrinsic &VPIntrin) {
8281 SDLoc DL = getCurSDLoc();
8282
8283 ISD::CondCode Condition;
8285 bool IsFP = VPIntrin.getOperand(0)->getType()->isFPOrFPVectorTy();
8286 if (IsFP) {
8287 // FIXME: Regular fcmps are FPMathOperators which may have fast-math (nnan)
8288 // flags, but calls that don't return floating-point types can't be
8289 // FPMathOperators, like vp.fcmp. This affects constrained fcmp too.
8290 Condition = getFCmpCondCode(CondCode);
8291 if (TM.Options.NoNaNsFPMath)
8292 Condition = getFCmpCodeWithoutNaN(Condition);
8293 } else {
8294 Condition = getICmpCondCode(CondCode);
8295 }
8296
8297 SDValue Op1 = getValue(VPIntrin.getOperand(0));
8298 SDValue Op2 = getValue(VPIntrin.getOperand(1));
8299 // #2 is the condition code
8300 SDValue MaskOp = getValue(VPIntrin.getOperand(3));
8301 SDValue EVL = getValue(VPIntrin.getOperand(4));
8302 MVT EVLParamVT = TLI.getVPExplicitVectorLengthTy();
8303 assert(EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) &&
8304 "Unexpected target EVL type");
8305 EVL = DAG.getNode(ISD::ZERO_EXTEND, DL, EVLParamVT, EVL);
8306
8308 VPIntrin.getType());
8309 setValue(&VPIntrin,
8310 DAG.getSetCCVP(DL, DestVT, Op1, Op2, Condition, MaskOp, EVL));
8311}
8312
8313void SelectionDAGBuilder::visitVectorPredicationIntrinsic(
8314 const VPIntrinsic &VPIntrin) {
8315 SDLoc DL = getCurSDLoc();
8316 unsigned Opcode = getISDForVPIntrinsic(VPIntrin);
8317
8318 auto IID = VPIntrin.getIntrinsicID();
8319
8320 if (const auto *CmpI = dyn_cast<VPCmpIntrinsic>(&VPIntrin))
8321 return visitVPCmp(*CmpI);
8322
8323 SmallVector<EVT, 4> ValueVTs;
8325 ComputeValueVTs(TLI, DAG.getDataLayout(), VPIntrin.getType(), ValueVTs);
8326 SDVTList VTs = DAG.getVTList(ValueVTs);
8327
8328 auto EVLParamPos = VPIntrinsic::getVectorLengthParamPos(IID);
8329
8330 MVT EVLParamVT = TLI.getVPExplicitVectorLengthTy();
8331 assert(EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) &&
8332 "Unexpected target EVL type");
8333
8334 // Request operands.
8335 SmallVector<SDValue, 7> OpValues;
8336 for (unsigned I = 0; I < VPIntrin.arg_size(); ++I) {
8337 auto Op = getValue(VPIntrin.getArgOperand(I));
8338 if (I == EVLParamPos)
8339 Op = DAG.getNode(ISD::ZERO_EXTEND, DL, EVLParamVT, Op);
8340 OpValues.push_back(Op);
8341 }
8342
8343 switch (Opcode) {
8344 default: {
8345 SDNodeFlags SDFlags;
8346 if (auto *FPMO = dyn_cast<FPMathOperator>(&VPIntrin))
8347 SDFlags.copyFMF(*FPMO);
8348 SDValue Result = DAG.getNode(Opcode, DL, VTs, OpValues, SDFlags);
8349 setValue(&VPIntrin, Result);
8350 break;
8351 }
8352 case ISD::VP_LOAD:
8353 visitVPLoad(VPIntrin, ValueVTs[0], OpValues);
8354 break;
8355 case ISD::VP_GATHER:
8356 visitVPGather(VPIntrin, ValueVTs[0], OpValues);
8357 break;
8358 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
8359 visitVPStridedLoad(VPIntrin, ValueVTs[0], OpValues);
8360 break;
8361 case ISD::VP_STORE:
8362 visitVPStore(VPIntrin, OpValues);
8363 break;
8364 case ISD::VP_SCATTER:
8365 visitVPScatter(VPIntrin, OpValues);
8366 break;
8367 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
8368 visitVPStridedStore(VPIntrin, OpValues);
8369 break;
8370 case ISD::VP_FMULADD: {
8371 assert(OpValues.size() == 5 && "Unexpected number of operands");
8372 SDNodeFlags SDFlags;
8373 if (auto *FPMO = dyn_cast<FPMathOperator>(&VPIntrin))
8374 SDFlags.copyFMF(*FPMO);
8377 setValue(&VPIntrin, DAG.getNode(ISD::VP_FMA, DL, VTs, OpValues, SDFlags));
8378 } else {
8380 ISD::VP_FMUL, DL, VTs,
8381 {OpValues[0], OpValues[1], OpValues[3], OpValues[4]}, SDFlags);
8382 SDValue Add =
8383 DAG.getNode(ISD::VP_FADD, DL, VTs,
8384 {Mul, OpValues[2], OpValues[3], OpValues[4]}, SDFlags);
8385 setValue(&VPIntrin, Add);
8386 }
8387 break;
8388 }
8389 case ISD::VP_IS_FPCLASS: {
8390 const DataLayout DLayout = DAG.getDataLayout();
8391 EVT DestVT = TLI.getValueType(DLayout, VPIntrin.getType());
8392 auto Constant = OpValues[1]->getAsZExtVal();
8394 SDValue V = DAG.getNode(ISD::VP_IS_FPCLASS, DL, DestVT,
8395 {OpValues[0], Check, OpValues[2], OpValues[3]});
8396 setValue(&VPIntrin, V);
8397 return;
8398 }
8399 case ISD::VP_INTTOPTR: {
8400 SDValue N = OpValues[0];
8401 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), VPIntrin.getType());
8402 EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(), VPIntrin.getType());
8403 N = DAG.getVPPtrExtOrTrunc(getCurSDLoc(), DestVT, N, OpValues[1],
8404 OpValues[2]);
8405 N = DAG.getVPZExtOrTrunc(getCurSDLoc(), PtrMemVT, N, OpValues[1],
8406 OpValues[2]);
8407 setValue(&VPIntrin, N);
8408 break;
8409 }
8410 case ISD::VP_PTRTOINT: {
8411 SDValue N = OpValues[0];
8413 VPIntrin.getType());
8414 EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(),
8415 VPIntrin.getOperand(0)->getType());
8416 N = DAG.getVPPtrExtOrTrunc(getCurSDLoc(), PtrMemVT, N, OpValues[1],
8417 OpValues[2]);
8418 N = DAG.getVPZExtOrTrunc(getCurSDLoc(), DestVT, N, OpValues[1],
8419 OpValues[2]);
8420 setValue(&VPIntrin, N);
8421 break;
8422 }
8423 case ISD::VP_ABS:
8424 case ISD::VP_CTLZ:
8425 case ISD::VP_CTLZ_ZERO_UNDEF:
8426 case ISD::VP_CTTZ:
8427 case ISD::VP_CTTZ_ZERO_UNDEF:
8428 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
8429 case ISD::VP_CTTZ_ELTS: {
8430 SDValue Result =
8431 DAG.getNode(Opcode, DL, VTs, {OpValues[0], OpValues[2], OpValues[3]});
8432 setValue(&VPIntrin, Result);
8433 break;
8434 }
8435 }
8436}
8437
8438SDValue SelectionDAGBuilder::lowerStartEH(SDValue Chain,
8439 const BasicBlock *EHPadBB,
8440 MCSymbol *&BeginLabel) {
8442 MachineModuleInfo &MMI = MF.getMMI();
8443
8444 // Insert a label before the invoke call to mark the try range. This can be
8445 // used to detect deletion of the invoke via the MachineModuleInfo.
8446 BeginLabel = MMI.getContext().createTempSymbol();
8447
8448 // For SjLj, keep track of which landing pads go with which invokes
8449 // so as to maintain the ordering of pads in the LSDA.
8450 unsigned CallSiteIndex = MMI.getCurrentCallSite();
8451 if (CallSiteIndex) {
8452 MF.setCallSiteBeginLabel(BeginLabel, CallSiteIndex);
8453 LPadToCallSiteMap[FuncInfo.MBBMap[EHPadBB]].push_back(CallSiteIndex);
8454
8455 // Now that the call site is handled, stop tracking it.
8456 MMI.setCurrentCallSite(0);
8457 }
8458
8459 return DAG.getEHLabel(getCurSDLoc(), Chain, BeginLabel);
8460}
8461
8462SDValue SelectionDAGBuilder::lowerEndEH(SDValue Chain, const InvokeInst *II,
8463 const BasicBlock *EHPadBB,
8464 MCSymbol *BeginLabel) {
8465 assert(BeginLabel && "BeginLabel should've been set");
8466
8468 MachineModuleInfo &MMI = MF.getMMI();
8469
8470 // Insert a label at the end of the invoke call to mark the try range. This
8471 // can be used to detect deletion of the invoke via the MachineModuleInfo.
8472 MCSymbol *EndLabel = MMI.getContext().createTempSymbol();
8473 Chain = DAG.getEHLabel(getCurSDLoc(), Chain, EndLabel);
8474
8475 // Inform MachineModuleInfo of range.
8477 // There is a platform (e.g. wasm) that uses funclet style IR but does not
8478 // actually use outlined funclets and their LSDA info style.
8479 if (MF.hasEHFunclets() && isFuncletEHPersonality(Pers)) {
8480 assert(II && "II should've been set");
8481 WinEHFuncInfo *EHInfo = MF.getWinEHFuncInfo();
8482 EHInfo->addIPToStateRange(II, BeginLabel, EndLabel);
8483 } else if (!isScopedEHPersonality(Pers)) {
8484 assert(EHPadBB);
8485 MF.addInvoke(FuncInfo.MBBMap[EHPadBB], BeginLabel, EndLabel);
8486 }
8487
8488 return Chain;
8489}
8490
8491std::pair<SDValue, SDValue>
8493 const BasicBlock *EHPadBB) {
8494 MCSymbol *BeginLabel = nullptr;
8495
8496 if (EHPadBB) {
8497 // Both PendingLoads and PendingExports must be flushed here;
8498 // this call might not return.
8499 (void)getRoot();
8500 DAG.setRoot(lowerStartEH(getControlRoot(), EHPadBB, BeginLabel));
8501 CLI.setChain(getRoot());
8502 }
8503
8505 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
8506
8507 assert((CLI.IsTailCall || Result.second.getNode()) &&
8508 "Non-null chain expected with non-tail call!");
8509 assert((Result.second.getNode() || !Result.first.getNode()) &&
8510 "Null value expected with tail call!");
8511
8512 if (!Result.second.getNode()) {
8513 // As a special case, a null chain means that a tail call has been emitted
8514 // and the DAG root is already updated.
8515 HasTailCall = true;
8516
8517 // Since there's no actual continuation from this block, nothing can be
8518 // relying on us setting vregs for them.
8519 PendingExports.clear();
8520 } else {
8521 DAG.setRoot(Result.second);
8522 }
8523
8524 if (EHPadBB) {
8525 DAG.setRoot(lowerEndEH(getRoot(), cast_or_null<InvokeInst>(CLI.CB), EHPadBB,
8526 BeginLabel));
8527 }
8528
8529 return Result;
8530}
8531
8533 bool isTailCall,
8534 bool isMustTailCall,
8535 const BasicBlock *EHPadBB) {
8536 auto &DL = DAG.getDataLayout();
8537 FunctionType *FTy = CB.getFunctionType();
8538 Type *RetTy = CB.getType();
8539
8541 Args.reserve(CB.arg_size());
8542
8543 const Value *SwiftErrorVal = nullptr;
8545
8546 if (isTailCall) {
8547 // Avoid emitting tail calls in functions with the disable-tail-calls
8548 // attribute.
8549 auto *Caller = CB.getParent()->getParent();
8550 if (Caller->getFnAttribute("disable-tail-calls").getValueAsString() ==
8551 "true" && !isMustTailCall)
8552 isTailCall = false;
8553
8554 // We can't tail call inside a function with a swifterror argument. Lowering
8555 // does not support this yet. It would have to move into the swifterror
8556 // register before the call.
8557 if (TLI.supportSwiftError() &&
8558 Caller->getAttributes().hasAttrSomewhere(Attribute::SwiftError))
8559 isTailCall = false;
8560 }
8561
8562 for (auto I = CB.arg_begin(), E = CB.arg_end(); I != E; ++I) {
8564 const Value *V = *I;
8565
8566 // Skip empty types
8567 if (V->getType()->isEmptyTy())
8568 continue;
8569
8570 SDValue ArgNode = getValue(V);
8571 Entry.Node = ArgNode; Entry.Ty = V->getType();
8572
8573 Entry.setAttributes(&CB, I - CB.arg_begin());
8574
8575 // Use swifterror virtual register as input to the call.
8576 if (Entry.IsSwiftError && TLI.supportSwiftError()) {
8577 SwiftErrorVal = V;
8578 // We find the virtual register for the actual swifterror argument.
8579 // Instead of using the Value, we use the virtual register instead.
8580 Entry.Node =
8582 EVT(TLI.getPointerTy(DL)));
8583 }
8584
8585 Args.push_back(Entry);
8586
8587 // If we have an explicit sret argument that is an Instruction, (i.e., it
8588 // might point to function-local memory), we can't meaningfully tail-call.
8589 if (Entry.IsSRet && isa<Instruction>(V))
8590 isTailCall = false;
8591 }
8592
8593 // If call site has a cfguardtarget operand bundle, create and add an
8594 // additional ArgListEntry.
8595 if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_cfguardtarget)) {
8597 Value *V = Bundle->Inputs[0];
8598 SDValue ArgNode = getValue(V);
8599 Entry.Node = ArgNode;
8600 Entry.Ty = V->getType();
8601 Entry.IsCFGuardTarget = true;
8602 Args.push_back(Entry);
8603 }
8604
8605 // Check if target-independent constraints permit a tail call here.
8606 // Target-dependent constraints are checked within TLI->LowerCallTo.
8607 if (isTailCall && !isInTailCallPosition(CB, DAG.getTarget()))
8608 isTailCall = false;
8609
8610 // Disable tail calls if there is an swifterror argument. Targets have not
8611 // been updated to support tail calls.
8612 if (TLI.supportSwiftError() && SwiftErrorVal)
8613 isTailCall = false;
8614
8615 ConstantInt *CFIType = nullptr;
8616 if (CB.isIndirectCall()) {
8617 if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_kcfi)) {
8618 if (!TLI.supportKCFIBundles())
8620 "Target doesn't support calls with kcfi operand bundles.");
8621 CFIType = cast<ConstantInt>(Bundle->Inputs[0]);
8622 assert(CFIType->getType()->isIntegerTy(32) && "Invalid CFI type");
8623 }
8624 }
8625
8626 SDValue ConvControlToken;
8627 if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_convergencectrl)) {
8628 auto *Token = Bundle->Inputs[0].get();
8629 ConvControlToken = getValue(Token);
8630 }
8631
8634 .setChain(getRoot())
8635 .setCallee(RetTy, FTy, Callee, std::move(Args), CB)
8636 .setTailCall(isTailCall)
8640 .setCFIType(CFIType)
8641 .setConvergenceControlToken(ConvControlToken);
8642 std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
8643
8644 if (Result.first.getNode()) {
8645 Result.first = lowerRangeToAssertZExt(DAG, CB, Result.first);
8646 setValue(&CB, Result.first);
8647 }
8648
8649 // The last element of CLI.InVals has the SDValue for swifterror return.
8650 // Here we copy it to a virtual register and update SwiftErrorMap for
8651 // book-keeping.
8652 if (SwiftErrorVal && TLI.supportSwiftError()) {
8653 // Get the last element of InVals.
8654 SDValue Src = CLI.InVals.back();
8655 Register VReg =
8656 SwiftError.getOrCreateVRegDefAt(&CB, FuncInfo.MBB, SwiftErrorVal);
8657 SDValue CopyNode = CLI.DAG.getCopyToReg(Result.second, CLI.DL, VReg, Src);
8658 DAG.setRoot(CopyNode);
8659 }
8660}
8661
8662static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT,
8663 SelectionDAGBuilder &Builder) {
8664 // Check to see if this load can be trivially constant folded, e.g. if the
8665 // input is from a string literal.
8666 if (const Constant *LoadInput = dyn_cast<Constant>(PtrVal)) {
8667 // Cast pointer to the type we really want to load.
8668 Type *LoadTy =
8669 Type::getIntNTy(PtrVal->getContext(), LoadVT.getScalarSizeInBits());
8670 if (LoadVT.isVector())
8671 LoadTy = FixedVectorType::get(LoadTy, LoadVT.getVectorNumElements());
8672
8673 LoadInput = ConstantExpr::getBitCast(const_cast<Constant *>(LoadInput),
8674 PointerType::getUnqual(LoadTy));
8675
8676 if (const Constant *LoadCst =
8677 ConstantFoldLoadFromConstPtr(const_cast<Constant *>(LoadInput),
8678 LoadTy, Builder.DAG.getDataLayout()))
8679 return Builder.getValue(LoadCst);
8680 }
8681
8682 // Otherwise, we have to emit the load. If the pointer is to unfoldable but
8683 // still constant memory, the input chain can be the entry node.
8684 SDValue Root;
8685 bool ConstantMemory = false;
8686
8687 // Do not serialize (non-volatile) loads of constant memory with anything.
8688 if (Builder.AA && Builder.AA->pointsToConstantMemory(PtrVal)) {
8689 Root = Builder.DAG.getEntryNode();
8690 ConstantMemory = true;
8691 } else {
8692 // Do not serialize non-volatile loads against each other.
8693 Root = Builder.DAG.getRoot();
8694 }
8695
8696 SDValue Ptr = Builder.getValue(PtrVal);
8697 SDValue LoadVal =
8698 Builder.DAG.getLoad(LoadVT, Builder.getCurSDLoc(), Root, Ptr,
8699 MachinePointerInfo(PtrVal), Align(1));
8700
8701 if (!ConstantMemory)
8702 Builder.PendingLoads.push_back(LoadVal.getValue(1));
8703 return LoadVal;
8704}
8705
8706/// Record the value for an instruction that produces an integer result,
8707/// converting the type where necessary.
8708void SelectionDAGBuilder::processIntegerCallValue(const Instruction &I,
8709 SDValue Value,
8710 bool IsSigned) {
8712 I.getType(), true);
8713 Value = DAG.getExtOrTrunc(IsSigned, Value, getCurSDLoc(), VT);
8714 setValue(&I, Value);
8715}
8716
8717/// See if we can lower a memcmp/bcmp call into an optimized form. If so, return
8718/// true and lower it. Otherwise return false, and it will be lowered like a
8719/// normal call.
8720/// The caller already checked that \p I calls the appropriate LibFunc with a
8721/// correct prototype.
8722bool SelectionDAGBuilder::visitMemCmpBCmpCall(const CallInst &I) {
8723 const Value *LHS = I.getArgOperand(0), *RHS = I.getArgOperand(1);
8724 const Value *Size = I.getArgOperand(2);
8725 const ConstantSDNode *CSize = dyn_cast<ConstantSDNode>(getValue(Size));
8726 if (CSize && CSize->getZExtValue() == 0) {
8728 I.getType(), true);
8729 setValue(&I, DAG.getConstant(0, getCurSDLoc(), CallVT));
8730 return true;
8731 }
8732
8734 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForMemcmp(
8735 DAG, getCurSDLoc(), DAG.getRoot(), getValue(LHS), getValue(RHS),
8737 if (Res.first.getNode()) {
8738 processIntegerCallValue(I, Res.first, true);
8739 PendingLoads.push_back(Res.second);
8740 return true;
8741 }
8742
8743 // memcmp(S1,S2,2) != 0 -> (*(short*)LHS != *(short*)RHS) != 0
8744 // memcmp(S1,S2,4) != 0 -> (*(int*)LHS != *(int*)RHS) != 0
8745 if (!CSize || !isOnlyUsedInZeroEqualityComparison(&I))
8746 return false;
8747
8748 // If the target has a fast compare for the given size, it will return a
8749 // preferred load type for that size. Require that the load VT is legal and
8750 // that the target supports unaligned loads of that type. Otherwise, return
8751 // INVALID.
8752 auto hasFastLoadsAndCompare = [&](unsigned NumBits) {
8754 MVT LVT = TLI.hasFastEqualityCompare(NumBits);
8755 if (LVT != MVT::INVALID_SIMPLE_VALUE_TYPE) {
8756 // TODO: Handle 5 byte compare as 4-byte + 1 byte.
8757 // TODO: Handle 8 byte compare on x86-32 as two 32-bit loads.
8758 // TODO: Check alignment of src and dest ptrs.
8759 unsigned DstAS = LHS->getType()->getPointerAddressSpace();
8760 unsigned SrcAS = RHS->getType()->getPointerAddressSpace();
8761 if (!TLI.isTypeLegal(LVT) ||
8762 !TLI.allowsMisalignedMemoryAccesses(LVT, SrcAS) ||
8763 !TLI.allowsMisalignedMemoryAccesses(LVT, DstAS))
8765 }
8766
8767 return LVT;
8768 };
8769
8770 // This turns into unaligned loads. We only do this if the target natively
8771 // supports the MVT we'll be loading or if it is small enough (<= 4) that
8772 // we'll only produce a small number of byte loads.
8773 MVT LoadVT;
8774 unsigned NumBitsToCompare = CSize->getZExtValue() * 8;
8775 switch (NumBitsToCompare) {
8776 default:
8777 return false;
8778 case 16:
8779 LoadVT = MVT::i16;
8780 break;
8781 case 32:
8782 LoadVT = MVT::i32;
8783 break;
8784 case 64:
8785 case 128:
8786 case 256:
8787 LoadVT = hasFastLoadsAndCompare(NumBitsToCompare);
8788 break;
8789 }
8790
8791 if (LoadVT == MVT::INVALID_SIMPLE_VALUE_TYPE)
8792 return false;
8793
8794 SDValue LoadL = getMemCmpLoad(LHS, LoadVT, *this);
8795 SDValue LoadR = getMemCmpLoad(RHS, LoadVT, *this);
8796
8797 // Bitcast to a wide integer type if the loads are vectors.
8798 if (LoadVT.isVector()) {
8799 EVT CmpVT = EVT::getIntegerVT(LHS->getContext(), LoadVT.getSizeInBits());
8800 LoadL = DAG.getBitcast(CmpVT, LoadL);
8801 LoadR = DAG.getBitcast(CmpVT, LoadR);
8802 }
8803
8804 SDValue Cmp = DAG.getSetCC(getCurSDLoc(), MVT::i1, LoadL, LoadR, ISD::SETNE);
8805 processIntegerCallValue(I, Cmp, false);
8806 return true;
8807}
8808
8809/// See if we can lower a memchr call into an optimized form. If so, return
8810/// true and lower it. Otherwise return false, and it will be lowered like a
8811/// normal call.
8812/// The caller already checked that \p I calls the appropriate LibFunc with a
8813/// correct prototype.
8814bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) {
8815 const Value *Src = I.getArgOperand(0);
8816 const Value *Char = I.getArgOperand(1);
8817 const Value *Length = I.getArgOperand(2);
8818
8820 std::pair<SDValue, SDValue> Res =
8822 getValue(Src), getValue(Char), getValue(Length),
8823 MachinePointerInfo(Src));
8824 if (Res.first.getNode()) {
8825 setValue(&I, Res.first);
8826 PendingLoads.push_back(Res.second);
8827 return true;
8828 }
8829
8830 return false;
8831}
8832
8833/// See if we can lower a mempcpy call into an optimized form. If so, return
8834/// true and lower it. Otherwise return false, and it will be lowered like a
8835/// normal call.
8836/// The caller already checked that \p I calls the appropriate LibFunc with a
8837/// correct prototype.
8838bool SelectionDAGBuilder::visitMemPCpyCall(const CallInst &I) {
8839 SDValue Dst = getValue(I.getArgOperand(0));
8840 SDValue Src = getValue(I.getArgOperand(1));
8841 SDValue Size = getValue(I.getArgOperand(2));
8842
8843 Align DstAlign = DAG.InferPtrAlign(Dst).valueOrOne();
8844 Align SrcAlign = DAG.InferPtrAlign(Src).valueOrOne();
8845 // DAG::getMemcpy needs Alignment to be defined.
8846 Align Alignment = std::min(DstAlign, SrcAlign);
8847
8848 SDLoc sdl = getCurSDLoc();
8849
8850 // In the mempcpy context we need to pass in a false value for isTailCall
8851 // because the return pointer needs to be adjusted by the size of
8852 // the copied memory.
8853 SDValue Root = getMemoryRoot();
8854 SDValue MC = DAG.getMemcpy(Root, sdl, Dst, Src, Size, Alignment, false, false,
8855 /*isTailCall=*/false,
8856 MachinePointerInfo(I.getArgOperand(0)),
8857 MachinePointerInfo(I.getArgOperand(1)),
8858 I.getAAMetadata());
8859 assert(MC.getNode() != nullptr &&
8860 "** memcpy should not be lowered as TailCall in mempcpy context **");
8861 DAG.setRoot(MC);
8862
8863 // Check if Size needs to be truncated or extended.
8864 Size = DAG.getSExtOrTrunc(Size, sdl, Dst.getValueType());
8865
8866 // Adjust return pointer to point just past the last dst byte.
8867 SDValue DstPlusSize = DAG.getNode(ISD::ADD, sdl, Dst.getValueType(),
8868 Dst, Size);
8869 setValue(&I, DstPlusSize);
8870 return true;
8871}
8872
8873/// See if we can lower a strcpy call into an optimized form. If so, return
8874/// true and lower it, otherwise return false and it will be lowered like a
8875/// normal call.
8876/// The caller already checked that \p I calls the appropriate LibFunc with a
8877/// correct prototype.
8878bool SelectionDAGBuilder::visitStrCpyCall(const CallInst &I, bool isStpcpy) {
8879 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
8880
8882 std::pair<SDValue, SDValue> Res =
8884 getValue(Arg0), getValue(Arg1),
8885 MachinePointerInfo(Arg0),
8886 MachinePointerInfo(Arg1), isStpcpy);
8887 if (Res.first.getNode()) {
8888 setValue(&I, Res.first);
8889 DAG.setRoot(Res.second);
8890 return true;
8891 }
8892
8893 return false;
8894}
8895
8896/// See if we can lower a strcmp call into an optimized form. If so, return
8897/// true and lower it, otherwise return false and it will be lowered like a
8898/// normal call.
8899/// The caller already checked that \p I calls the appropriate LibFunc with a
8900/// correct prototype.
8901bool SelectionDAGBuilder::visitStrCmpCall(const CallInst &I) {
8902 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
8903
8905 std::pair<SDValue, SDValue> Res =
8907 getValue(Arg0), getValue(Arg1),
8908 MachinePointerInfo(Arg0),
8909 MachinePointerInfo(Arg1));
8910 if (Res.first.getNode()) {
8911 processIntegerCallValue(I, Res.first, true);
8912 PendingLoads.push_back(Res.second);
8913 return true;
8914 }
8915
8916 return false;
8917}
8918
8919/// See if we can lower a strlen call into an optimized form. If so, return
8920/// true and lower it, otherwise return false and it will be lowered like a
8921/// normal call.
8922/// The caller already checked that \p I calls the appropriate LibFunc with a
8923/// correct prototype.
8924bool SelectionDAGBuilder::visitStrLenCall(const CallInst &I) {
8925 const Value *Arg0 = I.getArgOperand(0);
8926
8928 std::pair<SDValue, SDValue> Res =
8930 getValue(Arg0), MachinePointerInfo(Arg0));
8931 if (Res.first.getNode()) {
8932 processIntegerCallValue(I, Res.first, false);
8933 PendingLoads.push_back(Res.second);
8934 return true;
8935 }
8936
8937 return false;
8938}
8939
8940/// See if we can lower a strnlen call into an optimized form. If so, return
8941/// true and lower it, otherwise return false and it will be lowered like a
8942/// normal call.
8943/// The caller already checked that \p I calls the appropriate LibFunc with a
8944/// correct prototype.
8945bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) {
8946 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
8947
8949 std::pair<SDValue, SDValue> Res =
8951 getValue(Arg0), getValue(Arg1),
8952 MachinePointerInfo(Arg0));
8953 if (Res.first.getNode()) {
8954 processIntegerCallValue(I, Res.first, false);
8955 PendingLoads.push_back(Res.second);
8956 return true;
8957 }
8958
8959 return false;
8960}
8961
8962/// See if we can lower a unary floating-point operation into an SDNode with
8963/// the specified Opcode. If so, return true and lower it, otherwise return
8964/// false and it will be lowered like a normal call.
8965/// The caller already checked that \p I calls the appropriate LibFunc with a
8966/// correct prototype.
8967bool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I,
8968 unsigned Opcode) {
8969 // We already checked this call's prototype; verify it doesn't modify errno.
8970 if (!I.onlyReadsMemory())
8971 return false;
8972
8974 Flags.copyFMF(cast<FPMathOperator>(I));
8975
8976 SDValue Tmp = getValue(I.getArgOperand(0));
8977 setValue(&I,
8978 DAG.getNode(Opcode, getCurSDLoc(), Tmp.getValueType(), Tmp, Flags));
8979 return true;
8980}
8981
8982/// See if we can lower a binary floating-point operation into an SDNode with
8983/// the specified Opcode. If so, return true and lower it. Otherwise return
8984/// false, and it will be lowered like a normal call.
8985/// The caller already checked that \p I calls the appropriate LibFunc with a
8986/// correct prototype.
8987bool SelectionDAGBuilder::visitBinaryFloatCall(const CallInst &I,
8988 unsigned Opcode) {
8989 // We already checked this call's prototype; verify it doesn't modify errno.
8990 if (!I.onlyReadsMemory())
8991 return false;
8992
8994 Flags.copyFMF(cast<FPMathOperator>(I));
8995
8996 SDValue Tmp0 = getValue(I.getArgOperand(0));
8997 SDValue Tmp1 = getValue(I.getArgOperand(1));
8998 EVT VT = Tmp0.getValueType();
8999 setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), VT, Tmp0, Tmp1, Flags));
9000 return true;
9001}
9002
9003void SelectionDAGBuilder::visitCall(const CallInst &I) {
9004 // Handle inline assembly differently.
9005 if (I.isInlineAsm()) {
9006 visitInlineAsm(I);
9007 return;
9008 }
9009
9011
9012 if (Function *F = I.getCalledFunction()) {
9013 if (F->isDeclaration()) {
9014 // Is this an LLVM intrinsic or a target-specific intrinsic?
9015 unsigned IID = F->getIntrinsicID();
9016 if (!IID)
9017 if (const TargetIntrinsicInfo *II = TM.getIntrinsicInfo())
9018 IID = II->getIntrinsicID(F);
9019
9020 if (IID) {
9021 visitIntrinsicCall(I, IID);
9022 return;
9023 }
9024 }
9025
9026 // Check for well-known libc/libm calls. If the function is internal, it
9027 // can't be a library call. Don't do the check if marked as nobuiltin for
9028 // some reason or the call site requires strict floating point semantics.
9029 LibFunc Func;
9030 if (!I.isNoBuiltin() && !I.isStrictFP() && !F->hasLocalLinkage() &&
9031 F->hasName() && LibInfo->getLibFunc(*F, Func) &&
9033 switch (Func) {
9034 default: break;
9035 case LibFunc_bcmp:
9036 if (visitMemCmpBCmpCall(I))
9037 return;
9038 break;
9039 case LibFunc_copysign:
9040 case LibFunc_copysignf:
9041 case LibFunc_copysignl:
9042 // We already checked this call's prototype; verify it doesn't modify
9043 // errno.
9044 if (I.onlyReadsMemory()) {
9045 SDValue LHS = getValue(I.getArgOperand(0));
9046 SDValue RHS = getValue(I.getArgOperand(1));
9048 LHS.getValueType(), LHS, RHS));
9049 return;
9050 }
9051 break;
9052 case LibFunc_fabs:
9053 case LibFunc_fabsf:
9054 case LibFunc_fabsl:
9055 if (visitUnaryFloatCall(I, ISD::FABS))
9056 return;
9057 break;
9058 case LibFunc_fmin:
9059 case LibFunc_fminf:
9060 case LibFunc_fminl:
9061 if (visitBinaryFloatCall(I, ISD::FMINNUM))
9062 return;
9063 break;
9064 case LibFunc_fmax:
9065 case LibFunc_fmaxf:
9066 case LibFunc_fmaxl:
9067 if (visitBinaryFloatCall(I, ISD::FMAXNUM))
9068 return;
9069 break;
9070 case LibFunc_sin:
9071 case LibFunc_sinf:
9072 case LibFunc_sinl:
9073 if (visitUnaryFloatCall(I, ISD::FSIN))
9074 return;
9075 break;
9076 case LibFunc_cos:
9077 case LibFunc_cosf:
9078 case LibFunc_cosl:
9079 if (visitUnaryFloatCall(I, ISD::FCOS))
9080 return;
9081 break;
9082 case LibFunc_sqrt:
9083 case LibFunc_sqrtf:
9084 case LibFunc_sqrtl:
9085 case LibFunc_sqrt_finite:
9086 case LibFunc_sqrtf_finite:
9087 case LibFunc_sqrtl_finite:
9088 if (visitUnaryFloatCall(I, ISD::FSQRT))
9089 return;
9090 break;
9091 case LibFunc_floor:
9092 case LibFunc_floorf:
9093 case LibFunc_floorl:
9094 if (visitUnaryFloatCall(I, ISD::FFLOOR))
9095 return;
9096 break;
9097 case LibFunc_nearbyint:
9098 case LibFunc_nearbyintf:
9099 case LibFunc_nearbyintl:
9100 if (visitUnaryFloatCall(I, ISD::FNEARBYINT))
9101 return;
9102 break;
9103 case LibFunc_ceil:
9104 case LibFunc_ceilf:
9105 case LibFunc_ceill:
9106 if (visitUnaryFloatCall(I, ISD::FCEIL))
9107 return;
9108 break;
9109 case LibFunc_rint:
9110 case LibFunc_rintf:
9111 case LibFunc_rintl:
9112 if (visitUnaryFloatCall(I, ISD::FRINT))
9113 return;
9114 break;
9115 case LibFunc_round:
9116 case LibFunc_roundf:
9117 case LibFunc_roundl:
9118 if (visitUnaryFloatCall(I, ISD::FROUND))
9119 return;
9120 break;
9121 case LibFunc_trunc:
9122 case LibFunc_truncf:
9123 case LibFunc_truncl:
9124 if (visitUnaryFloatCall(I, ISD::FTRUNC))
9125 return;
9126 break;
9127 case LibFunc_log2:
9128 case LibFunc_log2f:
9129 case LibFunc_log2l:
9130 if (visitUnaryFloatCall(I, ISD::FLOG2))
9131 return;
9132 break;
9133 case LibFunc_exp2:
9134 case LibFunc_exp2f:
9135 case LibFunc_exp2l:
9136 if (visitUnaryFloatCall(I, ISD::FEXP2))
9137 return;
9138 break;
9139 case LibFunc_exp10:
9140 case LibFunc_exp10f:
9141 case LibFunc_exp10l:
9142 if (visitUnaryFloatCall(I, ISD::FEXP10))
9143 return;
9144 break;
9145 case LibFunc_ldexp:
9146 case LibFunc_ldexpf:
9147 case LibFunc_ldexpl:
9148 if (visitBinaryFloatCall(I, ISD::FLDEXP))
9149 return;
9150 break;
9151 case LibFunc_memcmp:
9152 if (visitMemCmpBCmpCall(I))
9153 return;
9154 break;
9155 case LibFunc_mempcpy:
9156 if (visitMemPCpyCall(I))
9157 return;
9158 break;
9159 case LibFunc_memchr:
9160 if (visitMemChrCall(I))
9161 return;
9162 break;
9163 case LibFunc_strcpy:
9164 if (visitStrCpyCall(I, false))
9165 return;
9166 break;
9167 case LibFunc_stpcpy:
9168 if (visitStrCpyCall(I, true))
9169 return;
9170 break;
9171 case LibFunc_strcmp:
9172 if (visitStrCmpCall(I))
9173 return;
9174 break;
9175 case LibFunc_strlen:
9176 if (visitStrLenCall(I))
9177 return;
9178 break;
9179 case LibFunc_strnlen:
9180 if (visitStrNLenCall(I))
9181 return;
9182 break;
9183 }
9184 }
9185 }
9186
9187 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
9188 // have to do anything here to lower funclet bundles.
9189 // CFGuardTarget bundles are lowered in LowerCallTo.
9190 assert(!I.hasOperandBundlesOtherThan(
9191 {LLVMContext::OB_deopt, LLVMContext::OB_funclet,
9192 LLVMContext::OB_cfguardtarget, LLVMContext::OB_preallocated,
9193 LLVMContext::OB_clang_arc_attachedcall, LLVMContext::OB_kcfi,
9194 LLVMContext::OB_convergencectrl}) &&
9195 "Cannot lower calls with arbitrary operand bundles!");
9196
9197 SDValue Callee = getValue(I.getCalledOperand());
9198
9199 if (I.hasDeoptState())
9200 LowerCallSiteWithDeoptBundle(&I, Callee, nullptr);
9201 else
9202 // Check if we can potentially perform a tail call. More detailed checking
9203 // is be done within LowerCallTo, after more information about the call is
9204 // known.
9205 LowerCallTo(I, Callee, I.isTailCall(), I.isMustTailCall());
9206}
9207
9208namespace {
9209
9210/// AsmOperandInfo - This contains information for each constraint that we are
9211/// lowering.
9212class SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo {
9213public:
9214 /// CallOperand - If this is the result output operand or a clobber
9215 /// this is null, otherwise it is the incoming operand to the CallInst.
9216 /// This gets modified as the asm is processed.
9217 SDValue CallOperand;
9218
9219 /// AssignedRegs - If this is a register or register class operand, this
9220 /// contains the set of register corresponding to the operand.
9221 RegsForValue AssignedRegs;
9222
9223 explicit SDISelAsmOperandInfo(const TargetLowering::AsmOperandInfo &info)
9224 : TargetLowering::AsmOperandInfo(info), CallOperand(nullptr, 0) {
9225 }
9226
9227 /// Whether or not this operand accesses memory
9228 bool hasMemory(const TargetLowering &TLI) const {
9229 // Indirect operand accesses access memory.
9230 if (isIndirect)
9231 return true;
9232
9233 for (const auto &Code : Codes)
9235 return true;
9236
9237 return false;
9238 }
9239};
9240
9241
9242} // end anonymous namespace
9243
9244/// Make sure that the output operand \p OpInfo and its corresponding input
9245/// operand \p MatchingOpInfo have compatible constraint types (otherwise error
9246/// out).
9247static void patchMatchingInput(const SDISelAsmOperandInfo &OpInfo,
9248 SDISelAsmOperandInfo &MatchingOpInfo,
9249 SelectionDAG &DAG) {
9250 if (OpInfo.ConstraintVT == MatchingOpInfo.ConstraintVT)
9251 return;
9252
9254 const auto &TLI = DAG.getTargetLoweringInfo();
9255
9256 std::pair<unsigned, const TargetRegisterClass *> MatchRC =
9257 TLI.getRegForInlineAsmConstraint(TRI, OpInfo.ConstraintCode,
9258 OpInfo.ConstraintVT);
9259 std::pair<unsigned, const TargetRegisterClass *> InputRC =
9260 TLI.getRegForInlineAsmConstraint(TRI, MatchingOpInfo.ConstraintCode,
9261 MatchingOpInfo.ConstraintVT);
9262 if ((OpInfo.ConstraintVT.isInteger() !=
9263 MatchingOpInfo.ConstraintVT.isInteger()) ||
9264 (MatchRC.second != InputRC.second)) {
9265 // FIXME: error out in a more elegant fashion
9266 report_fatal_error("Unsupported asm: input constraint"
9267 " with a matching output constraint of"
9268 " incompatible type!");
9269 }
9270 MatchingOpInfo.ConstraintVT = OpInfo.ConstraintVT;
9271}
9272
9273/// Get a direct memory input to behave well as an indirect operand.
9274/// This may introduce stores, hence the need for a \p Chain.
9275/// \return The (possibly updated) chain.
9276static SDValue getAddressForMemoryInput(SDValue Chain, const SDLoc &Location,
9277 SDISelAsmOperandInfo &OpInfo,
9278 SelectionDAG &DAG) {
9279 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9280
9281 // If we don't have an indirect input, put it in the constpool if we can,
9282 // otherwise spill it to a stack slot.
9283 // TODO: This isn't quite right. We need to handle these according to
9284 // the addressing mode that the constraint wants. Also, this may take
9285 // an additional register for the computation and we don't want that
9286 // either.
9287
9288 // If the operand is a float, integer, or vector constant, spill to a
9289 // constant pool entry to get its address.
9290 const Value *OpVal = OpInfo.CallOperandVal;
9291 if (isa<ConstantFP>(OpVal) || isa<ConstantInt>(OpVal) ||
9292 isa<ConstantVector>(OpVal) || isa<ConstantDataVector>(OpVal)) {
9293 OpInfo.CallOperand = DAG.getConstantPool(
9294 cast<Constant>(OpVal), TLI.getPointerTy(DAG.getDataLayout()));
9295 return Chain;
9296 }
9297
9298 // Otherwise, create a stack slot and emit a store to it before the asm.
9299 Type *Ty = OpVal->getType();
9300 auto &DL = DAG.getDataLayout();
9301 uint64_t TySize = DL.getTypeAllocSize(Ty);
9303 int SSFI = MF.getFrameInfo().CreateStackObject(
9304 TySize, DL.getPrefTypeAlign(Ty), false);
9305 SDValue StackSlot = DAG.getFrameIndex(SSFI, TLI.getFrameIndexTy(DL));
9306 Chain = DAG.getTruncStore(Chain, Location, OpInfo.CallOperand, StackSlot,
9308 TLI.getMemValueType(DL, Ty));
9309 OpInfo.CallOperand = StackSlot;
9310
9311 return Chain;
9312}
9313
9314/// GetRegistersForValue - Assign registers (virtual or physical) for the
9315/// specified operand. We prefer to assign virtual registers, to allow the
9316/// register allocator to handle the assignment process. However, if the asm
9317/// uses features that we can't model on machineinstrs, we have SDISel do the
9318/// allocation. This produces generally horrible, but correct, code.
9319///
9320/// OpInfo describes the operand
9321/// RefOpInfo describes the matching operand if any, the operand otherwise
9322static std::optional<unsigned>
9324 SDISelAsmOperandInfo &OpInfo,
9325 SDISelAsmOperandInfo &RefOpInfo) {
9326 LLVMContext &Context = *DAG.getContext();
9327 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9328
9332
9333 // No work to do for memory/address operands.
9334 if (OpInfo.ConstraintType == TargetLowering::C_Memory ||
9335 OpInfo.ConstraintType == TargetLowering::C_Address)
9336 return std::nullopt;
9337
9338 // If this is a constraint for a single physreg, or a constraint for a
9339 // register class, find it.
9340 unsigned AssignedReg;
9341 const TargetRegisterClass *RC;
9342 std::tie(AssignedReg, RC) = TLI.getRegForInlineAsmConstraint(
9343 &TRI, RefOpInfo.ConstraintCode, RefOpInfo.ConstraintVT);
9344 // RC is unset only on failure. Return immediately.
9345 if (!RC)
9346 return std::nullopt;
9347
9348 // Get the actual register value type. This is important, because the user
9349 // may have asked for (e.g.) the AX register in i32 type. We need to
9350 // remember that AX is actually i16 to get the right extension.
9351 const MVT RegVT = *TRI.legalclasstypes_begin(*RC);
9352
9353 if (OpInfo.ConstraintVT != MVT::Other && RegVT != MVT::Untyped) {
9354 // If this is an FP operand in an integer register (or visa versa), or more
9355 // generally if the operand value disagrees with the register class we plan
9356 // to stick it in, fix the operand type.
9357 //
9358 // If this is an input value, the bitcast to the new type is done now.
9359 // Bitcast for output value is done at the end of visitInlineAsm().
9360 if ((OpInfo.Type == InlineAsm::isOutput ||
9361 OpInfo.Type == InlineAsm::isInput) &&
9362 !TRI.isTypeLegalForClass(*RC, OpInfo.ConstraintVT)) {
9363 // Try to convert to the first EVT that the reg class contains. If the
9364 // types are identical size, use a bitcast to convert (e.g. two differing
9365 // vector types). Note: output bitcast is done at the end of
9366 // visitInlineAsm().
9367 if (RegVT.getSizeInBits() == OpInfo.ConstraintVT.getSizeInBits()) {
9368 // Exclude indirect inputs while they are unsupported because the code
9369 // to perform the load is missing and thus OpInfo.CallOperand still
9370 // refers to the input address rather than the pointed-to value.
9371 if (OpInfo.Type == InlineAsm::isInput && !OpInfo.isIndirect)
9372 OpInfo.CallOperand =
9373 DAG.getNode(ISD::BITCAST, DL, RegVT, OpInfo.CallOperand);
9374 OpInfo.ConstraintVT = RegVT;
9375 // If the operand is an FP value and we want it in integer registers,
9376 // use the corresponding integer type. This turns an f64 value into
9377 // i64, which can be passed with two i32 values on a 32-bit machine.
9378 } else if (RegVT.isInteger() && OpInfo.ConstraintVT.isFloatingPoint()) {
9379 MVT VT = MVT::getIntegerVT(OpInfo.ConstraintVT.getSizeInBits());
9380 if (OpInfo.Type == InlineAsm::isInput)
9381 OpInfo.CallOperand =
9382 DAG.getNode(ISD::BITCAST, DL, VT, OpInfo.CallOperand);
9383 OpInfo.ConstraintVT = VT;
9384 }
9385 }
9386 }
9387
9388 // No need to allocate a matching input constraint since the constraint it's
9389 // matching to has already been allocated.
9390 if (OpInfo.isMatchingInputConstraint())
9391 return std::nullopt;
9392
9393 EVT ValueVT = OpInfo.ConstraintVT;
9394 if (OpInfo.ConstraintVT == MVT::Other)
9395 ValueVT = RegVT;
9396
9397 // Initialize NumRegs.
9398 unsigned NumRegs = 1;
9399 if (OpInfo.ConstraintVT != MVT::Other)
9400 NumRegs = TLI.getNumRegisters(Context, OpInfo.ConstraintVT, RegVT);
9401
9402 // If this is a constraint for a specific physical register, like {r17},
9403 // assign it now.
9404
9405 // If this associated to a specific register, initialize iterator to correct
9406 // place. If virtual, make sure we have enough registers
9407
9408 // Initialize iterator if necessary
9411
9412 // Do not check for single registers.
9413 if (AssignedReg) {
9414 I = std::find(I, RC->end(), AssignedReg);
9415 if (I == RC->end()) {
9416 // RC does not contain the selected register, which indicates a
9417 // mismatch between the register and the required type/bitwidth.
9418 return {AssignedReg};
9419 }
9420 }
9421
9422 for (; NumRegs; --NumRegs, ++I) {
9423 assert(I != RC->end() && "Ran out of registers to allocate!");
9424 Register R = AssignedReg ? Register(*I) : RegInfo.createVirtualRegister(RC);
9425 Regs.push_back(R);
9426 }
9427
9428 OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT);
9429 return std::nullopt;
9430}
9431
9432static unsigned
9434 const std::vector<SDValue> &AsmNodeOperands) {
9435 // Scan until we find the definition we already emitted of this operand.
9436 unsigned CurOp = InlineAsm::Op_FirstOperand;
9437 for (; OperandNo; --OperandNo) {
9438 // Advance to the next operand.
9439 unsigned OpFlag = AsmNodeOperands[CurOp]->getAsZExtVal();
9440 const InlineAsm::Flag F(OpFlag);
9441 assert(
9442 (F.isRegDefKind() || F.isRegDefEarlyClobberKind() || F.isMemKind()) &&
9443 "Skipped past definitions?");
9444 CurOp += F.getNumOperandRegisters() + 1;
9445 }
9446 return CurOp;
9447}
9448
9449namespace {
9450
9451class ExtraFlags {
9452 unsigned Flags = 0;
9453
9454public:
9455 explicit ExtraFlags(const CallBase &Call) {
9456 const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand());
9457 if (IA->hasSideEffects())
9459 if (IA->isAlignStack())
9461 if (Call.isConvergent())
9463 Flags |= IA->getDialect() * InlineAsm::Extra_AsmDialect;
9464 }
9465
9466 void update(const TargetLowering::AsmOperandInfo &OpInfo) {
9467 // Ideally, we would only check against memory constraints. However, the
9468 // meaning of an Other constraint can be target-specific and we can't easily
9469 // reason about it. Therefore, be conservative and set MayLoad/MayStore
9470 // for Other constraints as well.
9473 if (OpInfo.Type == InlineAsm::isInput)
9475 else if (OpInfo.Type == InlineAsm::isOutput)
9477 else if (OpInfo.Type == InlineAsm::isClobber)
9479 }
9480 }
9481
9482 unsigned get() const { return Flags; }
9483};
9484
9485} // end anonymous namespace
9486
9487static bool isFunction(SDValue Op) {
9488 if (Op && Op.getOpcode() == ISD::GlobalAddress) {
9489 if (auto *GA = dyn_cast<GlobalAddressSDNode>(Op)) {
9490 auto Fn = dyn_cast_or_null<Function>(GA->getGlobal());
9491
9492 // In normal "call dllimport func" instruction (non-inlineasm) it force
9493 // indirect access by specifing call opcode. And usually specially print
9494 // asm with indirect symbol (i.g: "*") according to opcode. Inline asm can
9495 // not do in this way now. (In fact, this is similar with "Data Access"
9496 // action). So here we ignore dllimport function.
9497 if (Fn && !Fn->hasDLLImportStorageClass())
9498 return true;
9499 }
9500 }
9501 return false;
9502}
9503
9504/// visitInlineAsm - Handle a call to an InlineAsm object.
9505void SelectionDAGBuilder::visitInlineAsm(const CallBase &Call,
9506 const BasicBlock *EHPadBB) {
9507 const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand());
9508
9509 /// ConstraintOperands - Information about all of the constraints.
9510 SmallVector<SDISelAsmOperandInfo, 16> ConstraintOperands;
9511
9515
9516 // First Pass: Calculate HasSideEffects and ExtraFlags (AlignStack,
9517 // AsmDialect, MayLoad, MayStore).
9518 bool HasSideEffect = IA->hasSideEffects();
9519 ExtraFlags ExtraInfo(Call);
9520
9521 for (auto &T : TargetConstraints) {
9522 ConstraintOperands.push_back(SDISelAsmOperandInfo(T));
9523 SDISelAsmOperandInfo &OpInfo = ConstraintOperands.back();
9524
9525 if (OpInfo.CallOperandVal)
9526 OpInfo.CallOperand = getValue(OpInfo.CallOperandVal);
9527
9528 if (!HasSideEffect)
9529 HasSideEffect = OpInfo.hasMemory(TLI);
9530
9531 // Determine if this InlineAsm MayLoad or MayStore based on the constraints.
9532 // FIXME: Could we compute this on OpInfo rather than T?
9533
9534 // Compute the constraint code and ConstraintType to use.
9536
9537 if (T.ConstraintType == TargetLowering::C_Immediate &&
9538 OpInfo.CallOperand && !isa<ConstantSDNode>(OpInfo.CallOperand))
9539 // We've delayed emitting a diagnostic like the "n" constraint because
9540 // inlining could cause an integer showing up.
9541 return emitInlineAsmError(Call, "constraint '" + Twine(T.ConstraintCode) +
9542 "' expects an integer constant "
9543 "expression");
9544
9545 ExtraInfo.update(T);
9546 }
9547
9548 // We won't need to flush pending loads if this asm doesn't touch
9549 // memory and is nonvolatile.
9550 SDValue Glue, Chain = (HasSideEffect) ? getRoot() : DAG.getRoot();
9551
9552 bool EmitEHLabels = isa<InvokeInst>(Call);
9553 if (EmitEHLabels) {
9554 assert(EHPadBB && "InvokeInst must have an EHPadBB");
9555 }
9556 bool IsCallBr = isa<CallBrInst>(Call);
9557
9558 if (IsCallBr || EmitEHLabels) {
9559 // If this is a callbr or invoke we need to flush pending exports since
9560 // inlineasm_br and invoke are terminators.
9561 // We need to do this before nodes are glued to the inlineasm_br node.
9562 Chain = getControlRoot();
9563 }
9564
9565 MCSymbol *BeginLabel = nullptr;
9566 if (EmitEHLabels) {
9567 Chain = lowerStartEH(Chain, EHPadBB, BeginLabel);
9568 }
9569
9570 int OpNo = -1;
9571 SmallVector<StringRef> AsmStrs;
9572 IA->collectAsmStrs(AsmStrs);
9573
9574 // Second pass over the constraints: compute which constraint option to use.
9575 for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {
9576 if (OpInfo.hasArg() || OpInfo.Type == InlineAsm::isOutput)
9577 OpNo++;
9578
9579 // If this is an output operand with a matching input operand, look up the
9580 // matching input. If their types mismatch, e.g. one is an integer, the
9581 // other is floating point, or their sizes are different, flag it as an
9582 // error.
9583 if (OpInfo.hasMatchingInput()) {
9584 SDISelAsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
9585 patchMatchingInput(OpInfo, Input, DAG);
9586 }
9587
9588 // Compute the constraint code and ConstraintType to use.
9589 TLI.ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG);
9590
9591 if ((OpInfo.ConstraintType == TargetLowering::C_Memory &&
9592 OpInfo.Type == InlineAsm::isClobber) ||
9593 OpInfo.ConstraintType == TargetLowering::C_Address)
9594 continue;
9595
9596 // In Linux PIC model, there are 4 cases about value/label addressing:
9597 //
9598 // 1: Function call or Label jmp inside the module.
9599 // 2: Data access (such as global variable, static variable) inside module.
9600 // 3: Function call or Label jmp outside the module.
9601 // 4: Data access (such as global variable) outside the module.
9602 //
9603 // Due to current llvm inline asm architecture designed to not "recognize"
9604 // the asm code, there are quite troubles for us to treat mem addressing
9605 // differently for same value/adress used in different instuctions.
9606 // For example, in pic model, call a func may in plt way or direclty
9607 // pc-related, but lea/mov a function adress may use got.
9608 //
9609 // Here we try to "recognize" function call for the case 1 and case 3 in
9610 // inline asm. And try to adjust the constraint for them.
9611 //
9612 // TODO: Due to current inline asm didn't encourage to jmp to the outsider
9613 // label, so here we don't handle jmp function label now, but we need to
9614 // enhance it (especilly in PIC model) if we meet meaningful requirements.
9615 if (OpInfo.isIndirect && isFunction(OpInfo.CallOperand) &&
9616 TLI.isInlineAsmTargetBranch(AsmStrs, OpNo) &&
9618 OpInfo.isIndirect = false;
9619 OpInfo.ConstraintType = TargetLowering::C_Address;
9620 }
9621
9622 // If this is a memory input, and if the operand is not indirect, do what we
9623 // need to provide an address for the memory input.
9624 if (OpInfo.ConstraintType == TargetLowering::C_Memory &&
9625 !OpInfo.isIndirect) {
9626 assert((OpInfo.isMultipleAlternative ||
9627 (OpInfo.Type == InlineAsm::isInput)) &&
9628 "Can only indirectify direct input operands!");
9629
9630 // Memory operands really want the address of the value.
9631 Chain = getAddressForMemoryInput(Chain, getCurSDLoc(), OpInfo, DAG);
9632
9633 // There is no longer a Value* corresponding to this operand.
9634 OpInfo.CallOperandVal = nullptr;
9635
9636 // It is now an indirect operand.
9637 OpInfo.isIndirect = true;
9638 }
9639
9640 }
9641
9642 // AsmNodeOperands - The operands for the ISD::INLINEASM node.
9643 std::vector<SDValue> AsmNodeOperands;
9644 AsmNodeOperands.push_back(SDValue()); // reserve space for input chain
9645 AsmNodeOperands.push_back(DAG.getTargetExternalSymbol(
9646 IA->getAsmString().c_str(), TLI.getProgramPointerTy(DAG.getDataLayout())));
9647
9648 // If we have a !srcloc metadata node associated with it, we want to attach
9649 // this to the ultimately generated inline asm machineinstr. To do this, we
9650 // pass in the third operand as this (potentially null) inline asm MDNode.
9651 const MDNode *SrcLoc = Call.getMetadata("srcloc");
9652 AsmNodeOperands.push_back(DAG.getMDNode(SrcLoc));
9653
9654 // Remember the HasSideEffect, AlignStack, AsmDialect, MayLoad and MayStore
9655 // bits as operand 3.
9656 AsmNodeOperands.push_back(DAG.getTargetConstant(
9657 ExtraInfo.get(), getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
9658
9659 // Third pass: Loop over operands to prepare DAG-level operands.. As part of
9660 // this, assign virtual and physical registers for inputs and otput.
9661 for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {
9662 // Assign Registers.
9663 SDISelAsmOperandInfo &RefOpInfo =
9664 OpInfo.isMatchingInputConstraint()
9665 ? ConstraintOperands[OpInfo.getMatchedOperand()]
9666 : OpInfo;
9667 const auto RegError =
9668 getRegistersForValue(DAG, getCurSDLoc(), OpInfo, RefOpInfo);
9669 if (RegError) {
9672 const char *RegName = TRI.getName(*RegError);
9673 emitInlineAsmError(Call, "register '" + Twine(RegName) +
9674 "' allocated for constraint '" +
9675 Twine(OpInfo.ConstraintCode) +
9676 "' does not match required type");
9677 return;
9678 }
9679
9680 auto DetectWriteToReservedRegister = [&]() {
9683 for (unsigned Reg : OpInfo.AssignedRegs.Regs) {
9685 TRI.isInlineAsmReadOnlyReg(MF, Reg)) {
9686 const char *RegName = TRI.getName(Reg);
9687 emitInlineAsmError(Call, "write to reserved register '" +
9688 Twine(RegName) + "'");
9689 return true;
9690 }
9691 }
9692 return false;
9693 };
9694 assert((OpInfo.ConstraintType != TargetLowering::C_Address ||
9695 (OpInfo.Type == InlineAsm::isInput &&
9696 !OpInfo.isMatchingInputConstraint())) &&
9697 "Only address as input operand is allowed.");
9698
9699 switch (OpInfo.Type) {
9701 if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
9702 const InlineAsm::ConstraintCode ConstraintID =
9703 TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
9705 "Failed to convert memory constraint code to constraint id.");
9706
9707 // Add information to the INLINEASM node to know about this output.
9709 OpFlags.setMemConstraint(ConstraintID);
9710 AsmNodeOperands.push_back(DAG.getTargetConstant(OpFlags, getCurSDLoc(),
9711 MVT::i32));
9712 AsmNodeOperands.push_back(OpInfo.CallOperand);
9713 } else {
9714 // Otherwise, this outputs to a register (directly for C_Register /
9715 // C_RegisterClass, and a target-defined fashion for
9716 // C_Immediate/C_Other). Find a register that we can use.
9717 if (OpInfo.AssignedRegs.Regs.empty()) {
9718 emitInlineAsmError(
9719 Call, "couldn't allocate output register for constraint '" +
9720 Twine(OpInfo.ConstraintCode) + "'");
9721 return;
9722 }
9723
9724 if (DetectWriteToReservedRegister())
9725 return;
9726
9727 // Add information to the INLINEASM node to know that this register is
9728 // set.
9729 OpInfo.AssignedRegs.AddInlineAsmOperands(
9730 OpInfo.isEarlyClobber ? InlineAsm::Kind::RegDefEarlyClobber
9732 false, 0, getCurSDLoc(), DAG, AsmNodeOperands);
9733 }
9734 break;
9735
9736 case InlineAsm::isInput:
9737 case InlineAsm::isLabel: {
9738 SDValue InOperandVal = OpInfo.CallOperand;
9739
9740 if (OpInfo.isMatchingInputConstraint()) {
9741 // If this is required to match an output register we have already set,
9742 // just use its register.
9743 auto CurOp = findMatchingInlineAsmOperand(OpInfo.getMatchedOperand(),
9744 AsmNodeOperands);
9745 InlineAsm::Flag Flag(AsmNodeOperands[CurOp]->getAsZExtVal());
9746 if (Flag.isRegDefKind() || Flag.isRegDefEarlyClobberKind()) {
9747 if (OpInfo.isIndirect) {
9748 // This happens on gcc/testsuite/gcc.dg/pr8788-1.c
9749 emitInlineAsmError(Call, "inline asm not supported yet: "
9750 "don't know how to handle tied "
9751 "indirect register inputs");
9752 return;
9753 }
9754
9759 auto *R = cast<RegisterSDNode>(AsmNodeOperands[CurOp+1]);
9760 Register TiedReg = R->getReg();
9761 MVT RegVT = R->getSimpleValueType(0);
9762 const TargetRegisterClass *RC =
9763 TiedReg.isVirtual() ? MRI.getRegClass(TiedReg)
9764 : RegVT != MVT::Untyped ? TLI.getRegClassFor(RegVT)
9765 : TRI.getMinimalPhysRegClass(TiedReg);
9766 for (unsigned i = 0, e = Flag.getNumOperandRegisters(); i != e; ++i)
9767 Regs.push_back(MRI.createVirtualRegister(RC));
9768
9769 RegsForValue MatchedRegs(Regs, RegVT, InOperandVal.getValueType());
9770
9771 SDLoc dl = getCurSDLoc();
9772 // Use the produced MatchedRegs object to
9773 MatchedRegs.getCopyToRegs(InOperandVal, DAG, dl, Chain, &Glue, &Call);
9774 MatchedRegs.AddInlineAsmOperands(InlineAsm::Kind::RegUse, true,
9775 OpInfo.getMatchedOperand(), dl, DAG,
9776 AsmNodeOperands);
9777 break;
9778 }
9779
9780 assert(Flag.isMemKind() && "Unknown matching constraint!");
9781 assert(Flag.getNumOperandRegisters() == 1 &&
9782 "Unexpected number of operands");
9783 // Add information to the INLINEASM node to know about this input.
9784 // See InlineAsm.h isUseOperandTiedToDef.
9785 Flag.clearMemConstraint();
9786 Flag.setMatchingOp(OpInfo.getMatchedOperand());
9787 AsmNodeOperands.push_back(DAG.getTargetConstant(
9788 Flag, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
9789 AsmNodeOperands.push_back(AsmNodeOperands[CurOp+1]);
9790 break;
9791 }
9792
9793 // Treat indirect 'X' constraint as memory.
9794 if (OpInfo.ConstraintType == TargetLowering::C_Other &&
9795 OpInfo.isIndirect)
9796 OpInfo.ConstraintType = TargetLowering::C_Memory;
9797
9798 if (OpInfo.ConstraintType == TargetLowering::C_Immediate ||
9799 OpInfo.ConstraintType == TargetLowering::C_Other) {
9800 std::vector<SDValue> Ops;
9801 TLI.LowerAsmOperandForConstraint(InOperandVal, OpInfo.ConstraintCode,
9802 Ops, DAG);
9803 if (Ops.empty()) {
9804 if (OpInfo.ConstraintType == TargetLowering::C_Immediate)
9805 if (isa<ConstantSDNode>(InOperandVal)) {
9806 emitInlineAsmError(Call, "value out of range for constraint '" +
9807 Twine(OpInfo.ConstraintCode) + "'");
9808 return;
9809 }
9810
9811 emitInlineAsmError(Call,
9812 "invalid operand for inline asm constraint '" +
9813 Twine(OpInfo.ConstraintCode) + "'");
9814 return;
9815 }
9816
9817 // Add information to the INLINEASM node to know about this input.
9818 InlineAsm::Flag ResOpType(InlineAsm::Kind::Imm, Ops.size());
9819 AsmNodeOperands.push_back(DAG.getTargetConstant(
9820 ResOpType, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
9821 llvm::append_range(AsmNodeOperands, Ops);
9822 break;
9823 }
9824
9825 if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
9826 assert((OpInfo.isIndirect ||
9827 OpInfo.ConstraintType != TargetLowering::C_Memory) &&
9828 "Operand must be indirect to be a mem!");
9829 assert(InOperandVal.getValueType() ==
9831 "Memory operands expect pointer values");
9832
9833 const InlineAsm::ConstraintCode ConstraintID =
9834 TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
9836 "Failed to convert memory constraint code to constraint id.");
9837
9838 // Add information to the INLINEASM node to know about this input.
9840 ResOpType.setMemConstraint(ConstraintID);
9841 AsmNodeOperands.push_back(DAG.getTargetConstant(ResOpType,
9842 getCurSDLoc(),
9843 MVT::i32));
9844 AsmNodeOperands.push_back(InOperandVal);
9845 break;
9846 }
9847
9848 if (OpInfo.ConstraintType == TargetLowering::C_Address) {
9849 const InlineAsm::ConstraintCode ConstraintID =
9850 TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
9852 "Failed to convert memory constraint code to constraint id.");
9853
9855
9856 SDValue AsmOp = InOperandVal;
9857 if (isFunction(InOperandVal)) {
9858 auto *GA = cast<GlobalAddressSDNode>(InOperandVal);
9859 ResOpType = InlineAsm::Flag(InlineAsm::Kind::Func, 1);
9860 AsmOp = DAG.getTargetGlobalAddress(GA->getGlobal(), getCurSDLoc(),
9861 InOperandVal.getValueType(),
9862 GA->getOffset());
9863 }
9864
9865 // Add information to the INLINEASM node to know about this input.
9866 ResOpType.setMemConstraint(ConstraintID);
9867
9868 AsmNodeOperands.push_back(
9869 DAG.getTargetConstant(ResOpType, getCurSDLoc(), MVT::i32));
9870
9871 AsmNodeOperands.push_back(AsmOp);
9872 break;
9873 }
9874
9875 assert((OpInfo.ConstraintType == TargetLowering::C_RegisterClass ||
9876 OpInfo.ConstraintType == TargetLowering::C_Register) &&
9877 "Unknown constraint type!");
9878
9879 // TODO: Support this.
9880 if (OpInfo.isIndirect) {
9881 emitInlineAsmError(
9882 Call, "Don't know how to handle indirect register inputs yet "
9883 "for constraint '" +
9884 Twine(OpInfo.ConstraintCode) + "'");
9885 return;
9886 }
9887
9888 // Copy the input into the appropriate registers.
9889 if (OpInfo.AssignedRegs.Regs.empty()) {
9890 emitInlineAsmError(Call,
9891 "couldn't allocate input reg for constraint '" +
9892 Twine(OpInfo.ConstraintCode) + "'");
9893 return;
9894 }
9895
9896 if (DetectWriteToReservedRegister())
9897 return;
9898
9899 SDLoc dl = getCurSDLoc();
9900
9901 OpInfo.AssignedRegs.getCopyToRegs(InOperandVal, DAG, dl, Chain, &Glue,
9902 &Call);
9903
9904 OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind::RegUse, false,
9905 0, dl, DAG, AsmNodeOperands);
9906 break;
9907 }
9909 // Add the clobbered value to the operand list, so that the register
9910 // allocator is aware that the physreg got clobbered.
9911 if (!OpInfo.AssignedRegs.Regs.empty())
9912 OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind::Clobber,
9913 false, 0, getCurSDLoc(), DAG,
9914 AsmNodeOperands);
9915 break;
9916 }
9917 }
9918
9919 // Finish up input operands. Set the input chain and add the flag last.
9920 AsmNodeOperands[InlineAsm::Op_InputChain] = Chain;
9921 if (Glue.getNode()) AsmNodeOperands.push_back(Glue);
9922
9923 unsigned ISDOpc = IsCallBr ? ISD::INLINEASM_BR : ISD::INLINEASM;
9924 Chain = DAG.getNode(ISDOpc, getCurSDLoc(),
9925 DAG.getVTList(MVT::Other, MVT::Glue), AsmNodeOperands);
9926 Glue = Chain.getValue(1);
9927
9928 // Do additional work to generate outputs.
9929
9930 SmallVector<EVT, 1> ResultVTs;
9931 SmallVector<SDValue, 1> ResultValues;
9932 SmallVector<SDValue, 8> OutChains;
9933
9934 llvm::Type *CallResultType = Call.getType();
9935 ArrayRef<Type *> ResultTypes;
9936 if (StructType *StructResult = dyn_cast<StructType>(CallResultType))
9937 ResultTypes = StructResult->elements();
9938 else if (!CallResultType->isVoidTy())
9939 ResultTypes = ArrayRef(CallResultType);
9940
9941 auto CurResultType = ResultTypes.begin();
9942 auto handleRegAssign = [&](SDValue V) {
9943 assert(CurResultType != ResultTypes.end() && "Unexpected value");
9944 assert((*CurResultType)->isSized() && "Unexpected unsized type");
9945 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), *CurResultType);
9946 ++CurResultType;
9947 // If the type of the inline asm call site return value is different but has
9948 // same size as the type of the asm output bitcast it. One example of this
9949 // is for vectors with different width / number of elements. This can
9950 // happen for register classes that can contain multiple different value
9951 // types. The preg or vreg allocated may not have the same VT as was
9952 // expected.
9953 //
9954 // This can also happen for a return value that disagrees with the register
9955 // class it is put in, eg. a double in a general-purpose register on a
9956 // 32-bit machine.
9957 if (ResultVT != V.getValueType() &&
9958 ResultVT.getSizeInBits() == V.getValueSizeInBits())
9959 V = DAG.getNode(ISD::BITCAST, getCurSDLoc(), ResultVT, V);
9960 else if (ResultVT != V.getValueType() && ResultVT.isInteger() &&
9961 V.getValueType().isInteger()) {
9962 // If a result value was tied to an input value, the computed result
9963 // may have a wider width than the expected result. Extract the
9964 // relevant portion.
9965 V = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), ResultVT, V);
9966 }
9967 assert(ResultVT == V.getValueType() && "Asm result value mismatch!");
9968 ResultVTs.push_back(ResultVT);
9969 ResultValues.push_back(V);
9970 };
9971
9972 // Deal with output operands.
9973 for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {
9974 if (OpInfo.Type == InlineAsm::isOutput) {
9975 SDValue Val;
9976 // Skip trivial output operands.
9977 if (OpInfo.AssignedRegs.Regs.empty())
9978 continue;
9979
9980 switch (OpInfo.ConstraintType) {
9983 Val = OpInfo.AssignedRegs.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(),
9984 Chain, &Glue, &Call);
9985 break;
9988 Val = TLI.LowerAsmOutputForConstraint(Chain, Glue, getCurSDLoc(),
9989 OpInfo, DAG);
9990 break;
9992 break; // Already handled.
9994 break; // Silence warning.
9996 assert(false && "Unexpected unknown constraint");
9997 }
9998
9999 // Indirect output manifest as stores. Record output chains.
10000 if (OpInfo.isIndirect) {
10001 const Value *Ptr = OpInfo.CallOperandVal;
10002 assert(Ptr && "Expected value CallOperandVal for indirect asm operand");
10003 SDValue Store = DAG.getStore(Chain, getCurSDLoc(), Val, getValue(Ptr),
10005 OutChains.push_back(Store);
10006 } else {
10007 // generate CopyFromRegs to associated registers.
10008 assert(!Call.getType()->isVoidTy() && "Bad inline asm!");
10009 if (Val.getOpcode() == ISD::MERGE_VALUES) {
10010 for (const SDValue &V : Val->op_values())
10011 handleRegAssign(V);
10012 } else
10013 handleRegAssign(Val);
10014 }
10015 }
10016 }
10017
10018 // Set results.
10019 if (!ResultValues.empty()) {
10020 assert(CurResultType == ResultTypes.end() &&
10021 "Mismatch in number of ResultTypes");
10022 assert(ResultValues.size() == ResultTypes.size() &&
10023 "Mismatch in number of output operands in asm result");
10024
10026 DAG.getVTList(ResultVTs), ResultValues);
10027 setValue(&Call, V);
10028 }
10029
10030 // Collect store chains.
10031 if (!OutChains.empty())
10032 Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, OutChains);
10033
10034 if (EmitEHLabels) {
10035 Chain = lowerEndEH(Chain, cast<InvokeInst>(&Call), EHPadBB, BeginLabel);
10036 }
10037
10038 // Only Update Root if inline assembly has a memory effect.
10039 if (ResultValues.empty() || HasSideEffect || !OutChains.empty() || IsCallBr ||
10040 EmitEHLabels)
10041 DAG.setRoot(Chain);
10042}
10043
10044void SelectionDAGBuilder::emitInlineAsmError(const CallBase &Call,
10045 const Twine &Message) {
10046 LLVMContext &Ctx = *DAG.getContext();
10047 Ctx.emitError(&Call, Message);
10048
10049 // Make sure we leave the DAG in a valid state
10051 SmallVector<EVT, 1> ValueVTs;
10052 ComputeValueVTs(TLI, DAG.getDataLayout(), Call.getType(), ValueVTs);
10053
10054 if (ValueVTs.empty())
10055 return;
10056
10058 for (unsigned i = 0, e = ValueVTs.size(); i != e; ++i)
10059 Ops.push_back(DAG.getUNDEF(ValueVTs[i]));
10060
10061 setValue(&Call, DAG.getMergeValues(Ops, getCurSDLoc()));
10062}
10063
10064void SelectionDAGBuilder::visitVAStart(const CallInst &I) {
10066 MVT::Other, getRoot(),
10067 getValue(I.getArgOperand(0)),
10068 DAG.getSrcValue(I.getArgOperand(0))));
10069}
10070
10071void SelectionDAGBuilder::visitVAArg(const VAArgInst &I) {
10073 const DataLayout &DL = DAG.getDataLayout();
10075 TLI.getMemValueType(DAG.getDataLayout(), I.getType()), getCurSDLoc(),
10076 getRoot(), getValue(I.getOperand(0)), DAG.getSrcValue(I.getOperand(0)),
10077 DL.getABITypeAlign(I.getType()).value());
10078 DAG.setRoot(V.getValue(1));
10079
10080 if (I.getType()->isPointerTy())
10082 V, getCurSDLoc(), TLI.getValueType(DAG.getDataLayout(), I.getType()));
10083 setValue(&I, V);
10084}
10085
10086void SelectionDAGBuilder::visitVAEnd(const CallInst &I) {
10088 MVT::Other, getRoot(),
10089 getValue(I.getArgOperand(0)),
10090 DAG.getSrcValue(I.getArgOperand(0))));
10091}
10092
10093void SelectionDAGBuilder::visitVACopy(const CallInst &I) {
10095 MVT::Other, getRoot(),
10096 getValue(I.getArgOperand(0)),
10097 getValue(I.getArgOperand(1)),
10098 DAG.getSrcValue(I.getArgOperand(0)),
10099 DAG.getSrcValue(I.getArgOperand(1))));
10100}
10101
10103 const Instruction &I,
10104 SDValue Op) {
10105 const MDNode *Range = getRangeMetadata(I);
10106 if (!Range)
10107 return Op;
10108
10110 if (CR.isFullSet() || CR.isEmptySet() || CR.isUpperWrapped())
10111 return Op;
10112
10113 APInt Lo = CR.getUnsignedMin();
10114 if (!Lo.isMinValue())
10115 return Op;
10116
10117 APInt Hi = CR.getUnsignedMax();
10118 unsigned Bits = std::max(Hi.getActiveBits(),
10119 static_cast<unsigned>(IntegerType::MIN_INT_BITS));
10120
10121 EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), Bits);
10122
10123 SDLoc SL = getCurSDLoc();
10124
10125 SDValue ZExt = DAG.getNode(ISD::AssertZext, SL, Op.getValueType(), Op,
10126 DAG.getValueType(SmallVT));
10127 unsigned NumVals = Op.getNode()->getNumValues();
10128 if (NumVals == 1)
10129 return ZExt;
10130
10132
10133 Ops.push_back(ZExt);
10134 for (unsigned I = 1; I != NumVals; ++I)
10135 Ops.push_back(Op.getValue(I));
10136
10137 return DAG.getMergeValues(Ops, SL);
10138}
10139
10140/// Populate a CallLowerinInfo (into \p CLI) based on the properties of
10141/// the call being lowered.
10142///
10143/// This is a helper for lowering intrinsics that follow a target calling
10144/// convention or require stack pointer adjustment. Only a subset of the
10145/// intrinsic's operands need to participate in the calling convention.
10148 unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy,
10149 AttributeSet RetAttrs, bool IsPatchPoint) {
10151 Args.reserve(NumArgs);
10152
10153 // Populate the argument list.
10154 // Attributes for args start at offset 1, after the return attribute.
10155 for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs;
10156 ArgI != ArgE; ++ArgI) {
10157 const Value *V = Call->getOperand(ArgI);
10158
10159 assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
10160
10162 Entry.Node = getValue(V);
10163 Entry.Ty = V->getType();
10164 Entry.setAttributes(Call, ArgI);
10165 Args.push_back(Entry);
10166 }
10167
10169 .setChain(getRoot())
10170 .setCallee(Call->getCallingConv(), ReturnTy, Callee, std::move(Args),
10171 RetAttrs)
10172 .setDiscardResult(Call->use_empty())
10173 .setIsPatchPoint(IsPatchPoint)
10175 Call->countOperandBundlesOfType(LLVMContext::OB_preallocated) != 0);
10176}
10177
10178/// Add a stack map intrinsic call's live variable operands to a stackmap
10179/// or patchpoint target node's operand list.
10180///
10181/// Constants are converted to TargetConstants purely as an optimization to
10182/// avoid constant materialization and register allocation.
10183///
10184/// FrameIndex operands are converted to TargetFrameIndex so that ISEL does not
10185/// generate addess computation nodes, and so FinalizeISel can convert the
10186/// TargetFrameIndex into a DirectMemRefOp StackMap location. This avoids
10187/// address materialization and register allocation, but may also be required
10188/// for correctness. If a StackMap (or PatchPoint) intrinsic directly uses an
10189/// alloca in the entry block, then the runtime may assume that the alloca's
10190/// StackMap location can be read immediately after compilation and that the
10191/// location is valid at any point during execution (this is similar to the
10192/// assumption made by the llvm.gcroot intrinsic). If the alloca's location were
10193/// only available in a register, then the runtime would need to trap when
10194/// execution reaches the StackMap in order to read the alloca's location.
10195static void addStackMapLiveVars(const CallBase &Call, unsigned StartIdx,
10196 const SDLoc &DL, SmallVectorImpl<SDValue> &Ops,
10197 SelectionDAGBuilder &Builder) {
10198 SelectionDAG &DAG = Builder.DAG;
10199 for (unsigned I = StartIdx; I < Call.arg_size(); I++) {
10200 SDValue Op = Builder.getValue(Call.getArgOperand(I));
10201
10202 // Things on the stack are pointer-typed, meaning that they are already
10203 // legal and can be emitted directly to target nodes.
10204 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Op)) {
10205 Ops.push_back(DAG.getTargetFrameIndex(FI->getIndex(), Op.getValueType()));
10206 } else {
10207 // Otherwise emit a target independent node to be legalised.
10208 Ops.push_back(Builder.getValue(Call.getArgOperand(I)));
10209 }
10210 }
10211}
10212
10213/// Lower llvm.experimental.stackmap.
10214void SelectionDAGBuilder::visitStackmap(const CallInst &CI) {
10215 // void @llvm.experimental.stackmap(i64 <id>, i32 <numShadowBytes>,
10216 // [live variables...])
10217
10218 assert(CI.getType()->isVoidTy() && "Stackmap cannot return a value.");
10219
10220 SDValue Chain, InGlue, Callee;
10222
10223 SDLoc DL = getCurSDLoc();
10225
10226 // The stackmap intrinsic only records the live variables (the arguments
10227 // passed to it) and emits NOPS (if requested). Unlike the patchpoint
10228 // intrinsic, this won't be lowered to a function call. This means we don't
10229 // have to worry about calling conventions and target specific lowering code.
10230 // Instead we perform the call lowering right here.
10231 //
10232 // chain, flag = CALLSEQ_START(chain, 0, 0)
10233 // chain, flag = STACKMAP(id, nbytes, ..., chain, flag)
10234 // chain, flag = CALLSEQ_END(chain, 0, 0, flag)
10235 //
10236 Chain = DAG.getCALLSEQ_START(getRoot(), 0, 0, DL);
10237 InGlue = Chain.getValue(1);
10238
10239 // Add the STACKMAP operands, starting with DAG house-keeping.
10240 Ops.push_back(Chain);
10241 Ops.push_back(InGlue);
10242
10243 // Add the <id>, <numShadowBytes> operands.
10244 //
10245 // These do not require legalisation, and can be emitted directly to target
10246 // constant nodes.
10248 assert(ID.getValueType() == MVT::i64);
10249 SDValue IDConst =
10250 DAG.getTargetConstant(ID->getAsZExtVal(), DL, ID.getValueType());
10251 Ops.push_back(IDConst);
10252
10253 SDValue Shad = getValue(CI.getArgOperand(1));
10254 assert(Shad.getValueType() == MVT::i32);
10255 SDValue ShadConst =
10257 Ops.push_back(ShadConst);
10258
10259 // Add the live variables.
10260 addStackMapLiveVars(CI, 2, DL, Ops, *this);
10261
10262 // Create the STACKMAP node.
10263 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
10264 Chain = DAG.getNode(ISD::STACKMAP, DL, NodeTys, Ops);
10265 InGlue = Chain.getValue(1);
10266
10267 Chain = DAG.getCALLSEQ_END(Chain, 0, 0, InGlue, DL);
10268
10269 // Stackmaps don't generate values, so nothing goes into the NodeMap.
10270
10271 // Set the root to the target-lowered call chain.
10272 DAG.setRoot(Chain);
10273
10274 // Inform the Frame Information that we have a stackmap in this function.
10276}
10277
10278/// Lower llvm.experimental.patchpoint directly to its target opcode.
10279void SelectionDAGBuilder::visitPatchpoint(const CallBase &CB,
10280 const BasicBlock *EHPadBB) {
10281 // <ty> @llvm.experimental.patchpoint.<ty>(i64 <id>,
10282 // i32 <numBytes>,
10283 // i8* <target>,
10284 // i32 <numArgs>,
10285 // [Args...],
10286 // [live variables...])
10287
10289 bool IsAnyRegCC = CC == CallingConv::AnyReg;
10290 bool HasDef = !CB.getType()->isVoidTy();
10291 SDLoc dl = getCurSDLoc();
10293
10294 // Handle immediate and symbolic callees.
10295 if (auto* ConstCallee = dyn_cast<ConstantSDNode>(Callee))
10296 Callee = DAG.getIntPtrConstant(ConstCallee->getZExtValue(), dl,
10297 /*isTarget=*/true);
10298 else if (auto* SymbolicCallee = dyn_cast<GlobalAddressSDNode>(Callee))
10299 Callee = DAG.getTargetGlobalAddress(SymbolicCallee->getGlobal(),
10300 SDLoc(SymbolicCallee),
10301 SymbolicCallee->getValueType(0));
10302
10303 // Get the real number of arguments participating in the call <numArgs>
10305 unsigned NumArgs = NArgVal->getAsZExtVal();
10306
10307 // Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs>
10308 // Intrinsics include all meta-operands up to but not including CC.
10309 unsigned NumMetaOpers = PatchPointOpers::CCPos;
10310 assert(CB.arg_size() >= NumMetaOpers + NumArgs &&
10311 "Not enough arguments provided to the patchpoint intrinsic");
10312
10313 // For AnyRegCC the arguments are lowered later on manually.
10314 unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs;
10315 Type *ReturnTy =
10316 IsAnyRegCC ? Type::getVoidTy(*DAG.getContext()) : CB.getType();
10317
10319 populateCallLoweringInfo(CLI, &CB, NumMetaOpers, NumCallArgs, Callee,
10320 ReturnTy, CB.getAttributes().getRetAttrs(), true);
10321 std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
10322
10323 SDNode *CallEnd = Result.second.getNode();
10324 if (HasDef && (CallEnd->getOpcode() == ISD::CopyFromReg))
10325 CallEnd = CallEnd->getOperand(0).getNode();
10326
10327 /// Get a call instruction from the call sequence chain.
10328 /// Tail calls are not allowed.
10329 assert(CallEnd->getOpcode() == ISD::CALLSEQ_END &&
10330 "Expected a callseq node.");
10331 SDNode *Call = CallEnd->getOperand(0).getNode();
10332 bool HasGlue = Call->getGluedNode();
10333
10334 // Replace the target specific call node with the patchable intrinsic.
10336
10337 // Push the chain.
10338 Ops.push_back(*(Call->op_begin()));
10339
10340 // Optionally, push the glue (if any).
10341 if (HasGlue)
10342 Ops.push_back(*(Call->op_end() - 1));
10343
10344 // Push the register mask info.
10345 if (HasGlue)
10346 Ops.push_back(*(Call->op_end() - 2));
10347 else
10348 Ops.push_back(*(Call->op_end() - 1));
10349
10350 // Add the <id> and <numBytes> constants.
10352 Ops.push_back(DAG.getTargetConstant(IDVal->getAsZExtVal(), dl, MVT::i64));
10354 Ops.push_back(DAG.getTargetConstant(NBytesVal->getAsZExtVal(), dl, MVT::i32));
10355
10356 // Add the callee.
10357 Ops.push_back(Callee);
10358
10359 // Adjust <numArgs> to account for any arguments that have been passed on the
10360 // stack instead.
10361 // Call Node: Chain, Target, {Args}, RegMask, [Glue]
10362 unsigned NumCallRegArgs = Call->getNumOperands() - (HasGlue ? 4 : 3);
10363 NumCallRegArgs = IsAnyRegCC ? NumArgs : NumCallRegArgs;
10364 Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, dl, MVT::i32));
10365
10366 // Add the calling convention
10367 Ops.push_back(DAG.getTargetConstant((unsigned)CC, dl, MVT::i32));
10368
10369 // Add the arguments we omitted previously. The register allocator should
10370 // place these in any free register.
10371 if (IsAnyRegCC)
10372 for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i)
10373 Ops.push_back(getValue(CB.getArgOperand(i)));
10374
10375 // Push the arguments from the call instruction.
10376 SDNode::op_iterator e = HasGlue ? Call->op_end()-2 : Call->op_end()-1;
10377 Ops.append(Call->op_begin() + 2, e);
10378
10379 // Push live variables for the stack map.
10380 addStackMapLiveVars(CB, NumMetaOpers + NumArgs, dl, Ops, *this);
10381
10382 SDVTList NodeTys;
10383 if (IsAnyRegCC && HasDef) {
10384 // Create the return types based on the intrinsic definition
10386 SmallVector<EVT, 3> ValueVTs;
10387 ComputeValueVTs(TLI, DAG.getDataLayout(), CB.getType(), ValueVTs);
10388 assert(ValueVTs.size() == 1 && "Expected only one return value type.");
10389
10390 // There is always a chain and a glue type at the end
10391 ValueVTs.push_back(MVT::Other);
10392 ValueVTs.push_back(MVT::Glue);
10393 NodeTys = DAG.getVTList(ValueVTs);
10394 } else
10395 NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
10396
10397 // Replace the target specific call node with a PATCHPOINT node.
10398 SDValue PPV = DAG.getNode(ISD::PATCHPOINT, dl, NodeTys, Ops);
10399
10400 // Update the NodeMap.
10401 if (HasDef) {
10402 if (IsAnyRegCC)
10403 setValue(&CB, SDValue(PPV.getNode(), 0));
10404 else
10405 setValue(&CB, Result.first);
10406 }
10407
10408 // Fixup the consumers of the intrinsic. The chain and glue may be used in the
10409 // call sequence. Furthermore the location of the chain and glue can change
10410 // when the AnyReg calling convention is used and the intrinsic returns a
10411 // value.
10412 if (IsAnyRegCC && HasDef) {
10413 SDValue From[] = {SDValue(Call, 0), SDValue(Call, 1)};
10414 SDValue To[] = {PPV.getValue(1), PPV.getValue(2)};
10416 } else
10417 DAG.ReplaceAllUsesWith(Call, PPV.getNode());
10418 DAG.DeleteNode(Call);
10419
10420 // Inform the Frame Information that we have a patchpoint in this function.
10422}
10423
10424void SelectionDAGBuilder::visitVectorReduce(const CallInst &I,
10425 unsigned Intrinsic) {
10427 SDValue Op1 = getValue(I.getArgOperand(0));
10428 SDValue Op2;
10429 if (I.arg_size() > 1)
10430 Op2 = getValue(I.getArgOperand(1));
10431 SDLoc dl = getCurSDLoc();
10432 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
10433 SDValue Res;
10434 SDNodeFlags SDFlags;
10435 if (auto *FPMO = dyn_cast<FPMathOperator>(&I))
10436 SDFlags.copyFMF(*FPMO);
10437
10438 switch (Intrinsic) {
10439 case Intrinsic::vector_reduce_fadd:
10440 if (SDFlags.hasAllowReassociation())
10441 Res = DAG.getNode(ISD::FADD, dl, VT, Op1,
10442 DAG.getNode(ISD::VECREDUCE_FADD, dl, VT, Op2, SDFlags),
10443 SDFlags);
10444 else
10445 Res = DAG.getNode(ISD::VECREDUCE_SEQ_FADD, dl, VT, Op1, Op2, SDFlags);
10446 break;
10447 case Intrinsic::vector_reduce_fmul:
10448 if (SDFlags.hasAllowReassociation())
10449 Res = DAG.getNode(ISD::FMUL, dl, VT, Op1,
10450 DAG.getNode(ISD::VECREDUCE_FMUL, dl, VT, Op2, SDFlags),
10451 SDFlags);
10452 else
10453 Res = DAG.getNode(ISD::VECREDUCE_SEQ_FMUL, dl, VT, Op1, Op2, SDFlags);
10454 break;
10455 case Intrinsic::vector_reduce_add:
10456 Res = DAG.getNode(ISD::VECREDUCE_ADD, dl, VT, Op1);
10457 break;
10458 case Intrinsic::vector_reduce_mul:
10459 Res = DAG.getNode(ISD::VECREDUCE_MUL, dl, VT, Op1);
10460 break;
10461 case Intrinsic::vector_reduce_and:
10462 Res = DAG.getNode(ISD::VECREDUCE_AND, dl, VT, Op1);
10463 break;
10464 case Intrinsic::vector_reduce_or:
10465 Res = DAG.getNode(ISD::VECREDUCE_OR, dl, VT, Op1);
10466 break;
10467 case Intrinsic::vector_reduce_xor:
10468 Res = DAG.getNode(ISD::VECREDUCE_XOR, dl, VT, Op1);
10469 break;
10470 case Intrinsic::vector_reduce_smax:
10471 Res = DAG.getNode(ISD::VECREDUCE_SMAX, dl, VT, Op1);
10472 break;
10473 case Intrinsic::vector_reduce_smin:
10474 Res = DAG.getNode(ISD::VECREDUCE_SMIN, dl, VT, Op1);
10475 break;
10476 case Intrinsic::vector_reduce_umax:
10477 Res = DAG.getNode(ISD::VECREDUCE_UMAX, dl, VT, Op1);
10478 break;
10479 case Intrinsic::vector_reduce_umin:
10480 Res = DAG.getNode(ISD::VECREDUCE_UMIN, dl, VT, Op1);
10481 break;
10482 case Intrinsic::vector_reduce_fmax:
10483 Res = DAG.getNode(ISD::VECREDUCE_FMAX, dl, VT, Op1, SDFlags);
10484 break;
10485 case Intrinsic::vector_reduce_fmin:
10486 Res = DAG.getNode(ISD::VECREDUCE_FMIN, dl, VT, Op1, SDFlags);
10487 break;
10488 case Intrinsic::vector_reduce_fmaximum:
10489 Res = DAG.getNode(ISD::VECREDUCE_FMAXIMUM, dl, VT, Op1, SDFlags);
10490 break;
10491 case Intrinsic::vector_reduce_fminimum:
10492 Res = DAG.getNode(ISD::VECREDUCE_FMINIMUM, dl, VT, Op1, SDFlags);
10493 break;
10494 default:
10495 llvm_unreachable("Unhandled vector reduce intrinsic");
10496 }
10497 setValue(&I, Res);
10498}
10499
10500/// Returns an AttributeList representing the attributes applied to the return
10501/// value of the given call.
10504 if (CLI.RetSExt)
10505 Attrs.push_back(Attribute::SExt);
10506 if (CLI.RetZExt)
10507 Attrs.push_back(Attribute::ZExt);
10508 if (CLI.IsInReg)
10509 Attrs.push_back(Attribute::InReg);
10510
10512 Attrs);
10513}
10514
10515/// TargetLowering::LowerCallTo - This is the default LowerCallTo
10516/// implementation, which just calls LowerCall.
10517/// FIXME: When all targets are
10518/// migrated to using LowerCall, this hook should be integrated into SDISel.
10519std::pair<SDValue, SDValue>
10521 // Handle the incoming return values from the call.
10522 CLI.Ins.clear();
10523 Type *OrigRetTy = CLI.RetTy;
10524 SmallVector<EVT, 4> RetTys;
10526 auto &DL = CLI.DAG.getDataLayout();
10527 ComputeValueVTs(*this, DL, CLI.RetTy, RetTys, &Offsets);
10528
10529 if (CLI.IsPostTypeLegalization) {
10530 // If we are lowering a libcall after legalization, split the return type.
10531 SmallVector<EVT, 4> OldRetTys;
10532 SmallVector<TypeSize, 4> OldOffsets;
10533 RetTys.swap(OldRetTys);
10534 Offsets.swap(OldOffsets);
10535
10536 for (size_t i = 0, e = OldRetTys.size(); i != e; ++i) {
10537 EVT RetVT = OldRetTys[i];
10538 uint64_t Offset = OldOffsets[i];
10539 MVT RegisterVT = getRegisterType(CLI.RetTy->getContext(), RetVT);
10540 unsigned NumRegs = getNumRegisters(CLI.RetTy->getContext(), RetVT);
10541 unsigned RegisterVTByteSZ = RegisterVT.getSizeInBits() / 8;
10542 RetTys.append(NumRegs, RegisterVT);
10543 for (unsigned j = 0; j != NumRegs; ++j)
10544 Offsets.push_back(TypeSize::getFixed(Offset + j * RegisterVTByteSZ));
10545 }
10546 }
10547
10549 GetReturnInfo(CLI.CallConv, CLI.RetTy, getReturnAttrs(CLI), Outs, *this, DL);
10550
10551 bool CanLowerReturn =
10553 CLI.IsVarArg, Outs, CLI.RetTy->getContext());
10554
10555 SDValue DemoteStackSlot;
10556 int DemoteStackIdx = -100;
10557 if (!CanLowerReturn) {
10558 // FIXME: equivalent assert?
10559 // assert(!CS.hasInAllocaArgument() &&
10560 // "sret demotion is incompatible with inalloca");
10561 uint64_t TySize = DL.getTypeAllocSize(CLI.RetTy);
10562 Align Alignment = DL.getPrefTypeAlign(CLI.RetTy);
10564 DemoteStackIdx =
10565 MF.getFrameInfo().CreateStackObject(TySize, Alignment, false);
10566 Type *StackSlotPtrType = PointerType::get(CLI.RetTy,
10567 DL.getAllocaAddrSpace());
10568
10569 DemoteStackSlot = CLI.DAG.getFrameIndex(DemoteStackIdx, getFrameIndexTy(DL));
10570 ArgListEntry Entry;
10571 Entry.Node = DemoteStackSlot;
10572 Entry.Ty = StackSlotPtrType;
10573 Entry.IsSExt = false;
10574 Entry.IsZExt = false;
10575 Entry.IsInReg = false;
10576 Entry.IsSRet = true;
10577 Entry.IsNest = false;
10578 Entry.IsByVal = false;
10579 Entry.IsByRef = false;
10580 Entry.IsReturned = false;
10581 Entry.IsSwiftSelf = false;
10582 Entry.IsSwiftAsync = false;
10583 Entry.IsSwiftError = false;
10584 Entry.IsCFGuardTarget = false;
10585 Entry.Alignment = Alignment;
10586 CLI.getArgs().insert(CLI.getArgs().begin(), Entry);
10587 CLI.NumFixedArgs += 1;
10588 CLI.getArgs()[0].IndirectType = CLI.RetTy;
10589 CLI.RetTy = Type::getVoidTy(CLI.RetTy->getContext());
10590
10591 // sret demotion isn't compatible with tail-calls, since the sret argument
10592 // points into the callers stack frame.
10593 CLI.IsTailCall = false;
10594 } else {
10595 bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters(
10596 CLI.RetTy, CLI.CallConv, CLI.IsVarArg, DL);
10597 for (unsigned I = 0, E = RetTys.size(); I != E; ++I) {
10598 ISD::ArgFlagsTy Flags;
10599 if (NeedsRegBlock) {
10600 Flags.setInConsecutiveRegs();
10601 if (I == RetTys.size() - 1)
10602 Flags.setInConsecutiveRegsLast();
10603 }
10604 EVT VT = RetTys[I];
10606 CLI.CallConv, VT);
10607 unsigned NumRegs = getNumRegistersForCallingConv(CLI.RetTy->getContext(),
10608 CLI.CallConv, VT);
10609 for (unsigned i = 0; i != NumRegs; ++i) {
10610 ISD::InputArg MyFlags;
10611 MyFlags.Flags = Flags;
10612 MyFlags.VT = RegisterVT;
10613 MyFlags.ArgVT = VT;
10614 MyFlags.Used = CLI.IsReturnValueUsed;
10615 if (CLI.RetTy->isPointerTy()) {
10616 MyFlags.Flags.setPointer();
10617 MyFlags.Flags.setPointerAddrSpace(
10618 cast<PointerType>(CLI.RetTy)->getAddressSpace());
10619 }
10620 if (CLI.RetSExt)
10621 MyFlags.Flags.setSExt();
10622 if (CLI.RetZExt)
10623 MyFlags.Flags.setZExt();
10624 if (CLI.IsInReg)
10625 MyFlags.Flags.setInReg();
10626 CLI.Ins.push_back(MyFlags);
10627 }
10628 }
10629 }
10630
10631 // We push in swifterror return as the last element of CLI.Ins.
10632 ArgListTy &Args = CLI.getArgs();
10633 if (supportSwiftError()) {
10634 for (const ArgListEntry &Arg : Args) {
10635 if (Arg.IsSwiftError) {
10636 ISD::InputArg MyFlags;
10637 MyFlags.VT = getPointerTy(DL);
10638 MyFlags.ArgVT = EVT(getPointerTy(DL));
10639 MyFlags.Flags.setSwiftError();
10640 CLI.Ins.push_back(MyFlags);
10641 }
10642 }
10643 }
10644
10645 // Handle all of the outgoing arguments.
10646 CLI.Outs.clear();
10647 CLI.OutVals.clear();
10648 for (unsigned i = 0, e = Args.size(); i != e; ++i) {
10649 SmallVector<EVT, 4> ValueVTs;
10650 ComputeValueVTs(*this, DL, Args[i].Ty, ValueVTs);
10651 // FIXME: Split arguments if CLI.IsPostTypeLegalization
10652 Type *FinalType = Args[i].Ty;
10653 if (Args[i].IsByVal)
10654 FinalType = Args[i].IndirectType;
10655 bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters(
10656 FinalType, CLI.CallConv, CLI.IsVarArg, DL);
10657 for (unsigned Value = 0, NumValues = ValueVTs.size(); Value != NumValues;
10658 ++Value) {
10659 EVT VT = ValueVTs[Value];
10660 Type *ArgTy = VT.getTypeForEVT(CLI.RetTy->getContext());
10661 SDValue Op = SDValue(Args[i].Node.getNode(),
10662 Args[i].Node.getResNo() + Value);
10663 ISD::ArgFlagsTy Flags;
10664
10665 // Certain targets (such as MIPS), may have a different ABI alignment
10666 // for a type depending on the context. Give the target a chance to
10667 // specify the alignment it wants.
10668 const Align OriginalAlignment(getABIAlignmentForCallingConv(ArgTy, DL));
10669 Flags.setOrigAlign(OriginalAlignment);
10670
10671 if (Args[i].Ty->isPointerTy()) {
10672 Flags.setPointer();
10673 Flags.setPointerAddrSpace(
10674 cast<PointerType>(Args[i].Ty)->getAddressSpace());
10675 }
10676 if (Args[i].IsZExt)
10677 Flags.setZExt();
10678 if (Args[i].IsSExt)
10679 Flags.setSExt();
10680 if (Args[i].IsInReg) {
10681 // If we are using vectorcall calling convention, a structure that is
10682 // passed InReg - is surely an HVA
10684 isa<StructType>(FinalType)) {
10685 // The first value of a structure is marked
10686 if (0 == Value)
10687 Flags.setHvaStart();
10688 Flags.setHva();
10689 }
10690 // Set InReg Flag
10691 Flags.setInReg();
10692 }
10693 if (Args[i].IsSRet)
10694 Flags.setSRet();
10695 if (Args[i].IsSwiftSelf)
10696 Flags.setSwiftSelf();
10697 if (Args[i].IsSwiftAsync)
10698 Flags.setSwiftAsync();
10699 if (Args[i].IsSwiftError)
10700 Flags.setSwiftError();
10701 if (Args[i].IsCFGuardTarget)
10702 Flags.setCFGuardTarget();
10703 if (Args[i].IsByVal)
10704 Flags.setByVal();
10705 if (Args[i].IsByRef)
10706 Flags.setByRef();
10707 if (Args[i].IsPreallocated) {
10708 Flags.setPreallocated();
10709 // Set the byval flag for CCAssignFn callbacks that don't know about
10710 // preallocated. This way we can know how many bytes we should've
10711 // allocated and how many bytes a callee cleanup function will pop. If
10712 // we port preallocated to more targets, we'll have to add custom
10713 // preallocated handling in the various CC lowering callbacks.
10714 Flags.setByVal();
10715 }
10716 if (Args[i].IsInAlloca) {
10717 Flags.setInAlloca();
10718 // Set the byval flag for CCAssignFn callbacks that don't know about
10719 // inalloca. This way we can know how many bytes we should've allocated
10720 // and how many bytes a callee cleanup function will pop. If we port
10721 // inalloca to more targets, we'll have to add custom inalloca handling
10722 // in the various CC lowering callbacks.
10723 Flags.setByVal();
10724 }
10725 Align MemAlign;
10726 if (Args[i].IsByVal || Args[i].IsInAlloca || Args[i].IsPreallocated) {
10727 unsigned FrameSize = DL.getTypeAllocSize(Args[i].IndirectType);
10728 Flags.setByValSize(FrameSize);
10729
10730 // info is not there but there are cases it cannot get right.
10731 if (auto MA = Args[i].Alignment)
10732 MemAlign = *MA;
10733 else
10734 MemAlign = Align(getByValTypeAlignment(Args[i].IndirectType, DL));
10735 } else if (auto MA = Args[i].Alignment) {
10736 MemAlign = *MA;
10737 } else {
10738 MemAlign = OriginalAlignment;
10739 }
10740 Flags.setMemAlign(MemAlign);
10741 if (Args[i].IsNest)
10742 Flags.setNest();
10743 if (NeedsRegBlock)
10744 Flags.setInConsecutiveRegs();
10745
10747 CLI.CallConv, VT);
10748 unsigned NumParts = getNumRegistersForCallingConv(CLI.RetTy->getContext(),
10749 CLI.CallConv, VT);
10750 SmallVector<SDValue, 4> Parts(NumParts);
10751 ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
10752
10753 if (Args[i].IsSExt)
10754 ExtendKind = ISD::SIGN_EXTEND;
10755 else if (Args[i].IsZExt)
10756 ExtendKind = ISD::ZERO_EXTEND;
10757
10758 // Conservatively only handle 'returned' on non-vectors that can be lowered,
10759 // for now.
10760 if (Args[i].IsReturned && !Op.getValueType().isVector() &&
10762 assert((CLI.RetTy == Args[i].Ty ||
10763 (CLI.RetTy->isPointerTy() && Args[i].Ty->isPointerTy() &&
10765 Args[i].Ty->getPointerAddressSpace())) &&
10766 RetTys.size() == NumValues && "unexpected use of 'returned'");
10767 // Before passing 'returned' to the target lowering code, ensure that
10768 // either the register MVT and the actual EVT are the same size or that
10769 // the return value and argument are extended in the same way; in these
10770 // cases it's safe to pass the argument register value unchanged as the
10771 // return register value (although it's at the target's option whether
10772 // to do so)
10773 // TODO: allow code generation to take advantage of partially preserved
10774 // registers rather than clobbering the entire register when the
10775 // parameter extension method is not compatible with the return
10776 // extension method
10777 if ((NumParts * PartVT.getSizeInBits() == VT.getSizeInBits()) ||
10778 (ExtendKind != ISD::ANY_EXTEND && CLI.RetSExt == Args[i].IsSExt &&
10779 CLI.RetZExt == Args[i].IsZExt))
10780 Flags.setReturned();
10781 }
10782
10783 getCopyToParts(CLI.DAG, CLI.DL, Op, &Parts[0], NumParts, PartVT, CLI.CB,
10784 CLI.CallConv, ExtendKind);
10785
10786 for (unsigned j = 0; j != NumParts; ++j) {
10787 // if it isn't first piece, alignment must be 1
10788 // For scalable vectors the scalable part is currently handled
10789 // by individual targets, so we just use the known minimum size here.
10790 ISD::OutputArg MyFlags(
10791 Flags, Parts[j].getValueType().getSimpleVT(), VT,
10792 i < CLI.NumFixedArgs, i,
10793 j * Parts[j].getValueType().getStoreSize().getKnownMinValue());
10794 if (NumParts > 1 && j == 0)
10795 MyFlags.Flags.setSplit();
10796 else if (j != 0) {
10797 MyFlags.Flags.setOrigAlign(Align(1));
10798 if (j == NumParts - 1)
10799 MyFlags.Flags.setSplitEnd();
10800 }
10801
10802 CLI.Outs.push_back(MyFlags);
10803 CLI.OutVals.push_back(Parts[j]);
10804 }
10805
10806 if (NeedsRegBlock && Value == NumValues - 1)
10807 CLI.Outs[CLI.Outs.size() - 1].Flags.setInConsecutiveRegsLast();
10808 }
10809 }
10810
10812 CLI.Chain = LowerCall(CLI, InVals);
10813
10814 // Update CLI.InVals to use outside of this function.
10815 CLI.InVals = InVals;
10816
10817 // Verify that the target's LowerCall behaved as expected.
10818 assert(CLI.Chain.getNode() && CLI.Chain.getValueType() == MVT::Other &&
10819 "LowerCall didn't return a valid chain!");
10820 assert((!CLI.IsTailCall || InVals.empty()) &&
10821 "LowerCall emitted a return value for a tail call!");
10822 assert((CLI.IsTailCall || InVals.size() == CLI.Ins.size()) &&
10823 "LowerCall didn't emit the correct number of values!");
10824
10825 // For a tail call, the return value is merely live-out and there aren't
10826 // any nodes in the DAG representing it. Return a special value to
10827 // indicate that a tail call has been emitted and no more Instructions
10828 // should be processed in the current block.
10829 if (CLI.IsTailCall) {
10830 CLI.DAG.setRoot(CLI.Chain);
10831 return std::make_pair(SDValue(), SDValue());
10832 }
10833
10834#ifndef NDEBUG
10835 for (unsigned i = 0, e = CLI.Ins.size(); i != e; ++i) {
10836 assert(InVals[i].getNode() && "LowerCall emitted a null value!");
10837 assert(EVT(CLI.Ins[i].VT) == InVals[i].getValueType() &&
10838 "LowerCall emitted a value with the wrong type!");
10839 }
10840#endif
10841
10842 SmallVector<SDValue, 4> ReturnValues;
10843 if (!CanLowerReturn) {
10844 // The instruction result is the result of loading from the
10845 // hidden sret parameter.
10847 Type *PtrRetTy =
10848 PointerType::get(OrigRetTy->getContext(), DL.getAllocaAddrSpace());
10849
10850 ComputeValueVTs(*this, DL, PtrRetTy, PVTs);
10851 assert(PVTs.size() == 1 && "Pointers should fit in one register");
10852 EVT PtrVT = PVTs[0];
10853
10854 unsigned NumValues = RetTys.size();
10855 ReturnValues.resize(NumValues);
10856 SmallVector<SDValue, 4> Chains(NumValues);
10857
10858 // An aggregate return value cannot wrap around the address space, so
10859 // offsets to its parts don't wrap either.
10860 SDNodeFlags Flags;
10861 Flags.setNoUnsignedWrap(true);
10862
10864 Align HiddenSRetAlign = MF.getFrameInfo().getObjectAlign(DemoteStackIdx);
10865 for (unsigned i = 0; i < NumValues; ++i) {
10866 SDValue Add = CLI.DAG.getNode(ISD::ADD, CLI.DL, PtrVT, DemoteStackSlot,
10867 CLI.DAG.getConstant(Offsets[i], CLI.DL,
10868 PtrVT), Flags);
10869 SDValue L = CLI.DAG.getLoad(
10870 RetTys[i], CLI.DL, CLI.Chain, Add,
10872 DemoteStackIdx, Offsets[i]),
10873 HiddenSRetAlign);
10874 ReturnValues[i] = L;
10875 Chains[i] = L.getValue(1);
10876 }
10877
10878 CLI.Chain = CLI.DAG.getNode(ISD::TokenFactor, CLI.DL, MVT::Other, Chains);
10879 } else {
10880 // Collect the legal value parts into potentially illegal values
10881 // that correspond to the original function's return values.
10882 std::optional<ISD::NodeType> AssertOp;
10883 if (CLI.RetSExt)
10884 AssertOp = ISD::AssertSext;
10885 else if (CLI.RetZExt)
10886 AssertOp = ISD::AssertZext;
10887 unsigned CurReg = 0;
10888 for (EVT VT : RetTys) {
10890 CLI.CallConv, VT);
10891 unsigned NumRegs = getNumRegistersForCallingConv(CLI.RetTy->getContext(),
10892 CLI.CallConv, VT);
10893
10894 ReturnValues.push_back(getCopyFromParts(
10895 CLI.DAG, CLI.DL, &InVals[CurReg], NumRegs, RegisterVT, VT, nullptr,
10896 CLI.Chain, CLI.CallConv, AssertOp));
10897 CurReg += NumRegs;
10898 }
10899
10900 // For a function returning void, there is no return value. We can't create
10901 // such a node, so we just return a null return value in that case. In
10902 // that case, nothing will actually look at the value.
10903 if (ReturnValues.empty())
10904 return std::make_pair(SDValue(), CLI.Chain);
10905 }
10906
10907 SDValue Res = CLI.DAG.getNode(ISD::MERGE_VALUES, CLI.DL,
10908 CLI.DAG.getVTList(RetTys), ReturnValues);
10909 return std::make_pair(Res, CLI.Chain);
10910}
10911
10912/// Places new result values for the node in Results (their number
10913/// and types must exactly match those of the original return values of
10914/// the node), or leaves Results empty, which indicates that the node is not
10915/// to be custom lowered after all.
10918 SelectionDAG &DAG) const {
10919 SDValue Res = LowerOperation(SDValue(N, 0), DAG);
10920
10921 if (!Res.getNode())
10922 return;
10923
10924 // If the original node has one result, take the return value from
10925 // LowerOperation as is. It might not be result number 0.
10926 if (N->getNumValues() == 1) {
10927 Results.push_back(Res);
10928 return;
10929 }
10930
10931 // If the original node has multiple results, then the return node should
10932 // have the same number of results.
10933 assert((N->getNumValues() == Res->getNumValues()) &&
10934 "Lowering returned the wrong number of results!");
10935
10936 // Places new result values base on N result number.
10937 for (unsigned I = 0, E = N->getNumValues(); I != E; ++I)
10938 Results.push_back(Res.getValue(I));
10939}
10940
10942 llvm_unreachable("LowerOperation not implemented for this target!");
10943}
10944
10946 unsigned Reg,
10947 ISD::NodeType ExtendType) {
10949 assert((Op.getOpcode() != ISD::CopyFromReg ||
10950 cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) &&
10951 "Copy from a reg to the same reg!");
10952 assert(!Register::isPhysicalRegister(Reg) && "Is a physreg");
10953
10955 // If this is an InlineAsm we have to match the registers required, not the
10956 // notional registers required by the type.
10957
10958 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg, V->getType(),
10959 std::nullopt); // This is not an ABI copy.
10960 SDValue Chain = DAG.getEntryNode();
10961
10962 if (ExtendType == ISD::ANY_EXTEND) {
10963 auto PreferredExtendIt = FuncInfo.PreferredExtendType.find(V);
10964 if (PreferredExtendIt != FuncInfo.PreferredExtendType.end())
10965 ExtendType = PreferredExtendIt->second;
10966 }
10967 RFV.getCopyToRegs(Op, DAG, getCurSDLoc(), Chain, nullptr, V, ExtendType);
10968 PendingExports.push_back(Chain);
10969}
10970
10972
10973/// isOnlyUsedInEntryBlock - If the specified argument is only used in the
10974/// entry block, return true. This includes arguments used by switches, since
10975/// the switch may expand into multiple basic blocks.
10976static bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel) {
10977 // With FastISel active, we may be splitting blocks, so force creation
10978 // of virtual registers for all non-dead arguments.
10979 if (FastISel)
10980 return A->use_empty();
10981
10982 const BasicBlock &Entry = A->getParent()->front();
10983 for (const User *U : A->users())
10984 if (cast<Instruction>(U)->getParent() != &Entry || isa<SwitchInst>(U))
10985 return false; // Use not in entry block.
10986
10987 return true;
10988}
10989
10991 DenseMap<const Argument *,
10992 std::pair<const AllocaInst *, const StoreInst *>>;
10993
10994/// Scan the entry block of the function in FuncInfo for arguments that look
10995/// like copies into a local alloca. Record any copied arguments in
10996/// ArgCopyElisionCandidates.
10997static void
10999 FunctionLoweringInfo *FuncInfo,
11000 ArgCopyElisionMapTy &ArgCopyElisionCandidates) {
11001 // Record the state of every static alloca used in the entry block. Argument
11002 // allocas are all used in the entry block, so we need approximately as many
11003 // entries as we have arguments.
11004 enum StaticAllocaInfo { Unknown, Clobbered, Elidable };
11006 unsigned NumArgs = FuncInfo->Fn->arg_size();
11007 StaticAllocas.reserve(NumArgs * 2);
11008
11009 auto GetInfoIfStaticAlloca = [&](const Value *V) -> StaticAllocaInfo * {
11010 if (!V)
11011 return nullptr;
11012 V = V->stripPointerCasts();
11013 const auto *AI = dyn_cast<AllocaInst>(V);
11014 if (!AI || !AI->isStaticAlloca() || !FuncInfo->StaticAllocaMap.count(AI))
11015 return nullptr;
11016 auto Iter = StaticAllocas.insert({AI, Unknown});
11017 return &Iter.first->second;
11018 };
11019
11020 // Look for stores of arguments to static allocas. Look through bitcasts and
11021 // GEPs to handle type coercions, as long as the alloca is fully initialized
11022 // by the store. Any non-store use of an alloca escapes it and any subsequent
11023 // unanalyzed store might write it.
11024 // FIXME: Handle structs initialized with multiple stores.
11025 for (const Instruction &I : FuncInfo->Fn->getEntryBlock()) {
11026 // Look for stores, and handle non-store uses conservatively.
11027 const auto *SI = dyn_cast<StoreInst>(&I);
11028 if (!SI) {
11029 // We will look through cast uses, so ignore them completely.
11030 if (I.isCast())
11031 continue;
11032 // Ignore debug info and pseudo op intrinsics, they don't escape or store
11033 // to allocas.
11034 if (I.isDebugOrPseudoInst())
11035 continue;
11036 // This is an unknown instruction. Assume it escapes or writes to all
11037 // static alloca operands.
11038 for (const Use &U : I.operands()) {
11039 if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(U))
11040 *Info = StaticAllocaInfo::Clobbered;
11041 }
11042 continue;
11043 }
11044
11045 // If the stored value is a static alloca, mark it as escaped.
11046 if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(SI->getValueOperand()))
11047 *Info = StaticAllocaInfo::Clobbered;
11048
11049 // Check if the destination is a static alloca.
11050 const Value *Dst = SI->getPointerOperand()->stripPointerCasts();
11051 StaticAllocaInfo *Info = GetInfoIfStaticAlloca(Dst);
11052 if (!Info)
11053 continue;
11054 const AllocaInst *AI = cast<AllocaInst>(Dst);
11055
11056 // Skip allocas that have been initialized or clobbered.
11057 if (*Info != StaticAllocaInfo::Unknown)
11058 continue;
11059
11060 // Check if the stored value is an argument, and that this store fully
11061 // initializes the alloca.
11062 // If the argument type has padding bits we can't directly forward a pointer
11063 // as the upper bits may contain garbage.
11064 // Don't elide copies from the same argument twice.
11065 const Value *Val = SI->getValueOperand()->stripPointerCasts();
11066 const auto *Arg = dyn_cast<Argument>(Val);
11067 if (!Arg || Arg->hasPassPointeeByValueCopyAttr() ||
11068 Arg->getType()->isEmptyTy() ||
11069 DL.getTypeStoreSize(Arg->getType()) !=
11070 DL.getTypeAllocSize(AI->getAllocatedType()) ||
11071 !DL.typeSizeEqualsStoreSize(Arg->getType()) ||
11072 ArgCopyElisionCandidates.count(Arg)) {
11073 *Info = StaticAllocaInfo::Clobbered;
11074 continue;
11075 }
11076
11077 LLVM_DEBUG(dbgs() << "Found argument copy elision candidate: " << *AI
11078 << '\n');
11079
11080 // Mark this alloca and store for argument copy elision.
11081 *Info = StaticAllocaInfo::Elidable;
11082 ArgCopyElisionCandidates.insert({Arg, {AI, SI}});
11083
11084 // Stop scanning if we've seen all arguments. This will happen early in -O0
11085 // builds, which is useful, because -O0 builds have large entry blocks and
11086 // many allocas.
11087 if (ArgCopyElisionCandidates.size() == NumArgs)
11088 break;
11089 }
11090}
11091
11092/// Try to elide argument copies from memory into a local alloca. Succeeds if
11093/// ArgVal is a load from a suitable fixed stack object.
11096 DenseMap<int, int> &ArgCopyElisionFrameIndexMap,
11097 SmallPtrSetImpl<const Instruction *> &ElidedArgCopyInstrs,
11098 ArgCopyElisionMapTy &ArgCopyElisionCandidates, const Argument &Arg,
11099 ArrayRef<SDValue> ArgVals, bool &ArgHasUses) {
11100 // Check if this is a load from a fixed stack object.
11101 auto *LNode = dyn_cast<LoadSDNode>(ArgVals[0]);
11102 if (!LNode)
11103 return;
11104 auto *FINode = dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode());
11105 if (!FINode)
11106 return;
11107
11108 // Check that the fixed stack object is the right size and alignment.
11109 // Look at the alignment that the user wrote on the alloca instead of looking
11110 // at the stack object.
11111 auto ArgCopyIter = ArgCopyElisionCandidates.find(&Arg);
11112 assert(ArgCopyIter != ArgCopyElisionCandidates.end());
11113 const AllocaInst *AI = ArgCopyIter->second.first;
11114 int FixedIndex = FINode->getIndex();
11115 int &AllocaIndex = FuncInfo.StaticAllocaMap[AI];
11116 int OldIndex = AllocaIndex;
11117 MachineFrameInfo &MFI = FuncInfo.MF->getFrameInfo();
11118 if (MFI.getObjectSize(FixedIndex) != MFI.getObjectSize(OldIndex)) {
11119 LLVM_DEBUG(
11120 dbgs() << " argument copy elision failed due to bad fixed stack "
11121 "object size\n");
11122 return;
11123 }
11124 Align RequiredAlignment = AI->getAlign();
11125 if (MFI.getObjectAlign(FixedIndex) < RequiredAlignment) {
11126 LLVM_DEBUG(dbgs() << " argument copy elision failed: alignment of alloca "
11127 "greater than stack argument alignment ("
11128 << DebugStr(RequiredAlignment) << " vs "
11129 << DebugStr(MFI.getObjectAlign(FixedIndex)) << ")\n");
11130 return;
11131 }
11132
11133 // Perform the elision. Delete the old stack object and replace its only use
11134 // in the variable info map. Mark the stack object as mutable and aliased.
11135 LLVM_DEBUG({
11136 dbgs() << "Eliding argument copy from " << Arg << " to " << *AI << '\n'
11137 << " Replacing frame index " << OldIndex << " with " << FixedIndex
11138 << '\n';
11139 });
11140 MFI.RemoveStackObject(OldIndex);
11141 MFI.setIsImmutableObjectIndex(FixedIndex, false);
11142 MFI.setIsAliasedObjectIndex(FixedIndex, true);
11143 AllocaIndex = FixedIndex;
11144 ArgCopyElisionFrameIndexMap.insert({OldIndex, FixedIndex});
11145 for (SDValue ArgVal : ArgVals)
11146 Chains.push_back(ArgVal.getValue(1));
11147
11148 // Avoid emitting code for the store implementing the copy.
11149 const StoreInst *SI = ArgCopyIter->second.second;
11150 ElidedArgCopyInstrs.insert(SI);
11151
11152 // Check for uses of the argument again so that we can avoid exporting ArgVal
11153 // if it is't used by anything other than the store.
11154 for (const Value *U : Arg.users()) {
11155 if (U != SI) {
11156 ArgHasUses = true;
11157 break;
11158 }
11159 }
11160}
11161
11162void SelectionDAGISel::LowerArguments(const Function &F) {
11163 SelectionDAG &DAG = SDB->DAG;
11164 SDLoc dl = SDB->getCurSDLoc();
11165 const DataLayout &DL = DAG.getDataLayout();
11167
11168 // In Naked functions we aren't going to save any registers.
11169 if (F.hasFnAttribute(Attribute::Naked))
11170 return;
11171
11172 if (!FuncInfo->CanLowerReturn) {
11173 // Put in an sret pointer parameter before all the other parameters.
11174 SmallVector<EVT, 1> ValueVTs;
11176 PointerType::get(F.getContext(),
11178 ValueVTs);
11179
11180 // NOTE: Assuming that a pointer will never break down to more than one VT
11181 // or one register.
11183 Flags.setSRet();
11184 MVT RegisterVT = TLI->getRegisterType(*DAG.getContext(), ValueVTs[0]);
11185 ISD::InputArg RetArg(Flags, RegisterVT, ValueVTs[0], true,
11187 Ins.push_back(RetArg);
11188 }
11189
11190 // Look for stores of arguments to static allocas. Mark such arguments with a
11191 // flag to ask the target to give us the memory location of that argument if
11192 // available.
11193 ArgCopyElisionMapTy ArgCopyElisionCandidates;
11195 ArgCopyElisionCandidates);
11196
11197 // Set up the incoming argument description vector.
11198 for (const Argument &Arg : F.args()) {
11199 unsigned ArgNo = Arg.getArgNo();
11200 SmallVector<EVT, 4> ValueVTs;
11201 ComputeValueVTs(*TLI, DAG.getDataLayout(), Arg.getType(), ValueVTs);
11202 bool isArgValueUsed = !Arg.use_empty();
11203 unsigned PartBase = 0;
11204 Type *FinalType = Arg.getType();
11205 if (Arg.hasAttribute(Attribute::ByVal))
11206 FinalType = Arg.getParamByValType();
11207 bool NeedsRegBlock = TLI->functionArgumentNeedsConsecutiveRegisters(
11208 FinalType, F.getCallingConv(), F.isVarArg(), DL);
11209 for (unsigned Value = 0, NumValues = ValueVTs.size();
11210 Value != NumValues; ++Value) {
11211 EVT VT = ValueVTs[Value];
11212 Type *ArgTy = VT.getTypeForEVT(*DAG.getContext());
11214
11215
11216 if (Arg.getType()->isPointerTy()) {
11217 Flags.setPointer();
11218 Flags.setPointerAddrSpace(
11219 cast<PointerType>(Arg.getType())->getAddressSpace());
11220 }
11221 if (Arg.hasAttribute(Attribute::ZExt))
11222 Flags.setZExt();
11223 if (Arg.hasAttribute(Attribute::SExt))
11224 Flags.setSExt();
11225 if (Arg.hasAttribute(Attribute::InReg)) {
11226 // If we are using vectorcall calling convention, a structure that is
11227 // passed InReg - is surely an HVA
11228 if (F.getCallingConv() == CallingConv::X86_VectorCall &&
11229 isa<StructType>(Arg.getType())) {
11230 // The first value of a structure is marked
11231 if (0 == Value)
11232 Flags.setHvaStart();
11233 Flags.setHva();
11234 }
11235 // Set InReg Flag
11236 Flags.setInReg();
11237 }
11238 if (Arg.hasAttribute(Attribute::StructRet))
11239 Flags.setSRet();
11240 if (Arg.hasAttribute(Attribute::SwiftSelf))
11241 Flags.setSwiftSelf();
11242 if (Arg.hasAttribute(Attribute::SwiftAsync))
11243 Flags.setSwiftAsync();
11244 if (Arg.hasAttribute(Attribute::SwiftError))
11245 Flags.setSwiftError();
11246 if (Arg.hasAttribute(Attribute::ByVal))
11247 Flags.setByVal();
11248 if (Arg.hasAttribute(Attribute::ByRef))
11249 Flags.setByRef();
11250 if (Arg.hasAttribute(Attribute::InAlloca)) {
11251 Flags.setInAlloca();
11252 // Set the byval flag for CCAssignFn callbacks that don't know about
11253 // inalloca. This way we can know how many bytes we should've allocated
11254 // and how many bytes a callee cleanup function will pop. If we port
11255 // inalloca to more targets, we'll have to add custom inalloca handling
11256 // in the various CC lowering callbacks.
11257 Flags.setByVal();
11258 }
11259 if (Arg.hasAttribute(Attribute::Preallocated)) {
11260 Flags.setPreallocated();
11261 // Set the byval flag for CCAssignFn callbacks that don't know about
11262 // preallocated. This way we can know how many bytes we should've
11263 // allocated and how many bytes a callee cleanup function will pop. If
11264 // we port preallocated to more targets, we'll have to add custom
11265 // preallocated handling in the various CC lowering callbacks.
11266 Flags.setByVal();
11267 }
11268
11269 // Certain targets (such as MIPS), may have a different ABI alignment
11270 // for a type depending on the context. Give the target a chance to
11271 // specify the alignment it wants.
11272 const Align OriginalAlignment(
11274 Flags.setOrigAlign(OriginalAlignment);
11275
11276 Align MemAlign;
11277 Type *ArgMemTy = nullptr;
11278 if (Flags.isByVal() || Flags.isInAlloca() || Flags.isPreallocated() ||
11279 Flags.isByRef()) {
11280 if (!ArgMemTy)
11281 ArgMemTy = Arg.getPointeeInMemoryValueType();
11282
11283 uint64_t MemSize = DL.getTypeAllocSize(ArgMemTy);
11284
11285 // For in-memory arguments, size and alignment should be passed from FE.
11286 // BE will guess if this info is not there but there are cases it cannot
11287 // get right.
11288 if (auto ParamAlign = Arg.getParamStackAlign())
11289 MemAlign = *ParamAlign;
11290 else if ((ParamAlign = Arg.getParamAlign()))
11291 MemAlign = *ParamAlign;
11292 else
11293 MemAlign = Align(TLI->getByValTypeAlignment(ArgMemTy, DL));
11294 if (Flags.isByRef())
11295 Flags.setByRefSize(MemSize);
11296 else
11297 Flags.setByValSize(MemSize);
11298 } else if (auto ParamAlign = Arg.getParamStackAlign()) {
11299 MemAlign = *ParamAlign;
11300 } else {
11301 MemAlign = OriginalAlignment;
11302 }
11303 Flags.setMemAlign(MemAlign);
11304
11305 if (Arg.hasAttribute(Attribute::Nest))
11306 Flags.setNest();
11307 if (NeedsRegBlock)
11308 Flags.setInConsecutiveRegs();
11309 if (ArgCopyElisionCandidates.count(&Arg))
11310 Flags.setCopyElisionCandidate();
11311 if (Arg.hasAttribute(Attribute::Returned))
11312 Flags.setReturned();
11313
11315 *CurDAG->getContext(), F.getCallingConv(), VT);
11316 unsigned NumRegs = TLI->getNumRegistersForCallingConv(
11317 *CurDAG->getContext(), F.getCallingConv(), VT);
11318 for (unsigned i = 0; i != NumRegs; ++i) {
11319 // For scalable vectors, use the minimum size; individual targets
11320 // are responsible for handling scalable vector arguments and
11321 // return values.
11322 ISD::InputArg MyFlags(
11323 Flags, RegisterVT, VT, isArgValueUsed, ArgNo,
11324 PartBase + i * RegisterVT.getStoreSize().getKnownMinValue());
11325 if (NumRegs > 1 && i == 0)
11326 MyFlags.Flags.setSplit();
11327 // if it isn't first piece, alignment must be 1
11328 else if (i > 0) {
11329 MyFlags.Flags.setOrigAlign(Align(1));
11330 if (i == NumRegs - 1)
11331 MyFlags.Flags.setSplitEnd();
11332 }
11333 Ins.push_back(MyFlags);
11334 }
11335 if (NeedsRegBlock && Value == NumValues - 1)
11336 Ins[Ins.size() - 1].Flags.setInConsecutiveRegsLast();
11337 PartBase += VT.getStoreSize().getKnownMinValue();
11338 }
11339 }
11340
11341 // Call the target to set up the argument values.
11343 SDValue NewRoot = TLI->LowerFormalArguments(
11344 DAG.getRoot(), F.getCallingConv(), F.isVarArg(), Ins, dl, DAG, InVals);
11345
11346 // Verify that the target's LowerFormalArguments behaved as expected.
11347 assert(NewRoot.getNode() && NewRoot.getValueType() == MVT::Other &&
11348 "LowerFormalArguments didn't return a valid chain!");
11349 assert(InVals.size() == Ins.size() &&
11350 "LowerFormalArguments didn't emit the correct number of values!");
11351 LLVM_DEBUG({
11352 for (unsigned i = 0, e = Ins.size(); i != e; ++i) {
11353 assert(InVals[i].getNode() &&
11354 "LowerFormalArguments emitted a null value!");
11355 assert(EVT(Ins[i].VT) == InVals[i].getValueType() &&
11356 "LowerFormalArguments emitted a value with the wrong type!");
11357 }
11358 });
11359
11360 // Update the DAG with the new chain value resulting from argument lowering.
11361 DAG.setRoot(NewRoot);
11362
11363 // Set up the argument values.
11364 unsigned i = 0;
11365 if (!FuncInfo->CanLowerReturn) {
11366 // Create a virtual register for the sret pointer, and put in a copy
11367 // from the sret argument into it.
11368 SmallVector<EVT, 1> ValueVTs;
11370 PointerType::get(F.getContext(),
11372 ValueVTs);
11373 MVT VT = ValueVTs[0].getSimpleVT();
11374 MVT RegVT = TLI->getRegisterType(*CurDAG->getContext(), VT);
11375 std::optional<ISD::NodeType> AssertOp;
11376 SDValue ArgValue =
11377 getCopyFromParts(DAG, dl, &InVals[0], 1, RegVT, VT, nullptr, NewRoot,
11378 F.getCallingConv(), AssertOp);
11379
11380 MachineFunction& MF = SDB->DAG.getMachineFunction();
11382 Register SRetReg =
11383 RegInfo.createVirtualRegister(TLI->getRegClassFor(RegVT));
11384 FuncInfo->DemoteRegister = SRetReg;
11385 NewRoot =
11386 SDB->DAG.getCopyToReg(NewRoot, SDB->getCurSDLoc(), SRetReg, ArgValue);
11387 DAG.setRoot(NewRoot);
11388
11389 // i indexes lowered arguments. Bump it past the hidden sret argument.
11390 ++i;
11391 }
11392
11394 DenseMap<int, int> ArgCopyElisionFrameIndexMap;
11395 for (const Argument &Arg : F.args()) {
11396 SmallVector<SDValue, 4> ArgValues;
11397 SmallVector<EVT, 4> ValueVTs;
11398 ComputeValueVTs(*TLI, DAG.getDataLayout(), Arg.getType(), ValueVTs);
11399 unsigned NumValues = ValueVTs.size();
11400 if (NumValues == 0)
11401 continue;
11402
11403 bool ArgHasUses = !Arg.use_empty();
11404
11405 // Elide the copying store if the target loaded this argument from a
11406 // suitable fixed stack object.
11407 if (Ins[i].Flags.isCopyElisionCandidate()) {
11408 unsigned NumParts = 0;
11409 for (EVT VT : ValueVTs)
11411 F.getCallingConv(), VT);
11412
11413 tryToElideArgumentCopy(*FuncInfo, Chains, ArgCopyElisionFrameIndexMap,
11414 ElidedArgCopyInstrs, ArgCopyElisionCandidates, Arg,
11415 ArrayRef(&InVals[i], NumParts), ArgHasUses);
11416 }
11417
11418 // If this argument is unused then remember its value. It is used to generate
11419 // debugging information.
11420 bool isSwiftErrorArg =
11422 Arg.hasAttribute(Attribute::SwiftError);
11423 if (!ArgHasUses && !isSwiftErrorArg) {
11424 SDB->setUnusedArgValue(&Arg, InVals[i]);
11425
11426 // Also remember any frame index for use in FastISel.
11427 if (FrameIndexSDNode *FI =
11428 dyn_cast<FrameIndexSDNode>(InVals[i].getNode()))
11429 FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
11430 }
11431
11432 for (unsigned Val = 0; Val != NumValues; ++Val) {
11433 EVT VT = ValueVTs[Val];
11435 F.getCallingConv(), VT);
11436 unsigned NumParts = TLI->getNumRegistersForCallingConv(
11437 *CurDAG->getContext(), F.getCallingConv(), VT);
11438
11439 // Even an apparent 'unused' swifterror argument needs to be returned. So
11440 // we do generate a copy for it that can be used on return from the
11441 // function.
11442 if (ArgHasUses || isSwiftErrorArg) {
11443 std::optional<ISD::NodeType> AssertOp;
11444 if (Arg.hasAttribute(Attribute::SExt))
11445 AssertOp = ISD::AssertSext;
11446 else if (Arg.hasAttribute(Attribute::ZExt))
11447 AssertOp = ISD::AssertZext;
11448
11449 ArgValues.push_back(getCopyFromParts(DAG, dl, &InVals[i], NumParts,
11450 PartVT, VT, nullptr, NewRoot,
11451 F.getCallingConv(), AssertOp));
11452 }
11453
11454 i += NumParts;
11455 }
11456
11457 // We don't need to do anything else for unused arguments.
11458 if (ArgValues.empty())
11459 continue;
11460
11461 // Note down frame index.
11462 if (FrameIndexSDNode *FI =
11463 dyn_cast<FrameIndexSDNode>(ArgValues[0].getNode()))
11464 FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
11465
11466 SDValue Res = DAG.getMergeValues(ArrayRef(ArgValues.data(), NumValues),
11467 SDB->getCurSDLoc());
11468
11469 SDB->setValue(&Arg, Res);
11471 // We want to associate the argument with the frame index, among
11472 // involved operands, that correspond to the lowest address. The
11473 // getCopyFromParts function, called earlier, is swapping the order of
11474 // the operands to BUILD_PAIR depending on endianness. The result of
11475 // that swapping is that the least significant bits of the argument will
11476 // be in the first operand of the BUILD_PAIR node, and the most
11477 // significant bits will be in the second operand.
11478 unsigned LowAddressOp = DAG.getDataLayout().isBigEndian() ? 1 : 0;
11479 if (LoadSDNode *LNode =
11480 dyn_cast<LoadSDNode>(Res.getOperand(LowAddressOp).getNode()))
11481 if (FrameIndexSDNode *FI =
11482 dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
11483 FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
11484 }
11485
11486 // Analyses past this point are naive and don't expect an assertion.
11487 if (Res.getOpcode() == ISD::AssertZext)
11488 Res = Res.getOperand(0);
11489
11490 // Update the SwiftErrorVRegDefMap.
11491 if (Res.getOpcode() == ISD::CopyFromReg && isSwiftErrorArg) {
11492 unsigned Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg();
11495 Reg);
11496 }
11497
11498 // If this argument is live outside of the entry block, insert a copy from
11499 // wherever we got it to the vreg that other BB's will reference it as.
11500 if (Res.getOpcode() == ISD::CopyFromReg) {
11501 // If we can, though, try to skip creating an unnecessary vreg.
11502 // FIXME: This isn't very clean... it would be nice to make this more
11503 // general.
11504 unsigned Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg();
11505 if (Register::isVirtualRegister(Reg)) {
11506 FuncInfo->ValueMap[&Arg] = Reg;
11507 continue;
11508 }
11509 }
11511 FuncInfo->InitializeRegForValue(&Arg);
11512 SDB->CopyToExportRegsIfNeeded(&Arg);
11513 }
11514 }
11515
11516 if (!Chains.empty()) {
11517 Chains.push_back(NewRoot);
11518 NewRoot = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
11519 }
11520
11521 DAG.setRoot(NewRoot);
11522
11523 assert(i == InVals.size() && "Argument register count mismatch!");
11524
11525 // If any argument copy elisions occurred and we have debug info, update the
11526 // stale frame indices used in the dbg.declare variable info table.
11527 if (!ArgCopyElisionFrameIndexMap.empty()) {
11530 auto I = ArgCopyElisionFrameIndexMap.find(VI.getStackSlot());
11531 if (I != ArgCopyElisionFrameIndexMap.end())
11532 VI.updateStackSlot(I->second);
11533 }
11534 }
11535
11536 // Finally, if the target has anything special to do, allow it to do so.
11538}
11539
11540/// Handle PHI nodes in successor blocks. Emit code into the SelectionDAG to
11541/// ensure constants are generated when needed. Remember the virtual registers
11542/// that need to be added to the Machine PHI nodes as input. We cannot just
11543/// directly add them, because expansion might result in multiple MBB's for one
11544/// BB. As such, the start of the BB might correspond to a different MBB than
11545/// the end.
11546void
11547SelectionDAGBuilder::HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) {
11549
11551
11552 // Check PHI nodes in successors that expect a value to be available from this
11553 // block.
11554 for (const BasicBlock *SuccBB : successors(LLVMBB->getTerminator())) {
11555 if (!isa<PHINode>(SuccBB->begin())) continue;
11556 MachineBasicBlock *SuccMBB = FuncInfo.MBBMap[SuccBB];
11557
11558 // If this terminator has multiple identical successors (common for
11559 // switches), only handle each succ once.
11560 if (!SuccsHandled.insert(SuccMBB).second)
11561 continue;
11562
11564
11565 // At this point we know that there is a 1-1 correspondence between LLVM PHI
11566 // nodes and Machine PHI nodes, but the incoming operands have not been
11567 // emitted yet.
11568 for (const PHINode &PN : SuccBB->phis()) {
11569 // Ignore dead phi's.
11570 if (PN.use_empty())
11571 continue;
11572
11573 // Skip empty types
11574 if (PN.getType()->isEmptyTy())
11575 continue;
11576
11577 unsigned Reg;
11578 const Value *PHIOp = PN.getIncomingValueForBlock(LLVMBB);
11579
11580 if (const auto *C = dyn_cast<Constant>(PHIOp)) {
11581 unsigned &RegOut = ConstantsOut[C];
11582 if (RegOut == 0) {
11583 RegOut = FuncInfo.CreateRegs(C);
11584 // We need to zero/sign extend ConstantInt phi operands to match
11585 // assumptions in FunctionLoweringInfo::ComputePHILiveOutRegInfo.
11586 ISD::NodeType ExtendType = ISD::ANY_EXTEND;
11587 if (auto *CI = dyn_cast<ConstantInt>(C))
11588 ExtendType = TLI.signExtendConstant(CI) ? ISD::SIGN_EXTEND
11590 CopyValueToVirtualRegister(C, RegOut, ExtendType);
11591 }
11592 Reg = RegOut;
11593 } else {
11595 FuncInfo.ValueMap.find(PHIOp);
11596 if (I != FuncInfo.ValueMap.end())
11597 Reg = I->second;
11598 else {
11599 assert(isa<AllocaInst>(PHIOp) &&
11600 FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)) &&
11601 "Didn't codegen value into a register!??");
11602 Reg = FuncInfo.CreateRegs(PHIOp);
11603 CopyValueToVirtualRegister(PHIOp, Reg);
11604 }
11605 }
11606
11607 // Remember that this register needs to added to the machine PHI node as
11608 // the input for this MBB.
11609 SmallVector<EVT, 4> ValueVTs;
11610 ComputeValueVTs(TLI, DAG.getDataLayout(), PN.getType(), ValueVTs);
11611 for (EVT VT : ValueVTs) {
11612 const unsigned NumRegisters = TLI.getNumRegisters(*DAG.getContext(), VT);
11613 for (unsigned i = 0; i != NumRegisters; ++i)
11614 FuncInfo.PHINodesToUpdate.push_back(
11615 std::make_pair(&*MBBI++, Reg + i));
11616 Reg += NumRegisters;
11617 }
11618 }
11619 }
11620
11621 ConstantsOut.clear();
11622}
11623
11624MachineBasicBlock *SelectionDAGBuilder::NextBlock(MachineBasicBlock *MBB) {
11626 if (++I == FuncInfo.MF->end())
11627 return nullptr;
11628 return &*I;
11629}
11630
11631/// During lowering new call nodes can be created (such as memset, etc.).
11632/// Those will become new roots of the current DAG, but complications arise
11633/// when they are tail calls. In such cases, the call lowering will update
11634/// the root, but the builder still needs to know that a tail call has been
11635/// lowered in order to avoid generating an additional return.
11636void SelectionDAGBuilder::updateDAGForMaybeTailCall(SDValue MaybeTC) {
11637 // If the node is null, we do have a tail call.
11638 if (MaybeTC.getNode() != nullptr)
11639 DAG.setRoot(MaybeTC);
11640 else
11641 HasTailCall = true;
11642}
11643
11644void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond,
11645 MachineBasicBlock *SwitchMBB,
11646 MachineBasicBlock *DefaultMBB) {
11647 MachineFunction *CurMF = FuncInfo.MF;
11648 MachineBasicBlock *NextMBB = nullptr;
11650 if (++BBI != FuncInfo.MF->end())
11651 NextMBB = &*BBI;
11652
11653 unsigned Size = W.LastCluster - W.FirstCluster + 1;
11654
11656
11657 if (Size == 2 && W.MBB == SwitchMBB) {
11658 // If any two of the cases has the same destination, and if one value
11659 // is the same as the other, but has one bit unset that the other has set,
11660 // use bit manipulation to do two compares at once. For example:
11661 // "if (X == 6 || X == 4)" -> "if ((X|2) == 6)"
11662 // TODO: This could be extended to merge any 2 cases in switches with 3
11663 // cases.
11664 // TODO: Handle cases where W.CaseBB != SwitchBB.
11665 CaseCluster &Small = *W.FirstCluster;
11666 CaseCluster &Big = *W.LastCluster;
11667
11668 if (Small.Low == Small.High && Big.Low == Big.High &&
11669 Small.MBB == Big.MBB) {
11670 const APInt &SmallValue = Small.Low->getValue();
11671 const APInt &BigValue = Big.Low->getValue();
11672
11673 // Check that there is only one bit different.
11674 APInt CommonBit = BigValue ^ SmallValue;
11675 if (CommonBit.isPowerOf2()) {
11676 SDValue CondLHS = getValue(Cond);
11677 EVT VT = CondLHS.getValueType();
11678 SDLoc DL = getCurSDLoc();
11679
11680 SDValue Or = DAG.getNode(ISD::OR, DL, VT, CondLHS,
11681 DAG.getConstant(CommonBit, DL, VT));
11683 DL, MVT::i1, Or, DAG.getConstant(BigValue | SmallValue, DL, VT),
11684 ISD::SETEQ);
11685
11686 // Update successor info.
11687 // Both Small and Big will jump to Small.BB, so we sum up the
11688 // probabilities.
11689 addSuccessorWithProb(SwitchMBB, Small.MBB, Small.Prob + Big.Prob);
11690 if (BPI)
11691 addSuccessorWithProb(
11692 SwitchMBB, DefaultMBB,
11693 // The default destination is the first successor in IR.
11694 BPI->getEdgeProbability(SwitchMBB->getBasicBlock(), (unsigned)0));
11695 else
11696 addSuccessorWithProb(SwitchMBB, DefaultMBB);
11697
11698 // Insert the true branch.
11699 SDValue BrCond =
11700 DAG.getNode(ISD::BRCOND, DL, MVT::Other, getControlRoot(), Cond,
11701 DAG.getBasicBlock(Small.MBB));
11702 // Insert the false branch.
11703 BrCond = DAG.getNode(ISD::BR, DL, MVT::Other, BrCond,
11704 DAG.getBasicBlock(DefaultMBB));
11705
11706 DAG.setRoot(BrCond);
11707 return;
11708 }
11709 }
11710 }
11711
11712 if (TM.getOptLevel() != CodeGenOptLevel::None) {
11713 // Here, we order cases by probability so the most likely case will be
11714 // checked first. However, two clusters can have the same probability in
11715 // which case their relative ordering is non-deterministic. So we use Low
11716 // as a tie-breaker as clusters are guaranteed to never overlap.
11717 llvm::sort(W.FirstCluster, W.LastCluster + 1,
11718 [](const CaseCluster &a, const CaseCluster &b) {
11719 return a.Prob != b.Prob ?
11720 a.Prob > b.Prob :
11721 a.Low->getValue().slt(b.Low->getValue());
11722 });
11723
11724 // Rearrange the case blocks so that the last one falls through if possible
11725 // without changing the order of probabilities.
11726 for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster; ) {
11727 --I;
11728 if (I->Prob > W.LastCluster->Prob)
11729 break;
11730 if (I->Kind == CC_Range && I->MBB == NextMBB) {
11731 std::swap(*I, *W.LastCluster);
11732 break;
11733 }
11734 }
11735 }
11736
11737 // Compute total probability.
11738 BranchProbability DefaultProb = W.DefaultProb;
11739 BranchProbability UnhandledProbs = DefaultProb;
11740 for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I)
11741 UnhandledProbs += I->Prob;
11742
11743 MachineBasicBlock *CurMBB = W.MBB;
11744 for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) {
11745 bool FallthroughUnreachable = false;
11746 MachineBasicBlock *Fallthrough;
11747 if (I == W.LastCluster) {
11748 // For the last cluster, fall through to the default destination.
11749 Fallthrough = DefaultMBB;
11750 FallthroughUnreachable = isa<UnreachableInst>(
11751 DefaultMBB->getBasicBlock()->getFirstNonPHIOrDbg());
11752 } else {
11753 Fallthrough = CurMF->CreateMachineBasicBlock(CurMBB->getBasicBlock());
11754 CurMF->insert(BBI, Fallthrough);
11755 // Put Cond in a virtual register to make it available from the new blocks.
11757 }
11758 UnhandledProbs -= I->Prob;
11759
11760 switch (I->Kind) {
11761 case CC_JumpTable: {
11762 // FIXME: Optimize away range check based on pivot comparisons.
11763 JumpTableHeader *JTH = &SL->JTCases[I->JTCasesIndex].first;
11764 SwitchCG::JumpTable *JT = &SL->JTCases[I->JTCasesIndex].second;
11765
11766 // The jump block hasn't been inserted yet; insert it here.
11767 MachineBasicBlock *JumpMBB = JT->MBB;
11768 CurMF->insert(BBI, JumpMBB);
11769
11770 auto JumpProb = I->Prob;
11771 auto FallthroughProb = UnhandledProbs;
11772
11773 // If the default statement is a target of the jump table, we evenly
11774 // distribute the default probability to successors of CurMBB. Also
11775 // update the probability on the edge from JumpMBB to Fallthrough.
11776 for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(),
11777 SE = JumpMBB->succ_end();
11778 SI != SE; ++SI) {
11779 if (*SI == DefaultMBB) {
11780 JumpProb += DefaultProb / 2;
11781 FallthroughProb -= DefaultProb / 2;
11782 JumpMBB->setSuccProbability(SI, DefaultProb / 2);
11783 JumpMBB->normalizeSuccProbs();
11784 break;
11785 }
11786 }
11787
11788 // If the default clause is unreachable, propagate that knowledge into
11789 // JTH->FallthroughUnreachable which will use it to suppress the range
11790 // check.
11791 //
11792 // However, don't do this if we're doing branch target enforcement,
11793 // because a table branch _without_ a range check can be a tempting JOP
11794 // gadget - out-of-bounds inputs that are impossible in correct
11795 // execution become possible again if an attacker can influence the
11796 // control flow. So if an attacker doesn't already have a BTI bypass
11797 // available, we don't want them to be able to get one out of this
11798 // table branch.
11799 if (FallthroughUnreachable) {
11800 Function &CurFunc = CurMF->getFunction();
11801 bool HasBranchTargetEnforcement = false;
11802 if (CurFunc.hasFnAttribute("branch-target-enforcement")) {
11803 HasBranchTargetEnforcement =
11804 CurFunc.getFnAttribute("branch-target-enforcement")
11805 .getValueAsBool();
11806 } else {
11807 HasBranchTargetEnforcement =
11808 CurMF->getMMI().getModule()->getModuleFlag(
11809 "branch-target-enforcement");
11810 }
11811 if (!HasBranchTargetEnforcement)
11812 JTH->FallthroughUnreachable = true;
11813 }
11814
11815 if (!JTH->FallthroughUnreachable)
11816 addSuccessorWithProb(CurMBB, Fallthrough, FallthroughProb);
11817 addSuccessorWithProb(CurMBB, JumpMBB, JumpProb);
11818 CurMBB->normalizeSuccProbs();
11819
11820 // The jump table header will be inserted in our current block, do the
11821 // range check, and fall through to our fallthrough block.
11822 JTH->HeaderBB = CurMBB;
11823 JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader.
11824
11825 // If we're in the right place, emit the jump table header right now.
11826 if (CurMBB == SwitchMBB) {
11827 visitJumpTableHeader(*JT, *JTH, SwitchMBB);
11828 JTH->Emitted = true;
11829 }
11830 break;
11831 }
11832 case CC_BitTests: {
11833 // FIXME: Optimize away range check based on pivot comparisons.
11834 BitTestBlock *BTB = &SL->BitTestCases[I->BTCasesIndex];
11835
11836 // The bit test blocks haven't been inserted yet; insert them here.
11837 for (BitTestCase &BTC : BTB->Cases)
11838 CurMF->insert(BBI, BTC.ThisBB);
11839
11840 // Fill in fields of the BitTestBlock.
11841 BTB->Parent = CurMBB;
11842 BTB->Default = Fallthrough;
11843
11844 BTB->DefaultProb = UnhandledProbs;
11845 // If the cases in bit test don't form a contiguous range, we evenly
11846 // distribute the probability on the edge to Fallthrough to two
11847 // successors of CurMBB.
11848 if (!BTB->ContiguousRange) {
11849 BTB->Prob += DefaultProb / 2;
11850 BTB->DefaultProb -= DefaultProb / 2;
11851 }
11852
11853 if (FallthroughUnreachable)
11854 BTB->FallthroughUnreachable = true;
11855
11856 // If we're in the right place, emit the bit test header right now.
11857 if (CurMBB == SwitchMBB) {
11858 visitBitTestHeader(*BTB, SwitchMBB);
11859 BTB->Emitted = true;
11860 }
11861 break;
11862 }
11863 case CC_Range: {
11864 const Value *RHS, *LHS, *MHS;
11866 if (I->Low == I->High) {
11867 // Check Cond == I->Low.
11868 CC = ISD::SETEQ;
11869 LHS = Cond;
11870 RHS=I->Low;
11871 MHS = nullptr;
11872 } else {
11873 // Check I->Low <= Cond <= I->High.
11874 CC = ISD::SETLE;
11875 LHS = I->Low;
11876 MHS = Cond;
11877 RHS = I->High;
11878 }
11879
11880 // If Fallthrough is unreachable, fold away the comparison.
11881 if (FallthroughUnreachable)
11882 CC = ISD::SETTRUE;
11883
11884 // The false probability is the sum of all unhandled cases.
11885 CaseBlock CB(CC, LHS, RHS, MHS, I->MBB, Fallthrough, CurMBB,
11886 getCurSDLoc(), I->Prob, UnhandledProbs);
11887
11888 if (CurMBB == SwitchMBB)
11889 visitSwitchCase(CB, SwitchMBB);
11890 else
11891 SL->SwitchCases.push_back(CB);
11892
11893 break;
11894 }
11895 }
11896 CurMBB = Fallthrough;
11897 }
11898}
11899
11900void SelectionDAGBuilder::splitWorkItem(SwitchWorkList &WorkList,
11901 const SwitchWorkListItem &W,
11902 Value *Cond,
11903 MachineBasicBlock *SwitchMBB) {
11904 assert(W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) &&
11905 "Clusters not sorted?");
11906 assert(W.LastCluster - W.FirstCluster + 1 >= 2 && "Too small to split!");
11907
11908 auto [LastLeft, FirstRight, LeftProb, RightProb] =
11909 SL->computeSplitWorkItemInfo(W);
11910
11911 // Use the first element on the right as pivot since we will make less-than
11912 // comparisons against it.
11913 CaseClusterIt PivotCluster = FirstRight;
11914 assert(PivotCluster > W.FirstCluster);
11915 assert(PivotCluster <= W.LastCluster);
11916
11917 CaseClusterIt FirstLeft = W.FirstCluster;
11918 CaseClusterIt LastRight = W.LastCluster;
11919
11920 const ConstantInt *Pivot = PivotCluster->Low;
11921
11922 // New blocks will be inserted immediately after the current one.
11924 ++BBI;
11925
11926 // We will branch to the LHS if Value < Pivot. If LHS is a single cluster,
11927 // we can branch to its destination directly if it's squeezed exactly in
11928 // between the known lower bound and Pivot - 1.
11929 MachineBasicBlock *LeftMBB;
11930 if (FirstLeft == LastLeft && FirstLeft->Kind == CC_Range &&
11931 FirstLeft->Low == W.GE &&
11932 (FirstLeft->High->getValue() + 1LL) == Pivot->getValue()) {
11933 LeftMBB = FirstLeft->MBB;
11934 } else {
11935 LeftMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
11936 FuncInfo.MF->insert(BBI, LeftMBB);
11937 WorkList.push_back(
11938 {LeftMBB, FirstLeft, LastLeft, W.GE, Pivot, W.DefaultProb / 2});
11939 // Put Cond in a virtual register to make it available from the new blocks.
11941 }
11942
11943 // Similarly, we will branch to the RHS if Value >= Pivot. If RHS is a
11944 // single cluster, RHS.Low == Pivot, and we can branch to its destination
11945 // directly if RHS.High equals the current upper bound.
11946 MachineBasicBlock *RightMBB;
11947 if (FirstRight == LastRight && FirstRight->Kind == CC_Range &&
11948 W.LT && (FirstRight->High->getValue() + 1ULL) == W.LT->getValue()) {
11949 RightMBB = FirstRight->MBB;
11950 } else {
11951 RightMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
11952 FuncInfo.MF->insert(BBI, RightMBB);
11953 WorkList.push_back(
11954 {RightMBB, FirstRight, LastRight, Pivot, W.LT, W.DefaultProb / 2});
11955 // Put Cond in a virtual register to make it available from the new blocks.
11957 }
11958
11959 // Create the CaseBlock record that will be used to lower the branch.
11960 CaseBlock CB(ISD::SETLT, Cond, Pivot, nullptr, LeftMBB, RightMBB, W.MBB,
11961 getCurSDLoc(), LeftProb, RightProb);
11962
11963 if (W.MBB == SwitchMBB)
11964 visitSwitchCase(CB, SwitchMBB);
11965 else
11966 SL->SwitchCases.push_back(CB);
11967}
11968
11969// Scale CaseProb after peeling a case with the probablity of PeeledCaseProb
11970// from the swith statement.
11972 BranchProbability PeeledCaseProb) {
11973 if (PeeledCaseProb == BranchProbability::getOne())
11975 BranchProbability SwitchProb = PeeledCaseProb.getCompl();
11976
11977 uint32_t Numerator = CaseProb.getNumerator();
11978 uint32_t Denominator = SwitchProb.scale(CaseProb.getDenominator());
11979 return BranchProbability(Numerator, std::max(Numerator, Denominator));
11980}
11981
11982// Try to peel the top probability case if it exceeds the threshold.
11983// Return current MachineBasicBlock for the switch statement if the peeling
11984// does not occur.
11985// If the peeling is performed, return the newly created MachineBasicBlock
11986// for the peeled switch statement. Also update Clusters to remove the peeled
11987// case. PeeledCaseProb is the BranchProbability for the peeled case.
11988MachineBasicBlock *SelectionDAGBuilder::peelDominantCaseCluster(
11989 const SwitchInst &SI, CaseClusterVector &Clusters,
11990 BranchProbability &PeeledCaseProb) {
11991 MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
11992 // Don't perform if there is only one cluster or optimizing for size.
11993 if (SwitchPeelThreshold > 100 || !FuncInfo.BPI || Clusters.size() < 2 ||
11995 SwitchMBB->getParent()->getFunction().hasMinSize())
11996 return SwitchMBB;
11997
11999 unsigned PeeledCaseIndex = 0;
12000 bool SwitchPeeled = false;
12001 for (unsigned Index = 0; Index < Clusters.size(); ++Index) {
12002 CaseCluster &CC = Clusters[Index];
12003 if (CC.Prob < TopCaseProb)
12004 continue;
12005 TopCaseProb = CC.Prob;
12006 PeeledCaseIndex = Index;
12007 SwitchPeeled = true;
12008 }
12009 if (!SwitchPeeled)
12010 return SwitchMBB;
12011
12012 LLVM_DEBUG(dbgs() << "Peeled one top case in switch stmt, prob: "
12013 << TopCaseProb << "\n");
12014
12015 // Record the MBB for the peeled switch statement.
12016 MachineFunction::iterator BBI(SwitchMBB);
12017 ++BBI;
12018 MachineBasicBlock *PeeledSwitchMBB =
12020 FuncInfo.MF->insert(BBI, PeeledSwitchMBB);
12021
12022 ExportFromCurrentBlock(SI.getCondition());
12023 auto PeeledCaseIt = Clusters.begin() + PeeledCaseIndex;
12024 SwitchWorkListItem W = {SwitchMBB, PeeledCaseIt, PeeledCaseIt,
12025 nullptr, nullptr, TopCaseProb.getCompl()};
12026 lowerWorkItem(W, SI.getCondition(), SwitchMBB, PeeledSwitchMBB);
12027
12028 Clusters.erase(PeeledCaseIt);
12029 for (CaseCluster &CC : Clusters) {
12030 LLVM_DEBUG(
12031 dbgs() << "Scale the probablity for one cluster, before scaling: "
12032 << CC.Prob << "\n");
12033 CC.Prob = scaleCaseProbality(CC.Prob, TopCaseProb);
12034 LLVM_DEBUG(dbgs() << "After scaling: " << CC.Prob << "\n");
12035 }
12036 PeeledCaseProb = TopCaseProb;
12037 return PeeledSwitchMBB;
12038}
12039
12040void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) {
12041 // Extract cases from the switch.
12043 CaseClusterVector Clusters;
12044 Clusters.reserve(SI.getNumCases());
12045 for (auto I : SI.cases()) {
12046 MachineBasicBlock *Succ = FuncInfo.MBBMap[I.getCaseSuccessor()];
12047 const ConstantInt *CaseVal = I.getCaseValue();
12048 BranchProbability Prob =
12049 BPI ? BPI->getEdgeProbability(SI.getParent(), I.getSuccessorIndex())
12050 : BranchProbability(1, SI.getNumCases() + 1);
12051 Clusters.push_back(CaseCluster::range(CaseVal, CaseVal, Succ, Prob));
12052 }
12053
12054 MachineBasicBlock *DefaultMBB = FuncInfo.MBBMap[SI.getDefaultDest()];
12055
12056 // Cluster adjacent cases with the same destination. We do this at all
12057 // optimization levels because it's cheap to do and will make codegen faster
12058 // if there are many clusters.
12059 sortAndRangeify(Clusters);
12060
12061 // The branch probablity of the peeled case.
12063 MachineBasicBlock *PeeledSwitchMBB =
12064 peelDominantCaseCluster(SI, Clusters, PeeledCaseProb);
12065
12066 // If there is only the default destination, jump there directly.
12067 MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
12068 if (Clusters.empty()) {
12069 assert(PeeledSwitchMBB == SwitchMBB);
12070 SwitchMBB->addSuccessor(DefaultMBB);
12071 if (DefaultMBB != NextBlock(SwitchMBB)) {
12072 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
12073 getControlRoot(), DAG.getBasicBlock(DefaultMBB)));
12074 }
12075 return;
12076 }
12077
12078 SL->findJumpTables(Clusters, &SI, getCurSDLoc(), DefaultMBB, DAG.getPSI(),
12079 DAG.getBFI());
12080 SL->findBitTestClusters(Clusters, &SI);
12081
12082 LLVM_DEBUG({
12083 dbgs() << "Case clusters: ";
12084 for (const CaseCluster &C : Clusters) {
12085 if (C.Kind == CC_JumpTable)
12086 dbgs() << "JT:";
12087 if (C.Kind == CC_BitTests)
12088 dbgs() << "BT:";
12089
12090 C.Low->getValue().print(dbgs(), true);
12091 if (C.Low != C.High) {
12092 dbgs() << '-';
12093 C.High->getValue().print(dbgs(), true);
12094 }
12095 dbgs() << ' ';
12096 }
12097 dbgs() << '\n';
12098 });
12099
12100 assert(!Clusters.empty());
12101 SwitchWorkList WorkList;
12102 CaseClusterIt First = Clusters.begin();
12103 CaseClusterIt Last = Clusters.end() - 1;
12104 auto DefaultProb = getEdgeProbability(PeeledSwitchMBB, DefaultMBB);
12105 // Scale the branchprobability for DefaultMBB if the peel occurs and
12106 // DefaultMBB is not replaced.
12107 if (PeeledCaseProb != BranchProbability::getZero() &&
12108 DefaultMBB == FuncInfo.MBBMap[SI.getDefaultDest()])
12109 DefaultProb = scaleCaseProbality(DefaultProb, PeeledCaseProb);
12110 WorkList.push_back(
12111 {PeeledSwitchMBB, First, Last, nullptr, nullptr, DefaultProb});
12112
12113 while (!WorkList.empty()) {
12114 SwitchWorkListItem W = WorkList.pop_back_val();
12115 unsigned NumClusters = W.LastCluster - W.FirstCluster + 1;
12116
12117 if (NumClusters > 3 && TM.getOptLevel() != CodeGenOptLevel::None &&
12118 !DefaultMBB->getParent()->getFunction().hasMinSize()) {
12119 // For optimized builds, lower large range as a balanced binary tree.
12120 splitWorkItem(WorkList, W, SI.getCondition(), SwitchMBB);
12121 continue;
12122 }
12123
12124 lowerWorkItem(W, SI.getCondition(), SwitchMBB, DefaultMBB);
12125 }
12126}
12127
12128void SelectionDAGBuilder::visitStepVector(const CallInst &I) {
12130 auto DL = getCurSDLoc();
12131 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
12132 setValue(&I, DAG.getStepVector(DL, ResultVT));
12133}
12134
12135void SelectionDAGBuilder::visitVectorReverse(const CallInst &I) {
12137 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
12138
12139 SDLoc DL = getCurSDLoc();
12140 SDValue V = getValue(I.getOperand(0));
12141 assert(VT == V.getValueType() && "Malformed vector.reverse!");
12142
12143 if (VT.isScalableVector()) {
12145 return;
12146 }
12147
12148 // Use VECTOR_SHUFFLE for the fixed-length vector
12149 // to maintain existing behavior.
12151 unsigned NumElts = VT.getVectorMinNumElements();
12152 for (unsigned i = 0; i != NumElts; ++i)
12153 Mask.push_back(NumElts - 1 - i);
12154
12155 setValue(&I, DAG.getVectorShuffle(VT, DL, V, DAG.getUNDEF(VT), Mask));
12156}
12157
12158void SelectionDAGBuilder::visitVectorDeinterleave(const CallInst &I) {
12159 auto DL = getCurSDLoc();
12160 SDValue InVec = getValue(I.getOperand(0));
12161 EVT OutVT =
12163
12164 unsigned OutNumElts = OutVT.getVectorMinNumElements();
12165
12166 // ISD Node needs the input vectors split into two equal parts
12167 SDValue Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT, InVec,
12169 SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT, InVec,
12170 DAG.getVectorIdxConstant(OutNumElts, DL));
12171
12172 // Use VECTOR_SHUFFLE for fixed-length vectors to benefit from existing
12173 // legalisation and combines.
12174 if (OutVT.isFixedLengthVector()) {
12175 SDValue Even = DAG.getVectorShuffle(OutVT, DL, Lo, Hi,
12176 createStrideMask(0, 2, OutNumElts));
12177 SDValue Odd = DAG.getVectorShuffle(OutVT, DL, Lo, Hi,
12178 createStrideMask(1, 2, OutNumElts));
12179 SDValue Res = DAG.getMergeValues({Even, Odd}, getCurSDLoc());
12180 setValue(&I, Res);
12181 return;
12182 }
12183
12185 DAG.getVTList(OutVT, OutVT), Lo, Hi);
12186 setValue(&I, Res);
12187}
12188
12189void SelectionDAGBuilder::visitVectorInterleave(const CallInst &I) {
12190 auto DL = getCurSDLoc();
12191 EVT InVT = getValue(I.getOperand(0)).getValueType();
12192 SDValue InVec0 = getValue(I.getOperand(0));
12193 SDValue InVec1 = getValue(I.getOperand(1));
12195 EVT OutVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
12196
12197 // Use VECTOR_SHUFFLE for fixed-length vectors to benefit from existing
12198 // legalisation and combines.
12199 if (OutVT.isFixedLengthVector()) {
12200 unsigned NumElts = InVT.getVectorMinNumElements();
12201 SDValue V = DAG.getNode(ISD::CONCAT_VECTORS, DL, OutVT, InVec0, InVec1);
12202 setValue(&I, DAG.getVectorShuffle(OutVT, DL, V, DAG.getUNDEF(OutVT),
12203 createInterleaveMask(NumElts, 2)));
12204 return;
12205 }
12206
12208 DAG.getVTList(InVT, InVT), InVec0, InVec1);
12209 Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, OutVT, Res.getValue(0),
12210 Res.getValue(1));
12211 setValue(&I, Res);
12212}
12213
12214void SelectionDAGBuilder::visitFreeze(const FreezeInst &I) {
12215 SmallVector<EVT, 4> ValueVTs;
12217 ValueVTs);
12218 unsigned NumValues = ValueVTs.size();
12219 if (NumValues == 0) return;
12220
12221 SmallVector<SDValue, 4> Values(NumValues);
12222 SDValue Op = getValue(I.getOperand(0));
12223
12224 for (unsigned i = 0; i != NumValues; ++i)
12225 Values[i] = DAG.getNode(ISD::FREEZE, getCurSDLoc(), ValueVTs[i],
12226 SDValue(Op.getNode(), Op.getResNo() + i));
12227
12229 DAG.getVTList(ValueVTs), Values));
12230}
12231
12232void SelectionDAGBuilder::visitVectorSplice(const CallInst &I) {
12234 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
12235
12236 SDLoc DL = getCurSDLoc();
12237 SDValue V1 = getValue(I.getOperand(0));
12238 SDValue V2 = getValue(I.getOperand(1));
12239 int64_t Imm = cast<ConstantInt>(I.getOperand(2))->getSExtValue();
12240
12241 // VECTOR_SHUFFLE doesn't support a scalable mask so use a dedicated node.
12242 if (VT.isScalableVector()) {
12243 setValue(&I, DAG.getNode(ISD::VECTOR_SPLICE, DL, VT, V1, V2,
12244 DAG.getVectorIdxConstant(Imm, DL)));
12245 return;
12246 }
12247
12248 unsigned NumElts = VT.getVectorNumElements();
12249
12250 uint64_t Idx = (NumElts + Imm) % NumElts;
12251
12252 // Use VECTOR_SHUFFLE to maintain original behaviour for fixed-length vectors.
12254 for (unsigned i = 0; i < NumElts; ++i)
12255 Mask.push_back(Idx + i);
12256 setValue(&I, DAG.getVectorShuffle(VT, DL, V1, V2, Mask));
12257}
12258
12259// Consider the following MIR after SelectionDAG, which produces output in
12260// phyregs in the first case or virtregs in the second case.
12261//
12262// INLINEASM_BR ..., implicit-def $ebx, ..., implicit-def $edx
12263// %5:gr32 = COPY $ebx
12264// %6:gr32 = COPY $edx
12265// %1:gr32 = COPY %6:gr32
12266// %0:gr32 = COPY %5:gr32
12267//
12268// INLINEASM_BR ..., def %5:gr32, ..., def %6:gr32
12269// %1:gr32 = COPY %6:gr32
12270// %0:gr32 = COPY %5:gr32
12271//
12272// Given %0, we'd like to return $ebx in the first case and %5 in the second.
12273// Given %1, we'd like to return $edx in the first case and %6 in the second.
12274//
12275// If a callbr has outputs, it will have a single mapping in FuncInfo.ValueMap
12276// to a single virtreg (such as %0). The remaining outputs monotonically
12277// increase in virtreg number from there. If a callbr has no outputs, then it
12278// should not have a corresponding callbr landingpad; in fact, the callbr
12279// landingpad would not even be able to refer to such a callbr.
12281 MachineInstr *MI = MRI.def_begin(Reg)->getParent();
12282 // There is definitely at least one copy.
12283 assert(MI->getOpcode() == TargetOpcode::COPY &&
12284 "start of copy chain MUST be COPY");
12285 Reg = MI->getOperand(1).getReg();
12286 MI = MRI.def_begin(Reg)->getParent();
12287 // There may be an optional second copy.
12288 if (MI->getOpcode() == TargetOpcode::COPY) {
12289 assert(Reg.isVirtual() && "expected COPY of virtual register");
12290 Reg = MI->getOperand(1).getReg();
12291 assert(Reg.isPhysical() && "expected COPY of physical register");
12292 MI = MRI.def_begin(Reg)->getParent();
12293 }
12294 // The start of the chain must be an INLINEASM_BR.
12295 assert(MI->getOpcode() == TargetOpcode::INLINEASM_BR &&
12296 "end of copy chain MUST be INLINEASM_BR");
12297 return Reg;
12298}
12299
12300// We must do this walk rather than the simpler
12301// setValue(&I, getCopyFromRegs(CBR, CBR->getType()));
12302// otherwise we will end up with copies of virtregs only valid along direct
12303// edges.
12304void SelectionDAGBuilder::visitCallBrLandingPad(const CallInst &I) {
12305 SmallVector<EVT, 8> ResultVTs;
12306 SmallVector<SDValue, 8> ResultValues;
12307 const auto *CBR =
12308 cast<CallBrInst>(I.getParent()->getUniquePredecessor()->getTerminator());
12309
12313
12314 unsigned InitialDef = FuncInfo.ValueMap[CBR];
12315 SDValue Chain = DAG.getRoot();
12316
12317 // Re-parse the asm constraints string.
12318 TargetLowering::AsmOperandInfoVector TargetConstraints =
12319 TLI.ParseConstraints(DAG.getDataLayout(), TRI, *CBR);
12320 for (auto &T : TargetConstraints) {
12321 SDISelAsmOperandInfo OpInfo(T);
12322 if (OpInfo.Type != InlineAsm::isOutput)
12323 continue;
12324
12325 // Pencil in OpInfo.ConstraintType and OpInfo.ConstraintVT based on the
12326 // individual constraint.
12327 TLI.ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG);
12328
12329 switch (OpInfo.ConstraintType) {
12332 // Fill in OpInfo.AssignedRegs.Regs.
12333 getRegistersForValue(DAG, getCurSDLoc(), OpInfo, OpInfo);
12334
12335 // getRegistersForValue may produce 1 to many registers based on whether
12336 // the OpInfo.ConstraintVT is legal on the target or not.
12337 for (size_t i = 0, e = OpInfo.AssignedRegs.Regs.size(); i != e; ++i) {
12338 Register OriginalDef = FollowCopyChain(MRI, InitialDef++);
12339 if (Register::isPhysicalRegister(OriginalDef))
12340 FuncInfo.MBB->addLiveIn(OriginalDef);
12341 // Update the assigned registers to use the original defs.
12342 OpInfo.AssignedRegs.Regs[i] = OriginalDef;
12343 }
12344
12345 SDValue V = OpInfo.AssignedRegs.getCopyFromRegs(
12346 DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, CBR);
12347 ResultValues.push_back(V);
12348 ResultVTs.push_back(OpInfo.ConstraintVT);
12349 break;
12350 }
12352 SDValue Flag;
12353 SDValue V = TLI.LowerAsmOutputForConstraint(Chain, Flag, getCurSDLoc(),
12354 OpInfo, DAG);
12355 ++InitialDef;
12356 ResultValues.push_back(V);
12357 ResultVTs.push_back(OpInfo.ConstraintVT);
12358 break;
12359 }
12360 default:
12361 break;
12362 }
12363 }
12365 DAG.getVTList(ResultVTs), ResultValues);
12366 setValue(&I, V);
12367}
unsigned const MachineRegisterInfo * MRI
static unsigned getIntrinsicID(const SDNode *N)
unsigned RegSize
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
Function Alias Analysis Results
Atomic ordering constants.
This file contains the simple types necessary to represent the attributes associated with functions a...
static const Function * getParent(const Value *V)
This file implements the BitVector class.
BlockVerifier::State From
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
Analysis containing CSE Info
Definition: CSEInfo.cpp:27
This file contains the declarations for the subclasses of Constant, which represent the different fla...
return RetTy
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
#define LLVM_DEBUG(X)
Definition: Debug.h:101
uint64_t Size
bool End
Definition: ELF_riscv.cpp:480
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static AttributeList getReturnAttrs(FastISel::CallLoweringInfo &CLI)
Returns an AttributeList representing the attributes applied to the return value of the given call.
Definition: FastISel.cpp:943
#define Check(C,...)
Hexagon Common GEP
const HexagonInstrInfo * TII
IRTranslator LLVM IR MI
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
static void getRegistersForValue(MachineFunction &MF, MachineIRBuilder &MIRBuilder, GISelAsmOperandInfo &OpInfo, GISelAsmOperandInfo &RefOpInfo)
Assign virtual/physical registers for the specified register operand.
This file defines an InstructionCost class that is used when calculating the cost of an instruction,...
#define RegName(no)
lazy value info
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
unsigned const TargetRegisterInfo * TRI
static const Function * getCalledFunction(const Value *V, bool &IsNoBuiltin)
This file provides utility analysis objects describing memory locations.
This file provides utility for Memory Model Relaxation Annotations (MMRAs).
This file contains the declarations for metadata subclasses.
#define T1
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
Module.h This file contains the declarations for the Module class.
uint64_t High
LLVMContext & Context
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
#define P(N)
const SmallVectorImpl< MachineOperand > MachineBasicBlock * TBB
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file contains some templates that are useful if you are working with the STL at all.
static bool hasOnlySelectUsers(const Value *Cond)
static SDValue getLoadStackGuard(SelectionDAG &DAG, const SDLoc &DL, SDValue &Chain)
Create a LOAD_STACK_GUARD node, and let it carry the target specific global variable if there exists ...
static void addStackMapLiveVars(const CallBase &Call, unsigned StartIdx, const SDLoc &DL, SmallVectorImpl< SDValue > &Ops, SelectionDAGBuilder &Builder)
Add a stack map intrinsic call's live variable operands to a stackmap or patchpoint target node's ope...
static const unsigned MaxParallelChains
static void getUnderlyingArgRegs(SmallVectorImpl< std::pair< unsigned, TypeSize > > &Regs, const SDValue &N)
static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
visitPow - Lower a pow intrinsic.
static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index, ISD::MemIndexType &IndexType, SDValue &Scale, SelectionDAGBuilder *SDB, const BasicBlock *CurBB, uint64_t ElemSize)
static const CallBase * FindPreallocatedCall(const Value *PreallocatedSetup)
Given a @llvm.call.preallocated.setup, return the corresponding preallocated call.
static cl::opt< unsigned > SwitchPeelThreshold("switch-peel-threshold", cl::Hidden, cl::init(66), cl::desc("Set the case probability threshold for peeling the case from a " "switch statement. A value greater than 100 will void this " "optimization"))
static cl::opt< bool > InsertAssertAlign("insert-assert-align", cl::init(true), cl::desc("Insert the experimental `assertalign` node."), cl::ReallyHidden)
static unsigned getISDForVPIntrinsic(const VPIntrinsic &VPIntrin)
static bool handleDanglingVariadicDebugInfo(SelectionDAG &DAG, DILocalVariable *Variable, DebugLoc DL, unsigned Order, SmallVectorImpl< Value * > &Values, DIExpression *Expression)
static unsigned findMatchingInlineAsmOperand(unsigned OperandNo, const std::vector< SDValue > &AsmNodeOperands)
static void patchMatchingInput(const SDISelAsmOperandInfo &OpInfo, SDISelAsmOperandInfo &MatchingOpInfo, SelectionDAG &DAG)
Make sure that the output operand OpInfo and its corresponding input operand MatchingOpInfo have comp...
static void findUnwindDestinations(FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB, BranchProbability Prob, SmallVectorImpl< std::pair< MachineBasicBlock *, BranchProbability > > &UnwindDests)
When an invoke or a cleanupret unwinds to the next EH pad, there are many places it could ultimately ...
static unsigned FixedPointIntrinsicToOpcode(unsigned Intrinsic)
static BranchProbability scaleCaseProbality(BranchProbability CaseProb, BranchProbability PeeledCaseProb)
static SDValue expandExp2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandExp2 - Lower an exp2 intrinsic.
static SDValue expandDivFix(unsigned Opcode, const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue Scale, SelectionDAG &DAG, const TargetLowering &TLI)
static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt, const SDLoc &dl)
getF32Constant - Get 32-bit floating point constant.
static SDValue widenVectorToPartType(SelectionDAG &DAG, SDValue Val, const SDLoc &DL, EVT PartVT)
static SDValue expandLog10(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandLog10 - Lower a log10 intrinsic.
static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, const Value *V, std::optional< CallingConv::ID > CallConv)
getCopyToPartsVector - Create a series of nodes that contain the specified value split into legal par...
static void getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, const Value *V, std::optional< CallingConv::ID > CallConv=std::nullopt, ISD::NodeType ExtendKind=ISD::ANY_EXTEND)
getCopyToParts - Create a series of nodes that contain the specified value split into legal parts.
static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT, SelectionDAGBuilder &Builder)
static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandLog2 - Lower a log2 intrinsic.
static SDValue getAddressForMemoryInput(SDValue Chain, const SDLoc &Location, SDISelAsmOperandInfo &OpInfo, SelectionDAG &DAG)
Get a direct memory input to behave well as an indirect operand.
static bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel)
isOnlyUsedInEntryBlock - If the specified argument is only used in the entry block,...
static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V, const Twine &ErrMsg)
static bool collectInstructionDeps(SmallMapVector< const Instruction *, bool, 8 > *Deps, const Value *V, SmallMapVector< const Instruction *, bool, 8 > *Necessary=nullptr, unsigned Depth=0)
static void findArgumentCopyElisionCandidates(const DataLayout &DL, FunctionLoweringInfo *FuncInfo, ArgCopyElisionMapTy &ArgCopyElisionCandidates)
Scan the entry block of the function in FuncInfo for arguments that look like copies into a local all...
static bool isFunction(SDValue Op)
static SDValue GetExponent(SelectionDAG &DAG, SDValue Op, const TargetLowering &TLI, const SDLoc &dl)
GetExponent - Get the exponent:
static Register FollowCopyChain(MachineRegisterInfo &MRI, Register Reg)
static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS, SelectionDAG &DAG)
ExpandPowI - Expand a llvm.powi intrinsic.
static void findWasmUnwindDestinations(FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB, BranchProbability Prob, SmallVectorImpl< std::pair< MachineBasicBlock *, BranchProbability > > &UnwindDests)
static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandLog - Lower a log intrinsic.
static SDValue getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V, SDValue InChain, std::optional< CallingConv::ID > CC=std::nullopt, std::optional< ISD::NodeType > AssertOp=std::nullopt)
getCopyFromParts - Create a value that contains the specified legal parts combined into the value the...
static SDValue getLimitedPrecisionExp2(SDValue t0, const SDLoc &dl, SelectionDAG &DAG)
static SDValue GetSignificand(SelectionDAG &DAG, SDValue Op, const SDLoc &dl)
GetSignificand - Get the significand and build it into a floating-point number with exponent of 1:
static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandExp - Lower an exp intrinsic.
static const MDNode * getRangeMetadata(const Instruction &I)
static cl::opt< unsigned, true > LimitFPPrecision("limit-float-precision", cl::desc("Generate low-precision inline sequences " "for some float libcalls"), cl::location(LimitFloatPrecision), cl::Hidden, cl::init(0))
static void tryToElideArgumentCopy(FunctionLoweringInfo &FuncInfo, SmallVectorImpl< SDValue > &Chains, DenseMap< int, int > &ArgCopyElisionFrameIndexMap, SmallPtrSetImpl< const Instruction * > &ElidedArgCopyInstrs, ArgCopyElisionMapTy &ArgCopyElisionCandidates, const Argument &Arg, ArrayRef< SDValue > ArgVals, bool &ArgHasUses)
Try to elide argument copies from memory into a local alloca.
static unsigned LimitFloatPrecision
LimitFloatPrecision - Generate low-precision inline sequences for some float libcalls (6,...
static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V, SDValue InChain, std::optional< CallingConv::ID > CC)
getCopyFromPartsVector - Create a value that contains the specified legal parts combined into the val...
static bool InBlock(const Value *V, const BasicBlock *BB)
This file defines the SmallPtrSet class.
This file defines the SmallSet class.
static SymbolRef::Type getType(const Symbol *Sym)
Definition: TapiFile.cpp:40
This pass exposes codegen information to IR-level passes.
Value * RHS
Value * LHS
support::ulittle16_t & Lo
Definition: aarch32.cpp:206
support::ulittle16_t & Hi
Definition: aarch32.cpp:205
bool pointsToConstantMemory(const MemoryLocation &Loc, bool OrLocal=false)
Checks whether the given location points to constant memory, or if OrLocal is true whether it points ...
Class for arbitrary precision integers.
Definition: APInt.h:76
APInt sextOrTrunc(unsigned width) const
Sign extend or truncate to width.
Definition: APInt.cpp:1010
bool isNonNegative() const
Determine if this APInt Value is non-negative (>= 0)
Definition: APInt.h:312
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
Definition: APInt.h:418
an instruction to allocate memory on the stack
Definition: Instructions.h:59
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
Definition: Instructions.h:132
Type * getAllocatedType() const
Return the type that is being allocated by the instruction.
Definition: Instructions.h:125
This class represents an incoming formal argument to a Function.
Definition: Argument.h:31
bool hasAttribute(Attribute::AttrKind Kind) const
Check if an argument has a given attribute.
Definition: Function.cpp:346
unsigned getArgNo() const
Return the index of this formal argument in its containing function.
Definition: Argument.h:49
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
iterator end() const
Definition: ArrayRef.h:154
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:165
iterator begin() const
Definition: ArrayRef.h:153
bool empty() const
empty - Check if the array is empty.
Definition: ArrayRef.h:160
A cache of @llvm.assume calls within a function.
An instruction that atomically checks whether a specified value is in a memory location,...
Definition: Instructions.h:539
This class represents the atomic memcpy intrinsic i.e.
an instruction that atomically reads a memory location, combines it with another value,...
Definition: Instructions.h:748
@ Add
*p = old + v
Definition: Instructions.h:764
@ FAdd
*p = old + v
Definition: Instructions.h:785
@ Min
*p = old <signed v ? old : v
Definition: Instructions.h:778
@ Or
*p = old | v
Definition: Instructions.h:772
@ Sub
*p = old - v
Definition: Instructions.h:766
@ And
*p = old & v
Definition: Instructions.h:768
@ Xor
*p = old ^ v
Definition: Instructions.h:774
@ FSub
*p = old - v
Definition: Instructions.h:788
@ UIncWrap
Increment one up to a maximum value.
Definition: Instructions.h:800
@ Max
*p = old >signed v ? old : v
Definition: Instructions.h:776
@ UMin
*p = old <unsigned v ? old : v
Definition: Instructions.h:782
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
Definition: Instructions.h:796
@ UMax
*p = old >unsigned v ? old : v
Definition: Instructions.h:780
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
Definition: Instructions.h:792
@ UDecWrap
Decrement one until a minimum value or zero.
Definition: Instructions.h:804
@ Nand
*p = ~(old & v)
Definition: Instructions.h:770
static AttributeList get(LLVMContext &C, ArrayRef< std::pair< unsigned, Attribute > > Attrs)
Create an AttributeList with the specified parameters in it.
AttributeSet getRetAttrs() const
The attributes for the ret value are returned.
bool hasFnAttr(Attribute::AttrKind Kind) const
Return true if the attribute exists for the function.
bool getValueAsBool() const
Return the attribute's value as a boolean.
Definition: Attributes.cpp:335
LLVM Basic Block Representation.
Definition: BasicBlock.h:60
const Instruction * getFirstNonPHI() const
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
Definition: BasicBlock.cpp:360
bool isEntryBlock() const
Return true if this is the entry block of the containing function.
Definition: BasicBlock.cpp:564
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:206
const Instruction * getFirstNonPHIOrDbg(bool SkipPseudoOp=true) const
Returns a pointer to the first instruction in this block that is not a PHINode or a debug intrinsic,...
Definition: BasicBlock.cpp:379
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.h:221
const Instruction & back() const
Definition: BasicBlock.h:455
This class represents a no-op cast from one type to another.
bool test(unsigned Idx) const
Definition: BitVector.h:461
void resize(unsigned N, bool t=false)
resize - Grow or shrink the bitvector.
Definition: BitVector.h:341
BitVector & set()
Definition: BitVector.h:351
size_type size() const
size - Returns the number of bits in this bitvector.
Definition: BitVector.h:159
The address of a basic block.
Definition: Constants.h:889
Conditional or Unconditional Branch instruction.
Analysis providing branch probability information.
BranchProbability getEdgeProbability(const BasicBlock *Src, unsigned IndexInSuccessors) const
Get an edge's probability, relative to other out-edges of the Src.
bool isEdgeHot(const BasicBlock *Src, const BasicBlock *Dst) const
Test if an edge is hot relative to other out-edges of the Src.
static uint32_t getDenominator()
static BranchProbability getOne()
uint32_t getNumerator() const
uint64_t scale(uint64_t Num) const
Scale a large integer.
BranchProbability getCompl() const
static BranchProbability getZero()
static void normalizeProbabilities(ProbabilityIter Begin, ProbabilityIter End)
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1494
std::optional< OperandBundleUse > getOperandBundle(StringRef Name) const
Return an operand bundle by name, if present.
Definition: InstrTypes.h:2411
CallingConv::ID getCallingConv() const
Definition: InstrTypes.h:1800
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
Definition: InstrTypes.h:1662
bool isIndirectCall() const
Return true if the callsite is an indirect call.
unsigned countOperandBundlesOfType(StringRef Name) const
Return the number of operand bundles with the tag Name attached to this instruction.
Definition: InstrTypes.h:2387
Value * getCalledOperand() const
Definition: InstrTypes.h:1735
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1687
User::op_iterator arg_end()
Return the iterator pointing to the end of the argument list.
Definition: InstrTypes.h:1668
bool isConvergent() const
Determine if the invoke is convergent.
Definition: InstrTypes.h:2295
FunctionType * getFunctionType() const
Definition: InstrTypes.h:1600
Intrinsic::ID getIntrinsicID() const
Returns the intrinsic ID of the intrinsic called or Intrinsic::not_intrinsic if the called function i...
unsigned arg_size() const
Definition: InstrTypes.h:1685
AttributeList getAttributes() const
Return the parameter attributes for this call.
Definition: InstrTypes.h:1819
CallBr instruction, tracking function calls that may not return control but instead transfer it to a ...
This class represents a function call, abstracting a target machine's calling convention.
This class is the base class for the comparison instructions.
Definition: InstrTypes.h:983
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:993
ConstantDataSequential - A vector or array constant whose element type is a simple 1/2/4/8-byte integ...
Definition: Constants.h:583
A constant value that is initialized with an expression using other constant values.
Definition: Constants.h:1017
static Constant * getBitCast(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition: Constants.cpp:2140
ConstantFP - Floating Point Values [float, double].
Definition: Constants.h:268
This is the shared class of boolean and integer constants.
Definition: Constants.h:80
static ConstantInt * getTrue(LLVMContext &Context)
Definition: Constants.cpp:849
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition: Constants.h:205
static ConstantInt * getFalse(LLVMContext &Context)
Definition: Constants.cpp:856
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:154
const APInt & getValue() const
Return the constant as an APInt value reference.
Definition: Constants.h:145
This class represents a range of values.
Definition: ConstantRange.h:47
APInt getUnsignedMin() const
Return the smallest unsigned value contained in the ConstantRange.
bool isFullSet() const
Return true if this set contains all of the elements possible for this data-type.
bool isEmptySet() const
Return true if this set contains no members.
bool isUpperWrapped() const
Return true if the exclusive upper bound wraps around the unsigned domain.
APInt getUnsignedMax() const
Return the largest unsigned value contained in the ConstantRange.
uint64_t getZExtValue() const
Constant Vector Declarations.
Definition: Constants.h:507
This is an important base class in LLVM.
Definition: Constant.h:41
This is the common base class for constrained floating point intrinsics.
std::optional< fp::ExceptionBehavior > getExceptionBehavior() const
unsigned getNonMetadataArgCount() const
DWARF expression.
bool isEntryValue() const
Check if the expression consists of exactly one entry value operand.
static bool fragmentsOverlap(const FragmentInfo &A, const FragmentInfo &B)
Check if fragments overlap between a pair of FragmentInfos.
static DIExpression * appendOpsToArg(const DIExpression *Expr, ArrayRef< uint64_t > Ops, unsigned ArgNo, bool StackValue=false)
Create a copy of Expr by appending the given list of Ops to each instance of the operand DW_OP_LLVM_a...
static std::optional< FragmentInfo > getFragmentInfo(expr_op_iterator Start, expr_op_iterator End)
Retrieve the details of this fragment expression.
uint64_t getNumLocationOperands() const
Return the number of unique location operands referred to (via DW_OP_LLVM_arg) in this expression; th...
static std::optional< DIExpression * > createFragmentExpression(const DIExpression *Expr, unsigned OffsetInBits, unsigned SizeInBits)
Create a DIExpression to describe one part of an aggregate variable that is fragmented across multipl...
static const DIExpression * convertToUndefExpression(const DIExpression *Expr)
Removes all elements from Expr that do not apply to an undef debug value, which includes every operat...
static DIExpression * prepend(const DIExpression *Expr, uint8_t Flags, int64_t Offset=0)
Prepend DIExpr with a deref and offset operation and optionally turn it into a stack value or/and an ...
static DIExpression * prependOpcodes(const DIExpression *Expr, SmallVectorImpl< uint64_t > &Ops, bool StackValue=false, bool EntryValue=false)
Prepend DIExpr with the given opcodes and optionally turn it into a stack value.
bool isValidLocationForIntrinsic(const DILocation *DL) const
Check that a location is valid for this variable.
Debug location.
Base class for variables.
std::optional< uint64_t > getSizeInBits() const
Determines the size of the variable's type.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
const StructLayout * getStructLayout(StructType *Ty) const
Returns a StructLayout object, indicating the alignment of the struct, its size, and the offsets of i...
Definition: DataLayout.cpp:720
bool isBigEndian() const
Definition: DataLayout.h:239
unsigned getAllocaAddrSpace() const
Definition: DataLayout.h:276
unsigned getIndexSizeInBits(unsigned AS) const
Size in bits of index used for address calculation in getelementptr.
Definition: DataLayout.h:420
TypeSize getTypeStoreSize(Type *Ty) const
Returns the maximum number of bytes that may be overwritten by storing the specified type.
Definition: DataLayout.h:472
Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
Definition: DataLayout.cpp:874
This represents the llvm.dbg.label instruction.
DILabel * getLabel() const
Records a position in IR for a source label (DILabel).
Base class for non-instruction debug metadata records that have positions within IR.
DebugLoc getDebugLoc() const
This represents the llvm.dbg.value instruction.
iterator_range< location_op_iterator > getValues() const
DILocalVariable * getVariable() const
DIExpression * getExpression() const
Record of a variable value-assignment, aka a non instruction representation of the dbg....
DIExpression * getExpression() const
Value * getVariableLocationOp(unsigned OpIdx) const
DILocalVariable * getVariable() const
iterator_range< location_op_iterator > location_ops() const
Get the locations corresponding to the variable referenced by the debug info intrinsic.
A debug info location.
Definition: DebugLoc.h:33
DILocation * getInlinedAt() const
Definition: DebugLoc.cpp:39
iterator find(const_arg_type_t< KeyT > Val)
Definition: DenseMap.h:155
unsigned size() const
Definition: DenseMap.h:99
bool empty() const
Definition: DenseMap.h:98
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition: DenseMap.h:151
iterator end()
Definition: DenseMap.h:84
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition: DenseMap.h:220
void reserve(size_type NumEntries)
Grow the densemap so that it can contain at least NumEntries items before resizing again.
Definition: DenseMap.h:103
static constexpr ElementCount getFixed(ScalarTy MinVal)
Definition: TypeSize.h:308
static constexpr ElementCount get(ScalarTy MinVal, bool Scalable)
Definition: TypeSize.h:314
constexpr bool isScalar() const
Exactly one element.
Definition: TypeSize.h:319
Class representing an expression and its matching format.
This instruction extracts a struct member or array element value from an aggregate value.
This instruction compares its operands according to the predicate given to the constructor.
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
Definition: FastISel.h:66
bool allowReassoc() const
Flag queries.
Definition: FMF.h:65
An instruction for ordering other memory operations.
Definition: Instructions.h:460
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition: Type.cpp:692
This class represents a freeze function that returns random concrete value if an operand is either a ...
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
BranchProbabilityInfo * BPI
Register CreateRegs(const Value *V)
SmallPtrSet< const DbgVariableRecord *, 8 > PreprocessedDVRDeclares
Register DemoteRegister
DemoteRegister - if CanLowerReturn is false, DemoteRegister is a vreg allocated to hold a pointer to ...
BitVector DescribedArgs
Bitvector with a bit set if corresponding argument is described in ArgDbgValues.
DenseMap< const AllocaInst *, int > StaticAllocaMap
StaticAllocaMap - Keep track of frame indices for fixed sized allocas in the entry block.
int getArgumentFrameIndex(const Argument *A)
getArgumentFrameIndex - Get frame index for the byval argument.
DenseMap< const BasicBlock *, MachineBasicBlock * > MBBMap
MBBMap - A mapping from LLVM basic blocks to their machine code entry.
bool isExportedInst(const Value *V) const
isExportedInst - Return true if the specified value is an instruction exported from its block.
const LiveOutInfo * GetLiveOutRegInfo(Register Reg)
GetLiveOutRegInfo - Gets LiveOutInfo for a register, returning NULL if the register is a PHI destinat...
Register InitializeRegForValue(const Value *V)
unsigned ExceptionPointerVirtReg
If the current MBB is a landing pad, the exception pointer and exception selector registers are copie...
SmallPtrSet< const DbgDeclareInst *, 8 > PreprocessedDbgDeclares
Collection of dbg.declare instructions handled after argument lowering and before ISel proper.
DenseMap< const Value *, Register > ValueMap
ValueMap - Since we emit code for the function a basic block at a time, we must remember which virtua...
MachineBasicBlock::iterator InsertPt
MBB - The current insert position inside the current block.
MachineBasicBlock * MBB
MBB - The current block.
std::vector< std::pair< MachineInstr *, unsigned > > PHINodesToUpdate
PHINodesToUpdate - A list of phi instructions whose operand list will be updated after processing the...
SmallVector< MachineInstr *, 8 > ArgDbgValues
ArgDbgValues - A list of DBG_VALUE instructions created during isel for function arguments that are i...
MachineRegisterInfo * RegInfo
Register CreateReg(MVT VT, bool isDivergent=false)
CreateReg - Allocate a single virtual register for the given type.
bool CanLowerReturn
CanLowerReturn - true iff the function's return value can be lowered to registers.
DenseMap< const Value *, ISD::NodeType > PreferredExtendType
Record the preferred extend type (ISD::SIGN_EXTEND or ISD::ZERO_EXTEND) for a value.
Register getCatchPadExceptionPointerVReg(const Value *CPI, const TargetRegisterClass *RC)
Class to represent function types.
Definition: DerivedTypes.h:103
unsigned getNumParams() const
Return the number of fixed parameters this function type requires.
Definition: DerivedTypes.h:142
Type * getParamType(unsigned i) const
Parameter type accessors.
Definition: DerivedTypes.h:135
Type * getReturnType() const
Definition: DerivedTypes.h:124
Data structure describing the variable locations in a function.
const BasicBlock & getEntryBlock() const
Definition: Function.h:787
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
Definition: Function.cpp:703
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Definition: Function.h:232
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
Definition: Function.h:682
bool hasGC() const
hasGC/getGC/setGC/clearGC - The name of the garbage collection algorithm to use during code generatio...
Definition: Function.h:332
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition: Function.h:264
Constant * getPersonalityFn() const
Get the personality function associated with this function.
Definition: Function.cpp:1921
AttributeList getAttributes() const
Return the attribute list for this Function.
Definition: Function.h:340
bool isIntrinsic() const
isIntrinsic - Returns true if the function's name starts with "llvm.".
Definition: Function.h:237
size_t arg_size() const
Definition: Function.h:851
bool isVarArg() const
isVarArg - Return true if this function takes a variable number of arguments.
Definition: Function.h:215
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition: Function.cpp:677
Garbage collection metadata for a single function.
Definition: GCMetadata.h:78
void addStackRoot(int Num, const Constant *Metadata)
addStackRoot - Registers a root that lives on the stack.
Definition: GCMetadata.h:118
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
Definition: Instructions.h:973
static StringRef dropLLVMManglingEscape(StringRef Name)
If the given string begins with the GlobalValue name mangling escape character '\1',...
Definition: GlobalValue.h:567
bool hasDLLImportStorageClass() const
Definition: GlobalValue.h:278
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:656
This instruction compares its operands according to the predicate given to the constructor.
Indirect Branch Instruction.
This instruction inserts a struct field of array element value into an aggregate value.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition: Instruction.h:454
const BasicBlock * getParent() const
Definition: Instruction.h:152
FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
AAMDNodes getAAMetadata() const
Returns the AA metadata for this instruction.
Definition: Metadata.cpp:1706
@ MIN_INT_BITS
Minimum number of bits that can be specified.
Definition: DerivedTypes.h:51
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
Definition: IntrinsicInst.h:54
Invoke instruction.
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
void emitError(uint64_t LocCookie, const Twine &ErrorStr)
emitError - Emit an error message to the currently installed error handler with optional location inf...
The landingpad instruction holds all of the information necessary to generate correct exception handl...
An instruction for reading from memory.
Definition: Instructions.h:184
This class is used to represent ISD::LOAD nodes.
static LocationSize precise(uint64_t Value)
static constexpr LocationSize beforeOrAfterPointer()
Any location before or after the base pointer (but still within the underlying object).
MCSymbol * createTempSymbol()
Create a temporary symbol with a unique name.
Definition: MCContext.cpp:322
MCSymbol * getOrCreateFrameAllocSymbol(const Twine &FuncName, unsigned Idx)
Gets a symbol that will be defined to the final stack offset of a local variable after codegen.
Definition: MCContext.cpp:214
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:40
Metadata node.
Definition: Metadata.h:1067
Machine Value Type.
uint64_t getScalarSizeInBits() const
@ INVALID_SIMPLE_VALUE_TYPE
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
ElementCount getVectorElementCount() const
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool bitsGE(MVT VT) const
Return true if this has no less bits than VT.
bool isScalarInteger() const
Return true if this is an integer, not including vectors.
static MVT getVectorVT(MVT VT, unsigned NumElements)
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
static MVT getIntegerVT(unsigned BitWidth)
void normalizeSuccProbs()
Normalize probabilities of all successors so that the sum of them becomes one.
bool isEHPad() const
Returns true if the block is a landing pad.
void setIsEHCatchretTarget(bool V=true)
Indicates if this is a target block of a catchret.
void setIsCleanupFuncletEntry(bool V=true)
Indicates if this is the entry block of a cleanup funclet.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
void setSuccProbability(succ_iterator I, BranchProbability Prob)
Set successor probability of a given iterator.
std::vector< MachineBasicBlock * >::iterator succ_iterator
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void setIsEHFuncletEntry(bool V=true)
Indicates if this is the entry block of an EH funclet.
void setIsEHScopeEntry(bool V=true)
Indicates if this is the entry block of an EH scope, i.e., the block that that used to have a catchpa...
void setMachineBlockAddressTaken()
Set this block to indicate that its address is used as something other than the target of a terminato...
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
bool hasVarSizedObjects() const
This method may be called any time after instruction selection is complete to determine if the stack ...
void setIsImmutableObjectIndex(int ObjectIdx, bool IsImmutable)
Marks the immutability of an object.
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
void setHasPatchPoint(bool s=true)
void setHasStackMap(bool s=true)
bool hasOpaqueSPAdjustment() const
Returns true if the function contains opaque dynamic stack adjustments.
int getStackProtectorIndex() const
Return the index for the stack protector object.
void setStackProtectorIndex(int I)
void setIsAliasedObjectIndex(int ObjectIdx, bool IsAliased)
Set "maybe pointed to by an LLVM IR value" for an object.
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
void RemoveStackObject(int ObjectIdx)
Remove or mark dead a statically sized stack object.
void setFunctionContextIndex(int I)
Description of the location of a variable whose Address is valid and unchanging during function execu...
const WinEHFuncInfo * getWinEHFuncInfo() const
getWinEHFuncInfo - Return information about how the current function uses Windows exception handling.
void setCallsUnwindInit(bool b)
bool useDebugInstrRef() const
Returns true if the function's variable locations are tracked with instruction referencing.
void setCallSiteBeginLabel(MCSymbol *BeginLabel, unsigned Site)
Map the begin label for a call site.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
void setHasEHCatchret(bool V)
void setCallsEHReturn(bool b)
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
unsigned getTypeIDFor(const GlobalValue *TI)
Return the type id for the specified typeinfo. This is function wide.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
auto getInStackSlotVariableDbgInfo()
Returns the collection of variables for which we have debug info and that have been assigned a stack ...
void addCodeViewAnnotation(MCSymbol *Label, MDNode *MD)
Record annotations associated with a particular label.
Function & getFunction()
Return the LLVM function that this machine code represents.
MachineModuleInfo & getMMI() const
const MachineBasicBlock & front() const
void addInvoke(MachineBasicBlock *LandingPad, MCSymbol *BeginLabel, MCSymbol *EndLabel)
Provide the begin and end labels of an invoke style call and associate it with a try landing pad bloc...
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
void erase(iterator MBBI)
void insert(iterator MBBI, MachineBasicBlock *MBB)
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
Representation of each machine instruction.
Definition: MachineInstr.h:69
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MONonTemporal
The memory access is non-temporal.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
This class contains meta information specific to a module.
const MCContext & getContext() const
const Module * getModule() const
void setCurrentCallSite(unsigned Site)
Set the call site currently being processed.
unsigned getCurrentCallSite()
Get the call site currently being processed, if any.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
static MachineOperand CreateFI(int Idx)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
ArrayRef< std::pair< MCRegister, Register > > liveins() const
An SDNode that represents everything that will be needed to construct a MachineInstr.
bool contains(const KeyT &Key) const
Definition: MapVector.h:163
std::pair< iterator, bool > try_emplace(const KeyT &Key, Ts &&...Args)
Definition: MapVector.h:118
Representation for a specific memory location.
static MemoryLocation getAfter(const Value *Ptr, const AAMDNodes &AATags=AAMDNodes())
Return a location that may access any location after Ptr, while remaining within the underlying objec...
Metadata wrapper in the Value hierarchy.
Definition: Metadata.h:176
Root of the metadata hierarchy.
Definition: Metadata.h:62
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
Metadata * getModuleFlag(StringRef Key) const
Return the corresponding value if Key appears in module flags, otherwise return null.
Definition: Module.cpp:333
Utility class for integer operators which may exhibit overflow - Add, Sub, Mul, and Shl.
Definition: Operator.h:76
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Definition: DerivedTypes.h:662
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Definition: Constants.cpp:1827
A udiv or sdiv instruction, which can be marked as "exact", indicating that no bits are destroyed.
Definition: Operator.h:151
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
Definition: Register.h:91
static constexpr bool isVirtualRegister(unsigned Reg)
Return true if the specified register number is in the virtual register namespace.
Definition: Register.h:71
static constexpr bool isPhysicalRegister(unsigned Reg)
Return true if the specified register number is in the physical register namespace.
Definition: Register.h:65
Resume the propagation of an exception.
Return a value (possibly void), from a function.
Holds the information from a dbg_label node through SDISel.
static SDDbgOperand fromNode(SDNode *Node, unsigned ResNo)
static SDDbgOperand fromFrameIdx(unsigned FrameIdx)
static SDDbgOperand fromVReg(unsigned VReg)
static SDDbgOperand fromConst(const Value *Const)
Holds the information from a dbg_value node through SDISel.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
iterator_range< value_op_iterator > op_values() const
unsigned getIROrder() const
Return the node ordering.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
const SDValue & getOperand(unsigned Num) const
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
void dump() const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
SelectionDAGBuilder - This is the common target-independent lowering implementation that is parameter...
SDValue getValue(const Value *V)
getValue - Return an SDValue for the given Value.
void addDanglingDebugInfo(SmallVectorImpl< Value * > &Values, DILocalVariable *Var, DIExpression *Expr, bool IsVariadic, DebugLoc DL, unsigned Order)
Register a dbg_value which relies on a Value which we have not yet seen.
void visitDbgInfo(const Instruction &I)
void clearDanglingDebugInfo()
Clear the dangling debug information map.
void clear()
Clear out the current SelectionDAG and the associated state and prepare this SelectionDAGBuilder obje...
void visitBitTestHeader(SwitchCG::BitTestBlock &B, MachineBasicBlock *SwitchBB)
visitBitTestHeader - This function emits necessary code to produce value suitable for "bit tests"
void LowerStatepoint(const GCStatepointInst &I, const BasicBlock *EHPadBB=nullptr)
std::unique_ptr< SDAGSwitchLowering > SL
SDValue lowerRangeToAssertZExt(SelectionDAG &DAG, const Instruction &I, SDValue Op)
bool HasTailCall
This is set to true if a call in the current block has been translated as a tail call.
bool ShouldEmitAsBranches(const std::vector< SwitchCG::CaseBlock > &Cases)
If the set of cases should be emitted as a series of branches, return true.
void EmitBranchForMergedCondition(const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB, MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB, BranchProbability TProb, BranchProbability FProb, bool InvertCond)
EmitBranchForMergedCondition - Helper method for FindMergedConditions.
void LowerDeoptimizeCall(const CallInst *CI)
void LowerCallSiteWithDeoptBundle(const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB)
SwiftErrorValueTracking & SwiftError
Information about the swifterror values used throughout the function.
SDValue getNonRegisterValue(const Value *V)
getNonRegisterValue - Return an SDValue for the given Value, but don't look in FuncInfo....
void CopyValueToVirtualRegister(const Value *V, unsigned Reg, ISD::NodeType ExtendType=ISD::ANY_EXTEND)
DenseMap< MachineBasicBlock *, SmallVector< unsigned, 4 > > LPadToCallSiteMap
Map a landing pad to the call site indexes.
void handleDebugDeclare(Value *Address, DILocalVariable *Variable, DIExpression *Expression, DebugLoc DL)
void visitBitTestCase(SwitchCG::BitTestBlock &BB, MachineBasicBlock *NextMBB, BranchProbability BranchProbToNext, unsigned Reg, SwitchCG::BitTestCase &B, MachineBasicBlock *SwitchBB)
visitBitTestCase - this function produces one "bit test"
bool shouldKeepJumpConditionsTogether(const FunctionLoweringInfo &FuncInfo, const BranchInst &I, Instruction::BinaryOps Opc, const Value *Lhs, const Value *Rhs, TargetLoweringBase::CondMergingParams Params) const
void LowerCallTo(const CallBase &CB, SDValue Callee, bool IsTailCall, bool IsMustTailCall, const BasicBlock *EHPadBB=nullptr)
StatepointLoweringState StatepointLowering
State used while lowering a statepoint sequence (gc_statepoint, gc_relocate, and gc_result).
void init(GCFunctionInfo *gfi, AAResults *AA, AssumptionCache *AC, const TargetLibraryInfo *li)
DenseMap< const Constant *, unsigned > ConstantsOut
void populateCallLoweringInfo(TargetLowering::CallLoweringInfo &CLI, const CallBase *Call, unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy, AttributeSet RetAttrs, bool IsPatchPoint)
Populate a CallLowerinInfo (into CLI) based on the properties of the call being lowered.
void salvageUnresolvedDbgValue(const Value *V, DanglingDebugInfo &DDI)
For the given dangling debuginfo record, perform last-ditch efforts to resolve the debuginfo to somet...
SmallVector< SDValue, 8 > PendingLoads
Loads are not emitted to the program immediately.
GCFunctionInfo * GFI
Garbage collection metadata for the function.
SDValue getRoot()
Similar to getMemoryRoot, but also flushes PendingConstrainedFP(Strict) items.
void ExportFromCurrentBlock(const Value *V)
ExportFromCurrentBlock - If this condition isn't known to be exported from the current basic block,...
void resolveOrClearDbgInfo()
Evict any dangling debug information, attempting to salvage it first.
std::pair< SDValue, SDValue > lowerInvokable(TargetLowering::CallLoweringInfo &CLI, const BasicBlock *EHPadBB=nullptr)
SDValue getMemoryRoot()
Return the current virtual root of the Selection DAG, flushing any PendingLoad items.
void resolveDanglingDebugInfo(const Value *V, SDValue Val)
If we saw an earlier dbg_value referring to V, generate the debug data structures now that we've seen...
void visit(const Instruction &I)
void dropDanglingDebugInfo(const DILocalVariable *Variable, const DIExpression *Expr)
If we have dangling debug info that describes Variable, or an overlapping part of variable considerin...
SDValue getCopyFromRegs(const Value *V, Type *Ty)
If there was virtual register allocated for the value V emit CopyFromReg of the specified type Ty.
void CopyToExportRegsIfNeeded(const Value *V)
CopyToExportRegsIfNeeded - If the given value has virtual registers created for it,...
void handleKillDebugValue(DILocalVariable *Var, DIExpression *Expr, DebugLoc DbgLoc, unsigned Order)
Create a record for a kill location debug intrinsic.
void visitJumpTable(SwitchCG::JumpTable &JT)
visitJumpTable - Emit JumpTable node in the current MBB
void visitJumpTableHeader(SwitchCG::JumpTable &JT, SwitchCG::JumpTableHeader &JTH, MachineBasicBlock *SwitchBB)
visitJumpTableHeader - This function emits necessary code to produce index in the JumpTable from swit...
static const unsigned LowestSDNodeOrder
Lowest valid SDNodeOrder.
FunctionLoweringInfo & FuncInfo
Information about the function as a whole.
void setValue(const Value *V, SDValue NewN)
void FindMergedConditions(const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB, MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB, Instruction::BinaryOps Opc, BranchProbability TProb, BranchProbability FProb, bool InvertCond)
const TargetLibraryInfo * LibInfo
bool isExportableFromCurrentBlock(const Value *V, const BasicBlock *FromBB)
void visitSPDescriptorParent(StackProtectorDescriptor &SPD, MachineBasicBlock *ParentBB)
Codegen a new tail for a stack protector check ParentMBB which has had its tail spliced into a stack ...
bool handleDebugValue(ArrayRef< const Value * > Values, DILocalVariable *Var, DIExpression *Expr, DebugLoc DbgLoc, unsigned Order, bool IsVariadic)
For a given list of Values, attempt to create and record a SDDbgValue in the SelectionDAG.
SDValue getControlRoot()
Similar to getRoot, but instead of flushing all the PendingLoad items, flush all the PendingExports (...
void UpdateSplitBlock(MachineBasicBlock *First, MachineBasicBlock *Last)
When an MBB was split during scheduling, update the references that need to refer to the last resulti...
SDValue getValueImpl(const Value *V)
getValueImpl - Helper function for getValue and getNonRegisterValue.
void visitSwitchCase(SwitchCG::CaseBlock &CB, MachineBasicBlock *SwitchBB)
visitSwitchCase - Emits the necessary code to represent a single node in the binary search tree resul...
void visitSPDescriptorFailure(StackProtectorDescriptor &SPD)
Codegen the failure basic block for a stack protector check.
std::unique_ptr< FunctionLoweringInfo > FuncInfo
SmallPtrSet< const Instruction *, 4 > ElidedArgCopyInstrs
const TargetLowering * TLI
MachineFunction * MF
virtual void emitFunctionEntryCode()
SwiftErrorValueTracking * SwiftError
std::unique_ptr< SelectionDAGBuilder > SDB
Targets can subclass this to parameterize the SelectionDAG lowering and instruction selection process...
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrnlen(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src, SDValue MaxLength, MachinePointerInfo SrcPtrInfo) const
virtual std::pair< SDValue, SDValue > EmitTargetCodeForMemcmp(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, SDValue Op3, MachinePointerInfo Op1PtrInfo, MachinePointerInfo Op2PtrInfo) const
Emit target-specific code that performs a memcmp/bcmp, in cases where that is faster than a libcall.
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrcpy(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dest, SDValue Src, MachinePointerInfo DestPtrInfo, MachinePointerInfo SrcPtrInfo, bool isStpcpy) const
Emit target-specific code that performs a strcpy or stpcpy, in cases where that is faster than a libc...
virtual std::pair< SDValue, SDValue > EmitTargetCodeForMemchr(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Src, SDValue Char, SDValue Length, MachinePointerInfo SrcPtrInfo) const
Emit target-specific code that performs a memchr, in cases where that is faster than a libcall.
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrcmp(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, MachinePointerInfo Op1PtrInfo, MachinePointerInfo Op2PtrInfo) const
Emit target-specific code that performs a strcmp, in cases where that is faster than a libcall.
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrlen(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src, MachinePointerInfo SrcPtrInfo) const
virtual SDValue EmitTargetCodeForSetTag(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Addr, SDValue Size, MachinePointerInfo DstPtrInfo, bool ZeroData) const
Help to insert SDNodeFlags automatically in transforming.
Definition: SelectionDAG.h:361
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:225
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
Definition: SelectionDAG.h:722
SDValue getExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT, unsigned Opcode)
Convert Op, which must be of integer type, to the integer type VT, by either any/sign/zero-extending ...
Definition: SelectionDAG.h:954
SDValue getLabelNode(unsigned Opcode, const SDLoc &dl, SDValue Root, MCSymbol *Label)
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
Definition: SelectionDAG.h:551
SDValue getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), AAResults *AA=nullptr)
SDValue getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, ISD::LoadExtType ExtTy)
SDValue getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr, unsigned SrcAS, unsigned DestAS)
Return an AddrSpaceCastSDNode.
const TargetSubtargetInfo & getSubtarget() const
Definition: SelectionDAG.h:474
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
BlockFrequencyInfo * getBFI() const
Definition: SelectionDAG.h:488
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
void ExtractVectorElements(SDValue Op, SmallVectorImpl< SDValue > &Args, unsigned Start=0, unsigned Count=0, EVT EltVT=EVT())
Append the extracted elements from Start to Count out of the vector Op in Args.
SDValue getAtomicMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Value, SDValue Size, Type *SizeTy, unsigned ElemSz, bool isTailCall, MachinePointerInfo DstPtrInfo)
SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm, bool ConstantFold=true)
Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
SDValue getPseudoProbeNode(const SDLoc &Dl, SDValue Chain, uint64_t Guid, uint64_t Index, uint32_t Attr)
Creates a PseudoProbeSDNode with function GUID Guid and the index of the block Index it is probing,...
SDValue getConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offs=0, bool isT=false, unsigned TargetFlags=0)
SDValue getStridedLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL, SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, bool IsExpanding=false)
SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTs, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachineMemOperand *MMO)
Gets a node for an atomic cmpxchg op.
SDDbgValue * getVRegDbgValue(DIVariable *Var, DIExpression *Expr, unsigned VReg, bool IsIndirect, const DebugLoc &DL, unsigned O)
Creates a VReg SDDbgValue node.
void ReplaceAllUsesOfValuesWith(const SDValue *From, const SDValue *To, unsigned Num)
Like ReplaceAllUsesOfValueWith, but for multiple values at once.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
void addMMRAMetadata(const SDNode *Node, MDNode *MMRA)
Set MMRAMetadata to be associated with Node.
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
SDValue getElementCount(const SDLoc &DL, EVT VT, ElementCount EC, bool ConstantFold=true)
SDValue getGetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr, EVT MemVT, MachineMemOperand *MMO)
SDValue getAssertAlign(const SDLoc &DL, SDValue V, Align A)
Return an AssertAlignSDNode.
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getStepVector(const SDLoc &DL, EVT ResVT, const APInt &StepVal)
Returns a vector of type ResVT whose elements contain the linear sequence <0, Step,...
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
Align getEVTAlign(EVT MemoryVT) const
Compute the default alignment value for the given type.
bool shouldOptForSize() const
SDValue getVPZExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op, SDValue Mask, SDValue EVL)
Convert a vector-predicated Op, which must be an integer vector, to the vector-type VT,...
const TargetLowering & getTargetLoweringInfo() const
Definition: SelectionDAG.h:478
static constexpr unsigned MaxRecursionDepth
Definition: SelectionDAG.h:448
SDValue getStridedStoreVP(SDValue Chain, const SDLoc &DL, SDValue Val, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
void AddDbgValue(SDDbgValue *DB, bool isParameter)
Add a dbg_value SDNode.
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
SDValue getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
Definition: SelectionDAG.h:828
SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), AAResults *AA=nullptr)
void DeleteNode(SDNode *N)
Remove the specified node from the system.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDDbgValue * getDbgValueList(DIVariable *Var, DIExpression *Expr, ArrayRef< SDDbgOperand > Locs, ArrayRef< SDNode * > Dependencies, bool IsIndirect, const DebugLoc &DL, unsigned O, bool IsVariadic)
Creates a SDDbgValue node from a list of locations.
SDValue getNegative(SDValue Val, const SDLoc &DL, EVT VT)
Create negative operation as (SUB 0, Val).
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:472
ProfileSummaryInfo * getPSI() const
Definition: SelectionDAG.h:487
SDValue getTargetFrameIndex(int FI, EVT VT)
Definition: SelectionDAG.h:727
SDValue getTokenFactor(const SDLoc &DL, SmallVectorImpl< SDValue > &Vals)
Creates a new TokenFactor containing Vals.
const SelectionDAGTargetInfo & getSelectionDAGInfo() const
Definition: SelectionDAG.h:480
SDDbgLabel * getDbgLabel(DILabel *Label, const DebugLoc &DL, unsigned O)
Creates a SDDbgLabel node.
SDValue getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
SDValue getGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, bool isTargetGA=false, unsigned TargetFlags=0)
SDValue getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, bool isTailCall, MachinePointerInfo DstPtrInfo, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input.
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getMDNode(const MDNode *MD)
Return an MDNodeSDNode which holds an MDNode.
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getSrcValue(const Value *v)
Construct a node to track a Value* through the backend.
SDValue getSplatVector(EVT VT, const SDLoc &DL, SDValue Op)
Definition: SelectionDAG.h:862
SDValue getAtomicMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Type *SizeTy, unsigned ElemSz, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo)
MaybeAlign InferPtrAlign(SDValue Ptr) const
Infer alignment of a load / store address.
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
SDValue getRegister(unsigned Reg, EVT VT)
void AddDbgLabel(SDDbgLabel *DB)
Add a dbg_label SDNode.
SDValue getBasicBlock(MachineBasicBlock *MBB)
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
SDValue getEHLabel(const SDLoc &dl, SDValue Root, MCSymbol *Label)
SDValue getSetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr, EVT MemVT, MachineMemOperand *MMO)
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
Definition: SelectionDAG.h:473
SDValue getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either truncating it or perform...
SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N)
Definition: SelectionDAG.h:773
SDValue getBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset=0, bool isTarget=false, unsigned TargetFlags=0)
SDValue getLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment, MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, const MDNode *Ranges=nullptr, bool IsExpanding=false)
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDDbgValue * getConstantDbgValue(DIVariable *Var, DIExpression *Expr, const Value *C, const DebugLoc &DL, unsigned O)
Creates a constant SDDbgValue node.
SDValue getScatterVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getValueType(EVT)
SDValue getTargetConstantFP(double Val, const SDLoc &DL, EVT VT)
Definition: SelectionDAG.h:708
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of float type, to the float type VT, by either extending or rounding (by tr...
SDValue getAtomicMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Type *SizeTy, unsigned ElemSz, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo)
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:676
SDDbgValue * getFrameIndexDbgValue(DIVariable *Var, DIExpression *Expr, unsigned FI, bool IsIndirect, const DebugLoc &DL, unsigned O)
Creates a FrameIndex SDDbgValue node.
SDValue getJumpTable(int JTI, EVT VT, bool isTarget=false, unsigned TargetFlags=0)
SDValue getVPPtrExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op, SDValue Mask, SDValue EVL)
Convert a vector-predicated Op, which must be of integer type, to the vector-type integer type VT,...
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:469
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
Definition: SelectionDAG.h:799
SDValue getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to extend the Op as a pointer value assuming it was the smaller SrcTy ...
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
const FunctionVarLocs * getFunctionVarLocs() const
Returns the result of the AssignmentTrackingAnalysis pass if it's available, otherwise return nullptr...
Definition: SelectionDAG.h:484
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
SDValue getCondCode(ISD::CondCode Cond)
SDValue getLifetimeNode(bool IsStart, const SDLoc &dl, SDValue Chain, int FrameIndex, int64_t Size, int64_t Offset=-1)
Creates a LifetimeSDNode that starts (IsStart==true) or ends (IsStart==false) the lifetime of the por...
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object.
LLVMContext * getContext() const
Definition: SelectionDAG.h:485
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
Definition: SelectionDAG.h:560
void addPCSections(const SDNode *Node, MDNode *MD)
Set PCSections to be associated with Node.
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL, bool LegalTypes=true)
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, LocationSize Size=0, const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
SDValue getMCSymbol(MCSymbol *Sym, EVT VT)
SDValue getSetCCVP(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Mask, SDValue EVL)
Helper function to make it easier to build VP_SETCCs if you just have an ISD::CondCode instead of an ...
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:554
SDDbgValue * getDbgValue(DIVariable *Var, DIExpression *Expr, SDNode *N, unsigned R, bool IsIndirect, const DebugLoc &DL, unsigned O)
Creates a SDDbgValue node.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
SDValue getSplat(EVT VT, const SDLoc &DL, SDValue Op)
Returns a node representing a splat of one value into all lanes of the provided vector type.
Definition: SelectionDAG.h:878
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
SDValue getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, bool IsTruncating=false)
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
Definition: SmallPtrSet.h:321
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:342
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:427
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:135
bool empty() const
Definition: SmallVector.h:94
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:586
reference emplace_back(ArgTypes &&... Args)
Definition: SmallVector.h:950
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:696
void swap(SmallVectorImpl &RHS)
Definition: SmallVector.h:981
void resize(size_type N)
Definition: SmallVector.h:651
void push_back(const T &Elt)
Definition: SmallVector.h:426
pointer data()
Return a pointer to the vector's buffer, even if empty().
Definition: SmallVector.h:299
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
Encapsulates all of the information needed to generate a stack protector check, and signals to isel w...
MachineBasicBlock * getSuccessMBB()
MachineBasicBlock * getFailureMBB()
void clear()
Clear the memory usage of this object.
An instruction for storing to memory.
Definition: Instructions.h:317
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
constexpr bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:134
constexpr const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
Definition: StringRef.h:131
TypeSize getElementOffset(unsigned Idx) const
Definition: DataLayout.h:651
Class to represent struct types.
Definition: DerivedTypes.h:216
void setCurrentVReg(const MachineBasicBlock *MBB, const Value *, Register)
Set the swifterror virtual register in the VRegDefMap for this basic block.
Register getOrCreateVRegUseAt(const Instruction *, const MachineBasicBlock *, const Value *)
Get or create the swifterror value virtual register for a use of a swifterror by an instruction.
Register getOrCreateVRegDefAt(const Instruction *, const MachineBasicBlock *, const Value *)
Get or create the swifterror value virtual register for a def of a swifterror by an instruction.
const Value * getFunctionArg() const
Get the (unique) function argument that was marked swifterror, or nullptr if this function has no swi...
Multiway switch.
Align getStackAlign() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
TargetInstrInfo - Interface to description of machine instruction set.
TargetIntrinsicInfo - Interface to description of machine instruction set.
Provides information about what library functions are available for the current target.
bool hasOptimizedCodeGen(LibFunc F) const
Tests if the function is both available and a candidate for optimized code generation.
bool getLibFunc(StringRef funcName, LibFunc &F) const
Searches for a particular function name.
void setAttributes(const CallBase *Call, unsigned ArgIdx)
Set CallLoweringInfo attribute flags based on a call instruction and called function attributes.
virtual bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT) const
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
EVT getMemValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
virtual bool useStackGuardXorFP() const
If this function returns true, stack protection checks should XOR the frame pointer (or whichever poi...
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
virtual bool isLegalScaleForGatherScatter(uint64_t Scale, uint64_t ElemSize) const
virtual bool isSExtCheaperThanZExt(EVT FromTy, EVT ToTy) const
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
virtual CondMergingParams getJumpConditionMergingParams(Instruction::BinaryOps, const Value *, const Value *) const
const TargetMachine & getTargetMachine() const
virtual unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain targets require unusual breakdowns of certain types.
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
virtual MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
virtual Value * getSDagStackGuard(const Module &M) const
Return the variable that's previously inserted by insertSSPDeclarations, if any, otherwise return nul...
virtual unsigned getNumRegisters(LLVMContext &Context, EVT VT, std::optional< MVT > RegisterVT=std::nullopt) const
Return the number of registers that this ValueType will eventually require.
bool isJumpExpensive() const
Return true if Flow Control is an expensive operation that should be avoided.
unsigned getBitWidthForCttzElements(Type *RetTy, ElementCount EC, bool ZeroIsPoison, const ConstantRange *VScaleRange) const
Return the minimum number of bits required to hold the maximum possible number of trailing zero vecto...
virtual bool shouldExtendGSIndex(EVT VT, EVT &EltTy) const
Returns true if the index type for a masked gather/scatter requires extending.
virtual unsigned getVectorTypeBreakdownForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const
Certain targets such as MIPS require that some types such as vectors are always broken down into scal...
virtual Function * getSSPStackGuardCheck(const Module &M) const
If the target has a standard stack protection check function that performs validation and error handl...
Register getStackPointerRegisterToSaveRestore() const
If a physical register, this specifies the register that llvm.savestack/llvm.restorestack should save...
LegalizeAction getFixedPointOperationAction(unsigned Op, EVT VT, unsigned Scale) const
Some fixed point operations may be natively supported by the target but only for specific scales.
MachineMemOperand::Flags getAtomicMemOperandFlags(const Instruction &AI, const DataLayout &DL) const
virtual bool getTgtMemIntrinsic(IntrinsicInfo &, const CallInst &, MachineFunction &, unsigned) const
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
virtual bool allowsMisalignedMemoryAccesses(EVT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *=nullptr) const
Determine if the target supports unaligned memory accesses.
bool hasBigEndianPartOrdering(EVT VT, const DataLayout &DL) const
When splitting a value of the specified type into parts, does the Lo or Hi part come first?...
virtual Align getABIAlignmentForCallingConv(Type *ArgTy, const DataLayout &DL) const
Certain targets have context sensitive alignment requirements, where one type has the alignment requi...
virtual bool shouldExpandGetActiveLaneMask(EVT VT, EVT OpVT) const
Return true if the @llvm.get.active.lane.mask intrinsic should be expanded using generic code in Sele...
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL, bool LegalTypes=true) const
Returns the type for the shift amount of a shift opcode.
MachineMemOperand::Flags getLoadMemOperandFlags(const LoadInst &LI, const DataLayout &DL, AssumptionCache *AC=nullptr, const TargetLibraryInfo *LibInfo=nullptr) const
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
MVT getProgramPointerTy(const DataLayout &DL) const
Return the type for code pointers, which is determined by the program address space specified through...
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
virtual MVT getFenceOperandTy(const DataLayout &DL) const
Return the type for operands of fence.
virtual bool shouldExpandGetVectorLength(EVT CountVT, unsigned VF, bool IsScalable) const
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
virtual uint64_t getByValTypeAlignment(Type *Ty, const DataLayout &DL) const
Return the desired alignment for ByVal or InAlloca aggregate function arguments in the caller paramet...
virtual MVT hasFastEqualityCompare(unsigned NumBits) const
Return the preferred operand type if the target has a quick way to compare integer values of the give...
MachineMemOperand::Flags getStoreMemOperandFlags(const StoreInst &SI, const DataLayout &DL) const
virtual bool shouldExpandCttzElements(EVT VT) const
Return true if the @llvm.experimental.cttz.elts intrinsic should be expanded using generic code in Se...
virtual bool signExtendConstant(const ConstantInt *C) const
Return true if this constant should be sign extended when promoting to a larger type.
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
virtual Register getExceptionPointerRegister(const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception address on entry to an ...
bool supportsUnalignedAtomics() const
Whether the target supports unaligned atomic operations.
std::vector< ArgListEntry > ArgListTy
bool isBeneficialToExpandPowI(int64_t Exponent, bool OptForSize) const
Return true if it is beneficial to expand an @llvm.powi.
MVT getFrameIndexTy(const DataLayout &DL) const
Return the type for frame index, which is determined by the alloca address space specified through th...
virtual Register getExceptionSelectorRegister(const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception typeid on entry to a la...
virtual MVT getPointerMemTy(const DataLayout &DL, uint32_t AS=0) const
Return the in-memory pointer type for the given address space, defaults to the pointer type from the ...
MVT getRegisterType(MVT VT) const
Return the type of registers that this ValueType will eventually require.
bool isOperationLegalOrCustomOrPromote(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
unsigned getVectorTypeBreakdown(LLVMContext &Context, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const
Vector types are broken down into some number of legal first class types.
virtual MVT getVPExplicitVectorLengthTy() const
Returns the type to be used for the EVL/AVL operand of VP nodes: ISD::VP_ADD, ISD::VP_SUB,...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
virtual bool supportKCFIBundles() const
Return true if the target supports kcfi operand bundles.
virtual bool supportSwiftError() const
Return true if the target supports swifterror attribute.
virtual SDValue emitStackGuardXorFP(SelectionDAG &DAG, SDValue Val, const SDLoc &DL) const
virtual bool useLoadStackGuardNode() const
If this function returns true, SelectionDAGBuilder emits a LOAD_STACK_GUARD node when it is lowering ...
virtual EVT getTypeForExtReturn(LLVMContext &Context, EVT VT, ISD::NodeType) const
Return the type that should be used to zero or sign extend a zeroext/signext integer return value.
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
virtual InlineAsm::ConstraintCode getInlineAsmMemConstraint(StringRef ConstraintCode) const
std::vector< AsmOperandInfo > AsmOperandInfoVector
SDValue expandIS_FPCLASS(EVT ResultVT, SDValue Op, FPClassTest Test, SDNodeFlags Flags, const SDLoc &DL, SelectionDAG &DAG) const
Expand check for floating point class.
virtual SDValue prepareVolatileOrAtomicLoad(SDValue Chain, const SDLoc &DL, SelectionDAG &DAG) const
This callback is used to prepare for a volatile or atomic load.
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
virtual bool splitValueIntoRegisterParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, std::optional< CallingConv::ID > CC) const
Target-specific splitting of values into parts that fit a register storing a legal type.
virtual SDValue joinRegisterPartsIntoValue(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, std::optional< CallingConv::ID > CC) const
Target-specific combining of register parts into its original value.
virtual SDValue LowerCall(CallLoweringInfo &, SmallVectorImpl< SDValue > &) const
This hook must be implemented to lower calls into the specified DAG.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
virtual const char * getClearCacheBuiltinName() const
Return the builtin name for the __builtin___clear_cache intrinsic Default is to invoke the clear cach...
virtual SDValue LowerAsmOutputForConstraint(SDValue &Chain, SDValue &Glue, const SDLoc &DL, const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
virtual SDValue LowerFormalArguments(SDValue, CallingConv::ID, bool, const SmallVectorImpl< ISD::InputArg > &, const SDLoc &, SelectionDAG &, SmallVectorImpl< SDValue > &) const
This hook must be implemented to lower the incoming (formal) arguments, described by the Ins array,...
virtual AsmOperandInfoVector ParseConstraints(const DataLayout &DL, const TargetRegisterInfo *TRI, const CallBase &Call) const
Split up the constraint string from the inline assembly value into the specific constraints and their...
virtual SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const
This callback is invoked for operations that are unsupported by the target, which are registered to u...
virtual SDValue LowerReturn(SDValue, CallingConv::ID, bool, const SmallVectorImpl< ISD::OutputArg > &, const SmallVectorImpl< SDValue > &, const SDLoc &, SelectionDAG &) const
This hook must be implemented to lower outgoing return values, described by the Outs array,...
virtual bool functionArgumentNeedsConsecutiveRegisters(Type *Ty, CallingConv::ID CallConv, bool isVarArg, const DataLayout &DL) const
For some targets, an LLVM struct type must be broken down into multiple simple types,...
virtual void ComputeConstraintToUse(AsmOperandInfo &OpInfo, SDValue Op, SelectionDAG *DAG=nullptr) const
Determines the constraint code and constraint type to use for the specific AsmOperandInfo,...
virtual void CollectTargetIntrinsicOperands(const CallInst &I, SmallVectorImpl< SDValue > &Ops, SelectionDAG &DAG) const
virtual void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
virtual bool CanLowerReturn(CallingConv::ID, MachineFunction &, bool, const SmallVectorImpl< ISD::OutputArg > &, LLVMContext &) const
This hook should be implemented to check whether the return values described by the Outs array can fi...
virtual void LowerOperationWrapper(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const
This callback is invoked by the type legalizer to legalize nodes with an illegal operand type but leg...
virtual bool isInlineAsmTargetBranch(const SmallVectorImpl< StringRef > &AsmStrs, unsigned OpNo) const
On x86, return true if the operand with index OpNo is a CALL or JUMP instruction, which can use eithe...
virtual const TargetIntrinsicInfo * getIntrinsicInfo() const
If intrinsic information is available, return it. If not, return null.
virtual bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const
Returns true if a cast between SrcAS and DestAS is a noop.
const Triple & getTargetTriple() const
virtual TargetTransformInfo getTargetTransformInfo(const Function &F) const
Return a TargetTransformInfo for a given function.
TargetOptions Options
CodeModel::Model getCodeModel() const
Returns the code model.
CodeGenOptLevel getOptLevel() const
Returns the optimization level: None, Less, Default, or Aggressive.
unsigned NoNaNsFPMath
NoNaNsFPMath - This flag is enabled when the -enable-no-nans-fp-math flag is specified on the command...
unsigned EnableFastISel
EnableFastISel - This flag enables fast-path instruction selection which trades away generated code q...
unsigned NoTrapAfterNoreturn
Do not emit a trap instruction for 'unreachable' IR instructions behind noreturn calls,...
unsigned TrapUnreachable
Emit target-specific trap instruction for 'unreachable' IR instructions.
FPOpFusion::FPOpFusionMode AllowFPOpFusion
AllowFPOpFusion - This flag is set by the -fp-contract=xxx option.
unsigned getID() const
Return the register class ID number.
iterator begin() const
begin/end - Return all of the registers in this class.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const TargetFrameLowering * getFrameLowering() const
virtual const TargetInstrInfo * getInstrInfo() const
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
@ TCK_Latency
The latency of instruction.
InstructionCost getInstructionCost(const User *U, ArrayRef< const Value * > Operands, TargetCostKind CostKind) const
Estimate the cost of a given IR user when lowered.
Target - Wrapper for Target specific information.
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
ArchType getArch() const
Get the parsed architecture type of this triple.
Definition: Triple.h:373
bool isPS() const
Tests whether the target is the PS4 or PS5 platform.
Definition: Triple.h:766
bool isWasm() const
Tests whether the target is wasm (32- and 64-bit).
Definition: Triple.h:1021
bool isAArch64() const
Tests whether the target is AArch64 (little and big endian).
Definition: Triple.h:911
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
static constexpr TypeSize getFixed(ScalarTy ExactSize)
Definition: TypeSize.h:342
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
bool isVectorTy() const
True if this is an instance of VectorType.
Definition: Type.h:265
bool isPointerTy() const
True if this is an instance of PointerType.
Definition: Type.h:255
static IntegerType * getInt1Ty(LLVMContext &C)
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
TypeID
Definitions of all of the base types for the Type system.
Definition: Type.h:54
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
static Type * getVoidTy(LLVMContext &C)
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition: Type.h:129
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition: Type.h:228
bool isTokenTy() const
Return true if this is 'token'.
Definition: Type.h:225
bool isFPOrFPVectorTy() const
Return true if this is a FP type or a vector of FP.
Definition: Type.h:216
bool isVoidTy() const
Return true if this is 'void'.
Definition: Type.h:140
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition: Type.h:348
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
Definition: Constants.cpp:1808
This function has undefined behavior.
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
Value * getOperand(unsigned i) const
Definition: User.h:169
This class represents the va_arg llvm instruction, which returns an argument of the specified type gi...
CmpInst::Predicate getPredicate() const
This is the common base class for vector predication intrinsics.
static std::optional< unsigned > getVectorLengthParamPos(Intrinsic::ID IntrinsicID)
MaybeAlign getPointerAlignment() const
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition: Value.h:434
iterator_range< user_iterator > users()
Definition: Value.h:421
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:1074
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:309
Base class of all SIMD vector types.
Definition: DerivedTypes.h:403
Type * getElementType() const
Definition: DerivedTypes.h:436
constexpr ScalarTy getFixedValue() const
Definition: TypeSize.h:199
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
Definition: TypeSize.h:171
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
Definition: TypeSize.h:168
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: Lint.cpp:86
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:121
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
@ AnyReg
OBSOLETED - Used for stack based JavaScript calls.
Definition: CallingConv.h:60
@ AMDGPU_CS_Chain
Used on AMDGPUs to give the middle-end more control over argument placement.
Definition: CallingConv.h:245
@ X86_VectorCall
MSVC calling convention that passes vectors and vector aggregates in SSE registers.
Definition: CallingConv.h:163
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:40
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:237
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
Definition: ISDOpcodes.h:1133
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
Definition: ISDOpcodes.h:1129
@ CTLZ_ZERO_UNDEF
Definition: ISDOpcodes.h:724
@ CONVERGENCECTRL_ANCHOR
Definition: ISDOpcodes.h:1397
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
Definition: ISDOpcodes.h:477
@ ATOMIC_LOAD_FMAX
Definition: ISDOpcodes.h:1283
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
Definition: ISDOpcodes.h:44
@ SET_FPENV
Sets the current floating-point environment.
Definition: ISDOpcodes.h:1005
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
Definition: ISDOpcodes.h:1346
@ VECREDUCE_SMIN
Definition: ISDOpcodes.h:1377
@ EH_SJLJ_LONGJMP
OUTCHAIN = EH_SJLJ_LONGJMP(INCHAIN, buffer) This corresponds to the eh.sjlj.longjmp intrinsic.
Definition: ISDOpcodes.h:147
@ ATOMIC_LOAD_NAND
Definition: ISDOpcodes.h:1276
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
Definition: ISDOpcodes.h:560
@ BSWAP
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:715
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
Definition: ISDOpcodes.h:368
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
Definition: ISDOpcodes.h:1162
@ ATOMIC_LOAD_MAX
Definition: ISDOpcodes.h:1278
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
Definition: ISDOpcodes.h:1248
@ ATOMIC_LOAD_UMIN
Definition: ISDOpcodes.h:1279
@ RESET_FPENV
Set floating-point environment to default state.
Definition: ISDOpcodes.h:1009
@ ADD
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:240
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition: ISDOpcodes.h:374
@ SET_FPMODE
Sets the current dynamic floating-point control modes.
Definition: ISDOpcodes.h:1028
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:784
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
Definition: ISDOpcodes.h:484
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
Definition: ISDOpcodes.h:199
@ RETURNADDR
Definition: ISDOpcodes.h:95
@ EH_SJLJ_SETUP_DISPATCH
OUTCHAIN = EH_SJLJ_SETUP_DISPATCH(INCHAIN) The target initializes the dispatch table here.
Definition: ISDOpcodes.h:151
@ GlobalAddress
Definition: ISDOpcodes.h:78
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
Definition: ISDOpcodes.h:1261
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:791
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition: ISDOpcodes.h:544
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
Definition: ISDOpcodes.h:1362
@ FADD
Simple binary floating point operators.
Definition: ISDOpcodes.h:391
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
Definition: ISDOpcodes.h:1366
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
Definition: ISDOpcodes.h:689
@ ATOMIC_FENCE
OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope) This corresponds to the fence instruction.
Definition: ISDOpcodes.h:1240
@ RESET_FPMODE
Sets default dynamic floating-point control modes.
Definition: ISDOpcodes.h:1032
@ VECREDUCE_SMAX
Definition: ISDOpcodes.h:1376
@ STRICT_FSETCCS
Definition: ISDOpcodes.h:478
@ FPTRUNC_ROUND
Definition: ISDOpcodes.h:481
@ ATOMIC_LOAD_OR
Definition: ISDOpcodes.h:1274
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:904
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:230
@ ATOMIC_LOAD_XOR
Definition: ISDOpcodes.h:1275
@ INIT_TRAMPOLINE
INIT_TRAMPOLINE - This corresponds to the init_trampoline intrinsic.
Definition: ISDOpcodes.h:1206
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
Definition: ISDOpcodes.h:940
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
Definition: ISDOpcodes.h:381
@ ATOMIC_LOAD_FADD
Definition: ISDOpcodes.h:1281
@ FrameIndex
Definition: ISDOpcodes.h:80
@ EH_RETURN
OUTCHAIN = EH_RETURN(INCHAIN, OFFSET, HANDLER) - This node represents 'eh_return' gcc dwarf builtin,...
Definition: ISDOpcodes.h:135
@ ANNOTATION_LABEL
ANNOTATION_LABEL - Represents a mid basic block label used by annotations.
Definition: ISDOpcodes.h:1115
@ SET_ROUNDING
Set rounding mode.
Definition: ISDOpcodes.h:886
@ CONVERGENCECTRL_GLUE
Definition: ISDOpcodes.h:1403
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:775
@ PREALLOCATED_SETUP
Definition: ISDOpcodes.h:1167
@ READSTEADYCOUNTER
READSTEADYCOUNTER - This corresponds to the readfixedcounter intrinsic.
Definition: ISDOpcodes.h:1195
@ ADDROFRETURNADDR
ADDROFRETURNADDR - Represents the llvm.addressofreturnaddress intrinsic.
Definition: ISDOpcodes.h:101
@ CONVERGENCECTRL_ENTRY
Definition: ISDOpcodes.h:1398
@ BR
Control flow instructions. These all have token chains.
Definition: ISDOpcodes.h:1054
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
Definition: ISDOpcodes.h:1359
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
Definition: ISDOpcodes.h:723
@ WRITE_REGISTER
Definition: ISDOpcodes.h:119
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
Definition: ISDOpcodes.h:1228
@ VECREDUCE_FMIN
Definition: ISDOpcodes.h:1363
@ ATOMIC_LOAD_FSUB
Definition: ISDOpcodes.h:1282
@ SSUBO
Same for subtraction.
Definition: ISDOpcodes.h:328
@ ATOMIC_LOAD_MIN
Definition: ISDOpcodes.h:1277
@ PREALLOCATED_ARG
Definition: ISDOpcodes.h:1170
@ BRIND
BRIND - Indirect branch.
Definition: ISDOpcodes.h:1059
@ BR_JT
BR_JT - Jumptable branch.
Definition: ISDOpcodes.h:1063
@ VECTOR_INTERLEAVE
VECTOR_INTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the same...
Definition: ISDOpcodes.h:587
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
Definition: ISDOpcodes.h:501
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
Definition: ISDOpcodes.h:508
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
Definition: ISDOpcodes.h:350
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:728
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
Definition: ISDOpcodes.h:1244
@ VECREDUCE_UMAX
Definition: ISDOpcodes.h:1378
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
Definition: ISDOpcodes.h:223
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
Definition: ISDOpcodes.h:628
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
Definition: ISDOpcodes.h:1158
@ ATOMIC_LOAD_FMIN
Definition: ISDOpcodes.h:1284
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
Definition: ISDOpcodes.h:209
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition: ISDOpcodes.h:324
@ ARITH_FENCE
ARITH_FENCE - This corresponds to a arithmetic fence intrinsic.
Definition: ISDOpcodes.h:1232
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
Definition: ISDOpcodes.h:1371
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
Definition: ISDOpcodes.h:881
@ CLEANUPRET
CLEANUPRET - Represents a return from a cleanup block funclet.
Definition: ISDOpcodes.h:1124
@ GET_FPMODE
Reads the current dynamic floating-point control modes.
Definition: ISDOpcodes.h:1023
@ GET_FPENV
Gets the current floating-point environment.
Definition: ISDOpcodes.h:1000
@ SHL
Shift and rotation operations.
Definition: ISDOpcodes.h:706
@ ATOMIC_LOAD_AND
Definition: ISDOpcodes.h:1272
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
Definition: ISDOpcodes.h:574
@ EntryToken
EntryToken - This is the marker used to indicate the start of a region.
Definition: ISDOpcodes.h:47
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
Definition: ISDOpcodes.h:118
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:536
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:781
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
Definition: ISDOpcodes.h:1218
@ FP_TO_UINT_SAT
Definition: ISDOpcodes.h:857
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
Definition: ISDOpcodes.h:1336
@ ATOMIC_LOAD_UMAX
Definition: ISDOpcodes.h:1280
@ LOCAL_RECOVER
LOCAL_RECOVER - Represents the llvm.localrecover intrinsic.
Definition: ISDOpcodes.h:114
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
Definition: ISDOpcodes.h:972
@ UBSANTRAP
UBSANTRAP - Trap with an immediate describing the kind of sanitizer failure.
Definition: ISDOpcodes.h:1222
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
Definition: ISDOpcodes.h:360
@ SMULO
Same for multiplication.
Definition: ISDOpcodes.h:332
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
Definition: ISDOpcodes.h:1048
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
Definition: ISDOpcodes.h:675
@ VECTOR_REVERSE
VECTOR_REVERSE(VECTOR) - Returns a vector, of the same type as VECTOR, whose elements are shuffled us...
Definition: ISDOpcodes.h:592
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition: ISDOpcodes.h:387
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:889
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition: ISDOpcodes.h:737
@ VECREDUCE_UMIN
Definition: ISDOpcodes.h:1379
@ PCMARKER
PCMARKER - This corresponds to the pcmarker intrinsic.
Definition: ISDOpcodes.h:1181
@ INLINEASM_BR
INLINEASM_BR - Branching version of inline asm. Used by asm-goto.
Definition: ISDOpcodes.h:1104
@ EH_DWARF_CFA
EH_DWARF_CFA - This node represents the pointer to the DWARF Canonical Frame Address (CFA),...
Definition: ISDOpcodes.h:129
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
Definition: ISDOpcodes.h:94
@ ATOMIC_LOAD_UDEC_WRAP
Definition: ISDOpcodes.h:1286
@ ATOMIC_LOAD_ADD
Definition: ISDOpcodes.h:1270
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
Definition: ISDOpcodes.h:466
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
Definition: ISDOpcodes.h:991
@ ATOMIC_LOAD_SUB
Definition: ISDOpcodes.h:1271
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:837
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
Definition: ISDOpcodes.h:1189
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:681
@ TRAP
TRAP - Trapping instruction.
Definition: ISDOpcodes.h:1215
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
Definition: ISDOpcodes.h:184
@ VECREDUCE_FMUL
Definition: ISDOpcodes.h:1360
@ STRICT_FADD
Constrained versions of the binary floating point operators.
Definition: ISDOpcodes.h:401
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
Definition: ISDOpcodes.h:525
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition: ISDOpcodes.h:52
@ VECTOR_SPLICE
VECTOR_SPLICE(VEC1, VEC2, IMM) - Returns a subvector of the same type as VEC1/VEC2 from CONCAT_VECTOR...
Definition: ISDOpcodes.h:613
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
Definition: ISDOpcodes.h:1269
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
Definition: ISDOpcodes.h:945
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition: ISDOpcodes.h:870
@ SPONENTRY
SPONENTRY - Represents the llvm.sponentry intrinsic.
Definition: ISDOpcodes.h:106
@ CONVERGENCECTRL_LOOP
Definition: ISDOpcodes.h:1399
@ INLINEASM
INLINEASM - Represents an inline asm block.
Definition: ISDOpcodes.h:1101
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
Definition: ISDOpcodes.h:856
@ VECREDUCE_FMINIMUM
Definition: ISDOpcodes.h:1367
@ EH_SJLJ_SETJMP
RESULT, OUTCHAIN = EH_SJLJ_SETJMP(INCHAIN, buffer) This corresponds to the eh.sjlj....
Definition: ISDOpcodes.h:141
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:787
@ BRCOND
BRCOND - Conditional branch.
Definition: ISDOpcodes.h:1077
@ VECREDUCE_SEQ_FMUL
Definition: ISDOpcodes.h:1347
@ CATCHRET
CATCHRET - Represents a return from a catch block funclet.
Definition: ISDOpcodes.h:1120
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition: ISDOpcodes.h:61
@ ATOMIC_LOAD_UINC_WRAP
Definition: ISDOpcodes.h:1285
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition: ISDOpcodes.h:494
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
Definition: ISDOpcodes.h:341
@ AssertZext
Definition: ISDOpcodes.h:62
@ VECTOR_DEINTERLEAVE
VECTOR_DEINTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the sa...
Definition: ISDOpcodes.h:581
@ GET_DYNAMIC_AREA_OFFSET
GET_DYNAMIC_AREA_OFFSET - get offset from native SP to the address of the most recent dynamic alloca.
Definition: ISDOpcodes.h:1327
@ ADJUST_TRAMPOLINE
ADJUST_TRAMPOLINE - This corresponds to the adjust_trampoline intrinsic.
Definition: ISDOpcodes.h:1212
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
Definition: ISDOpcodes.h:192
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition: ISDOpcodes.h:516
MemIndexType
MemIndexType enum - This enum defines how to interpret MGATHER/SCATTER's index parameter when calcula...
Definition: ISDOpcodes.h:1492
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1530
Flag
These should be considered private to the implementation of the MCInstrDesc class.
Definition: MCInstrDesc.h:148
bool match(Val *V, const Pattern &P)
Definition: PatternMatch.h:49
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
Definition: PatternMatch.h:875
TwoOps_match< Val_t, Idx_t, Instruction::ExtractElement > m_ExtractElt(const Val_t &Val, const Idx_t &Idx)
Matches ExtractElementInst.
OneUse_match< T > m_OneUse(const T &SubPattern)
Definition: PatternMatch.h:67
auto m_LogicalOr()
Matches L || R where L and R are arbitrary values.
VScaleVal_match m_VScale()
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
Definition: PatternMatch.h:92
auto m_LogicalAnd()
Matches L && R where L and R are arbitrary values.
BinaryOp_match< cst_pred_ty< is_all_ones >, ValTy, Instruction::Xor, true > m_Not(const ValTy &V)
Matches a 'Not' as 'xor V, -1' or 'xor -1, V'.
Offsets
Offsets in bytes from the start of the input buffer.
Definition: SIInstrInfo.h:1542
std::vector< CaseCluster > CaseClusterVector
void sortAndRangeify(CaseClusterVector &Clusters)
Sort Clusters and merge adjacent cases.
CaseClusterVector::iterator CaseClusterIt
std::pair< JumpTableHeader, JumpTable > JumpTableBlock
@ CC_Range
A cluster of adjacent case labels with the same destination, or just one case.
@ CC_JumpTable
A cluster of cases suitable for jump table lowering.
@ CC_BitTests
A cluster of cases suitable for bit test lowering.
Reg
All possible values of the reg field in the ModR/M byte.
@ ReallyHidden
Definition: CommandLine.h:139
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:450
LocationClass< Ty > location(Ty &L)
Definition: CommandLine.h:470
@ DW_OP_LLVM_arg
Only used in LLVM metadata.
Definition: Dwarf.h:146
ExceptionBehavior
Exception behavior used for floating point operations.
Definition: FPEnv.h:38
@ ebStrict
This corresponds to "fpexcept.strict".
Definition: FPEnv.h:41
@ ebMayTrap
This corresponds to "fpexcept.maytrap".
Definition: FPEnv.h:40
@ ebIgnore
This corresponds to "fpexcept.ignore".
Definition: FPEnv.h:39
constexpr float log2ef
Definition: MathExtras.h:50
constexpr double e
Definition: MathExtras.h:31
constexpr float ln2f
Definition: MathExtras.h:48
NodeAddr< FuncNode * > Func
Definition: RDFGraph.h:393
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
Definition: MathExtras.h:337
@ Offset
Definition: DWP.cpp:456
@ Length
Definition: DWP.cpp:456
ISD::CondCode getICmpCondCode(ICmpInst::Predicate Pred)
getICmpCondCode - Return the ISD condition code corresponding to the given LLVM IR integer condition ...
Definition: Analysis.cpp:233
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1722
int popcount(T Value) noexcept
Count the number of set bits in a value.
Definition: bit.h:385
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition: STLExtras.h:1680
void GetReturnInfo(CallingConv::ID CC, Type *ReturnType, AttributeList attr, SmallVectorImpl< ISD::OutputArg > &Outs, const TargetLowering &TLI, const DataLayout &DL)
Given an LLVM IR type and return type attributes, compute the return value EVTs and flags,...
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
Definition: MathExtras.h:239
bool isOnlyUsedInZeroEqualityComparison(const Instruction *CxtI)
SDValue peekThroughBitcasts(SDValue V)
Return the non-bitcasted source operand of V if it exists.
@ Done
Definition: Threading.h:61
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
Definition: bit.h:307
void diagnoseDontCall(const CallInst &CI)
auto successors(const MachineBasicBlock *BB)
bool isIntOrFPConstant(SDValue V)
Return true if V is either a integer or FP constant.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
Definition: STLExtras.h:2073
Value * GetPointerBaseWithConstantOffset(Value *Ptr, int64_t &Offset, const DataLayout &DL, bool AllowNonInbounds=true)
Analyze the specified pointer to see if it can be expressed as a base pointer plus a constant offset.
gep_type_iterator gep_type_end(const User *GEP)
ConstantRange getConstantRangeFromMetadata(const MDNode &RangeMD)
Parse out a conservative ConstantRange from !range metadata.
bool isScopedEHPersonality(EHPersonality Pers)
Returns true if this personality uses scope-style EH IR instructions: catchswitch,...
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Definition: bit.h:215
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1729
llvm::SmallVector< int, 16 > createStrideMask(unsigned Start, unsigned Stride, unsigned VF)
Create a stride shuffle mask.
@ SPF_ABS
Floating point maxnum.
@ SPF_NABS
Absolute value.
@ SPF_FMAXNUM
Floating point minnum.
@ SPF_UMIN
Signed minimum.
@ SPF_UMAX
Signed maximum.
@ SPF_SMAX
Unsigned minimum.
@ SPF_FMINNUM
Unsigned maximum.
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
detail::zippy< detail::zip_first, T, U, Args... > zip_first(T &&t, U &&u, Args &&...args)
zip iterator that, for the sake of efficiency, assumes the first iteratee to be the shortest.
Definition: STLExtras.h:876
void sort(IteratorTy Start, IteratorTy End)
Definition: STLExtras.h:1647
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
SelectPatternResult matchSelectPattern(Value *V, Value *&LHS, Value *&RHS, Instruction::CastOps *CastOp=nullptr, unsigned Depth=0)
Pattern match integer [SU]MIN, [SU]MAX and ABS idioms, returning the kind and providing the out param...
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:156
bool hasSingleElement(ContainerTy &&C)
Returns true if the given container only contains a single element.
Definition: STLExtras.h:322
ConstantRange getVScaleRange(const Function *F, unsigned BitWidth)
Determine the possible constant range of vscale with the given bit width, based on the vscale_range f...
ISD::CondCode getFCmpCondCode(FCmpInst::Predicate Pred)
getFCmpCondCode - Return the ISD condition code corresponding to the given LLVM IR floating-point con...
Definition: Analysis.cpp:199
EHPersonality classifyEHPersonality(const Value *Pers)
See if the given exception handling personality function is one that we understand.
Value * salvageDebugInfoImpl(Instruction &I, uint64_t CurrentLocOps, SmallVectorImpl< uint64_t > &Ops, SmallVectorImpl< Value * > &AdditionalValues)
Definition: Local.cpp:2541
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
@ Global
Append to llvm.global_dtors.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
bool isFuncletEHPersonality(EHPersonality Pers)
Returns true if this is a personality function that invokes handler funclets (which must return to it...
void getUnderlyingObjects(const Value *V, SmallVectorImpl< const Value * > &Objects, LoopInfo *LI=nullptr, unsigned MaxLookup=6)
This method is similar to getUnderlyingObject except that it can look through phi and select instruct...
bool isAssignmentTrackingEnabled(const Module &M)
Return true if assignment tracking is enabled for module M.
Definition: DebugInfo.cpp:2430
llvm::SmallVector< int, 16 > createInterleaveMask(unsigned VF, unsigned NumVecs)
Create an interleave shuffle mask.
@ UMin
Unsigned integer min implemented in terms of select(cmp()).
@ Or
Bitwise or logical OR of integers.
@ Mul
Product of integers.
@ And
Bitwise or logical AND of integers.
@ Add
Sum of integers.
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Definition: Alignment.h:155
@ SPNB_RETURNS_NAN
NaN behavior not applicable.
@ SPNB_RETURNS_OTHER
Given one NaN input, returns the NaN.
@ SPNB_RETURNS_ANY
Given one NaN input, returns the non-NaN.
DWARFExpression::Operation Op
ISD::CondCode getFCmpCodeWithoutNaN(ISD::CondCode CC)
getFCmpCodeWithoutNaN - Given an ISD condition code comparing floats, return the equivalent code if w...
Definition: Analysis.cpp:221
void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl< EVT > &ValueVTs, SmallVectorImpl< EVT > *MemVTs, SmallVectorImpl< TypeSize > *Offsets=nullptr, TypeSize StartingOffset=TypeSize::getZero())
ComputeValueVTs - Given an LLVM IR type, compute a sequence of EVTs that represent all the individual...
Definition: Analysis.cpp:79
bool isAsynchronousEHPersonality(EHPersonality Pers)
Returns true if this personality function catches asynchronous exceptions.
std::optional< RoundingMode > convertStrToRoundingMode(StringRef)
Returns a valid RoundingMode enumerator when given a string that is valid as input in constrained int...
Definition: FPEnv.cpp:24
gep_type_iterator gep_type_begin(const User *GEP)
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
Definition: STLExtras.h:2051
GlobalValue * ExtractTypeInfo(Value *V)
ExtractTypeInfo - Returns the type info, possibly bitcast, encoded in V.
Definition: Analysis.cpp:177
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
Definition: Alignment.h:212
bool isInTailCallPosition(const CallBase &Call, const TargetMachine &TM)
Test if the given instruction is in a position to be optimized with a tail-call.
Definition: Analysis.cpp:535
bool all_equal(std::initializer_list< T > Values)
Returns true if all Values in the initializer lists are equal or the list.
Definition: STLExtras.h:2039
Constant * ConstantFoldLoadFromConstPtr(Constant *C, Type *Ty, APInt Offset, const DataLayout &DL)
Return the value that a load from C with offset Offset would produce if it is constant and determinab...
uint64_t alignDown(uint64_t Value, uint64_t Align, uint64_t Skew=0)
Returns the largest uint64_t less than or equal to Value and is Skew mod Align.
Definition: MathExtras.h:439
unsigned succ_size(const MachineBasicBlock *BB)
unsigned ComputeLinearIndex(Type *Ty, const unsigned *Indices, const unsigned *IndicesEnd, unsigned CurIndex=0)
Compute the linearized index of a member in a nested aggregate/struct/array.
Definition: Analysis.cpp:33
T bit_floor(T Value)
Returns the largest integral power of two no greater than Value if Value is nonzero.
Definition: bit.h:327
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
#define N
#define NC
Definition: regutils.h:42
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:760
static const fltSemantics & IEEEsingle() LLVM_READNONE
Definition: APFloat.cpp:249
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
uint64_t value() const
This is a hole in the type system and should not be abused.
Definition: Alignment.h:85
Extended Value Type.
Definition: ValueTypes.h:34
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition: ValueTypes.h:380
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
Definition: ValueTypes.h:73
uint64_t getScalarStoreSize() const
Definition: ValueTypes.h:387
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
Definition: ValueTypes.h:274
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition: ValueTypes.h:290
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
Definition: ValueTypes.h:146
ElementCount getVectorElementCount() const
Definition: ValueTypes.h:340
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:358
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
Definition: ValueTypes.h:349
uint64_t getScalarSizeInBits() const
Definition: ValueTypes.h:370
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:306
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:64
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
Definition: ValueTypes.h:366
bool isFixedLengthVector() const
Definition: ValueTypes.h:177
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:167
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition: ValueTypes.h:313
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
Definition: ValueTypes.h:282
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
Definition: ValueTypes.cpp:202
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
Definition: ValueTypes.h:173
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:318
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
Definition: ValueTypes.h:156
EVT changeVectorElementType(EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
Definition: ValueTypes.h:101
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:326
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
Definition: ValueTypes.h:438
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:151
void setPointerAddrSpace(unsigned AS)
InputArg - This struct carries flags and type information about a single incoming (formal) argument o...
static const unsigned NoArgIndex
Sentinel value for implicit machine-level input arguments.
OutputArg - This struct carries flags and a value for a single outgoing (actual) argument or outgoing...
ConstraintPrefix Type
Type - The basic type of the constraint: input/output/clobber/label.
Definition: InlineAsm.h:126
unsigned countMinLeadingZeros() const
Returns the minimum number of leading zero bits.
Definition: KnownBits.h:244
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getUnknownStack(MachineFunction &MF)
Stack memory without other information.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition: Alignment.h:117
Align valueOrOne() const
For convenience, returns a valid alignment or 1 if undefined.
Definition: Alignment.h:141
This struct represents the registers (physical or virtual) that a particular set of values is assigne...
SmallVector< unsigned, 4 > Regs
This list holds the registers assigned to the values.
RegsForValue()=default
SmallVector< unsigned, 4 > RegCount
This list holds the number of registers for each value.
SmallVector< EVT, 4 > ValueVTs
The value types of the values, which may not be legal, and may need be promoted or synthesized from o...
SmallVector< std::pair< unsigned, TypeSize >, 4 > getRegsAndSizes() const
Return a list of registers and their sizes.
void AddInlineAsmOperands(InlineAsm::Kind Code, bool HasMatching, unsigned MatchingIdx, const SDLoc &dl, SelectionDAG &DAG, std::vector< SDValue > &Ops) const
Add this value to the specified inlineasm node operand list.
SDValue getCopyFromRegs(SelectionDAG &DAG, FunctionLoweringInfo &FuncInfo, const SDLoc &dl, SDValue &Chain, SDValue *Glue, const Value *V=nullptr) const
Emit a series of CopyFromReg nodes that copies from this value and returns the result as a ValueVTs v...
SmallVector< MVT, 4 > RegVTs
The value types of the registers.
void getCopyToRegs(SDValue Val, SelectionDAG &DAG, const SDLoc &dl, SDValue &Chain, SDValue *Glue, const Value *V=nullptr, ISD::NodeType PreferredExtendType=ISD::ANY_EXTEND) const
Emit a series of CopyToReg nodes that copies the specified value into the registers specified by this...
std::optional< CallingConv::ID > CallConv
Records if this value needs to be treated in an ABI dependant manner, different to normal type legali...
bool occupiesMultipleRegs() const
Check if the total RegCount is greater than one.
These are IR-level optimization flags that may be propagated to SDNodes.
void copyFMF(const FPMathOperator &FPMO)
Propagate the fast-math-flags from an IR FPMathOperator.
bool hasAllowReassociation() const
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
A MapVector that performs no allocations if smaller than a certain size.
Definition: MapVector.h:254
This structure is used to communicate between SelectionDAGBuilder and SDISel for the code generation ...
SDLoc DL
The debug location of the instruction this CaseBlock was produced from.
A cluster of case labels.
static CaseCluster range(const ConstantInt *Low, const ConstantInt *High, MachineBasicBlock *MBB, BranchProbability Prob)
This contains information for each constraint that we are lowering.
TargetLowering::ConstraintType ConstraintType
Information about the constraint code, e.g.
This structure contains all information that is necessary for lowering calls.
CallLoweringInfo & setConvergent(bool Value=true)
CallLoweringInfo & setCFIType(const ConstantInt *Type)
SmallVector< ISD::InputArg, 32 > Ins
CallLoweringInfo & setDiscardResult(bool Value=true)
CallLoweringInfo & setIsPatchPoint(bool Value=true)
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
CallLoweringInfo & setTailCall(bool Value=true)
CallLoweringInfo & setIsPreallocated(bool Value=true)
CallLoweringInfo & setConvergenceControlToken(SDValue Token)
SmallVector< ISD::OutputArg, 32 > Outs
SmallVector< SDValue, 32 > OutVals
CallLoweringInfo & setChain(SDValue InChain)
CallLoweringInfo & setCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList, AttributeSet ResultAttrs={})
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setDiscardResult(bool Value=true)
void addIPToStateRange(const InvokeInst *II, MCSymbol *InvokeBegin, MCSymbol *InvokeEnd)